diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java index 9ebb180b055f..7519d7f390c5 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java @@ -98,11 +98,11 @@ public Answer execute(RestoreBackupCommand command, LibvirtComputingResource ser newVolumeId = getVolumeUuidFromPath(volumePath, volumePool); Long size = command.getRestoreVolumeSizes().get(0); restoreVolume(storagePoolMgr, backupPath, volumePool, volumePath, diskType, backupFile, size, - new Pair<>(vmName, command.getVmState()), mountDirectory, timeout); + new Pair<>(vmName, command.getVmState()), mountDirectory, timeout, mountTimeout); } else if (Boolean.TRUE.equals(vmExists)) { - restoreVolumesOfExistingVM(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backedVolumeUUIDs, backupPath, backupFiles, mountDirectory, timeout); + restoreVolumesOfExistingVM(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backedVolumeUUIDs, backupPath, backupFiles, mountDirectory, timeout, mountTimeout); } else { - restoreVolumesOfDestroyedVMs(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backupPath, backupFiles, mountDirectory, timeout); + restoreVolumesOfDestroyedVMs(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backupPath, backupFiles, mountDirectory, timeout, mountTimeout); } } catch (CloudRuntimeException e) { String errorMessage = e.getMessage() != null ? e.getMessage() : ""; @@ -123,7 +123,7 @@ private void verifyBackupFile(String backupPath, String volUuid) { private void restoreVolumesOfExistingVM(KVMStoragePoolManager storagePoolMgr, List restoreVolumePools, List restoreVolumePaths, List backedVolumesUUIDs, - String backupPath, List backupFiles, String mountDirectory, int timeout) { + String backupPath, List backupFiles, String mountDirectory, int timeout, Integer mountTimeout) { String diskType = "root"; try { for (int idx = 0; idx < restoreVolumePaths.size(); idx++) { @@ -140,13 +140,13 @@ private void restoreVolumesOfExistingVM(KVMStoragePoolManager storagePoolMgr, Li } } } finally { - unmountBackupDirectory(mountDirectory); + unmountBackupDirectory(mountDirectory, mountTimeout); deleteTemporaryDirectory(mountDirectory); } } private void restoreVolumesOfDestroyedVMs(KVMStoragePoolManager storagePoolMgr, List volumePools, - List volumePaths, String backupPath, List backupFiles, String mountDirectory, int timeout) { + List volumePaths, String backupPath, List backupFiles, String mountDirectory, int timeout, Integer mountTimeout) { String diskType = "root"; try { for (int i = 0; i < volumePaths.size(); i++) { @@ -162,13 +162,13 @@ private void restoreVolumesOfDestroyedVMs(KVMStoragePoolManager storagePoolMgr, } } } finally { - unmountBackupDirectory(mountDirectory); + unmountBackupDirectory(mountDirectory, mountTimeout); deleteTemporaryDirectory(mountDirectory); } } private void restoreVolume(KVMStoragePoolManager storagePoolMgr, String backupPath, PrimaryDataStoreTO volumePool, String volumePath, String diskType, String backupFile, - Long size, Pair vmNameAndState, String mountDirectory, int timeout) { + Long size, Pair vmNameAndState, String mountDirectory, int timeout, Integer mountTimeout) { String bkpPath; String volumeUuid; try { @@ -185,7 +185,7 @@ private void restoreVolume(KVMStoragePoolManager storagePoolMgr, String backupPa } } } finally { - unmountBackupDirectory(mountDirectory); + unmountBackupDirectory(mountDirectory, mountTimeout); deleteTemporaryDirectory(mountDirectory); } } @@ -201,6 +201,7 @@ private String mountBackupDirectory(String backupRepoAddress, String backupRepoT logger.error("Failed to create the tmp mount directory {} for restore", mountDirectory, e); throw new CloudRuntimeException("Failed to create the tmp mount directory for restore on the KVM host"); } + int exitValue; try { String mountPath = Script.getExecutableAbsolutePath("mount"); List mountCmd = new ArrayList<>(); @@ -221,23 +222,42 @@ private String mountBackupDirectory(String backupRepoAddress, String backupRepoT mountCmd.add("-o"); mountCmd.add(mountOptions); } - Script.executeCommand(mountCmd.toArray(new String[0])); + exitValue = Script.executeCommandForExitValue(mountTimeout, mountCmd.toArray(new String[0])); } catch (Exception e) { logger.error("Failed to mount repository {} of type {} to the directory {}", backupRepoAddress, backupRepoType, mountDirectory, e); throw new CloudRuntimeException("Failed to mount the backup repository on the KVM host"); } + if (exitValue != 0) { + logger.error("Failed to mount repository {} of type {} to the directory {}, mount exited with {}", backupRepoAddress, + backupRepoType, mountDirectory, exitValue); + removeTemporaryDirectoryQuietly(mountDirectory); + throw new CloudRuntimeException("Failed to mount the backup repository on the KVM host"); + } return mountDirectory; } - private void unmountBackupDirectory(String backupDirectory) { + private void unmountBackupDirectory(String backupDirectory, Integer mountTimeout) { + int exitValue; try { String umountPath = Script.getExecutableAbsolutePath("umount"); String[] umountCmd = new String[] { "sudo", umountPath, backupDirectory }; - Script.executeCommand(umountCmd); + exitValue = Script.executeCommandForExitValue(mountTimeout, umountCmd); } catch (Exception e) { logger.error("Failed to unmount backup directory {}", backupDirectory, e); throw new CloudRuntimeException("Failed to unmount the backup directory"); } + if (exitValue != 0) { + logger.error("Failed to unmount backup directory {}, umount exited with {}", backupDirectory, exitValue); + throw new CloudRuntimeException("Failed to unmount the backup directory"); + } + } + + private void removeTemporaryDirectoryQuietly(String backupDirectory) { + try { + Files.deleteIfExists(Paths.get(backupDirectory)); + } catch (IOException e) { + logger.warn("Failed to remove the temporary mount directory {} after the mount failed.", backupDirectory, e); + } } private void deleteTemporaryDirectory(String backupDirectory) { @@ -276,7 +296,7 @@ private boolean replaceVolumeWithBackup(KVMStoragePoolManager storagePoolMgr, Pr } String[] rsyncCmd = new String[] { Script.getExecutableAbsolutePath("rsync"), "-az", backupPath, volumePath }; - int exitValue = Script.executeCommandForExitValue(rsyncCmd); + int exitValue = Script.executeCommandForExitValue(timeout, rsyncCmd); return exitValue == 0; } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java index f028035e8dcc..94aa0920d114 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java @@ -261,8 +261,8 @@ public void testExecuteWithMountFailure() throws Exception { filesMock.when(() -> Files.createTempDirectory(anyString())).thenReturn(tempPath); try (MockedStatic