diff --git a/conf/springConfigXml/Kvm.xml b/conf/springConfigXml/Kvm.xml
index 5a9f5e18a5..2b88e17af4 100755
--- a/conf/springConfigXml/Kvm.xml
+++ b/conf/springConfigXml/Kvm.xml
@@ -288,6 +288,7 @@
+
diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java
index cc7c9083a7..f5b4f03de2 100755
--- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java
+++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java
@@ -3421,6 +3421,7 @@ public static class StopVmCmd extends AgentCommand {
private long timeout;
private boolean forceStopIfNoOperatingSystemDetected;
private List vmNics;
+ private List tpmBackupJobs;
public String getUuid() {
return uuid;
@@ -3461,6 +3462,14 @@ public List getVmNics() {
public void setVmNics(List vmNics) {
this.vmNics = vmNics;
}
+
+ public List getTpmBackupJobs() {
+ return tpmBackupJobs;
+ }
+
+ public void setTpmBackupJobs(List tpmBackupJobs) {
+ this.tpmBackupJobs = tpmBackupJobs;
+ }
}
public static class StopVmResponse extends AgentResponse {
diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/efi/KvmSecureBootExtensions.java b/plugin/kvm/src/main/java/org/zstack/kvm/efi/KvmSecureBootExtensions.java
index 27671066e2..092f20ff96 100644
--- a/plugin/kvm/src/main/java/org/zstack/kvm/efi/KvmSecureBootExtensions.java
+++ b/plugin/kvm/src/main/java/org/zstack/kvm/efi/KvmSecureBootExtensions.java
@@ -43,6 +43,7 @@
import org.zstack.header.vm.additions.VmHostBackupFileVO_;
import org.zstack.header.vm.additions.VmHostFileContentVO;
import org.zstack.header.vm.additions.VmHostFileContentVO_;
+import org.zstack.header.vm.additions.VmHostFileBackupJob;
import org.zstack.header.vm.additions.VmHostFileOperation;
import org.zstack.header.vm.additions.VmHostFileType;
import org.zstack.header.vm.additions.VmHostFileVO;
@@ -57,9 +58,11 @@
import org.zstack.header.volume.VolumeInventory;
import org.zstack.kvm.KVMAgentCommands;
import org.zstack.kvm.KVMAgentCommands.*;
+import org.zstack.kvm.KVMException;
import org.zstack.kvm.KVMGlobalConfig;
import org.zstack.kvm.KVMHostInventory;
import org.zstack.kvm.KVMStartVmExtensionPoint;
+import org.zstack.kvm.KVMStopVmExtensionPoint;
import org.zstack.kvm.KvmCommandSender;
import org.zstack.kvm.KvmResponseWrapper;
import org.zstack.kvm.VolumeTO;
@@ -93,7 +96,8 @@ public class KvmSecureBootExtensions implements KVMStartVmExtensionPoint,
VmInstanceMigrateExtensionPoint,
VolumeSnapshotCreationExtensionPoint,
BeforeHaStartVmInstanceExtensionPoint,
- ConvertVmInstanceToTemplatedVmExtensionPoint {
+ ConvertVmInstanceToTemplatedVmExtensionPoint,
+ KVMStopVmExtensionPoint {
private static final CLogger logger = Utils.getLogger(KvmSecureBootExtensions.class);
@Autowired
@@ -611,6 +615,39 @@ public void afterReimageVmInstance(VolumeInventory inventory) {
"deleted all VmHostFileVO and VmHostBackupFileVO records", vmUuid));
}
+ @Override
+ public void beforeStopVmOnKvm(KVMHostInventory host, VmInstanceInventory vm,
+ KVMAgentCommands.StopVmCmd cmd) throws KVMException {
+ String vmUuid = vm.getUuid();
+ String hostUuid = host.getUuid();
+
+ VmHostFileVO tpmFile = Q.New(VmHostFileVO.class)
+ .eq(VmHostFileVO_.vmInstanceUuid, vmUuid)
+ .eq(VmHostFileVO_.hostUuid, hostUuid)
+ .eq(VmHostFileVO_.type, VmHostFileType.TpmState)
+ .find();
+ if (tpmFile == null) {
+ return;
+ }
+
+ VmHostFileBackupJob job = new VmHostFileBackupJob();
+ job.setSrcPath(tpmFile.getPath());
+ job.setDestPath(buildTpmStateSnapshotBackupFilePath(vmUuid));
+ job.setType(VmHostFileType.TpmState.toString());
+ cmd.setTpmBackupJobs(Collections.singletonList(job));
+
+ logger.debug(String.format("set TPM backup jobs on StopVmCmd for VM[uuid:%s]: %s -> %s",
+ vmUuid, job.getSrcPath(), job.getDestPath()));
+ }
+
+ @Override
+ public void stopVmOnKvmSuccess(KVMHostInventory host, VmInstanceInventory vm) {
+ }
+
+ @Override
+ public void stopVmOnKvmFailed(KVMHostInventory host, VmInstanceInventory vm, ErrorCode err) {
+ }
+
@Override
public void releaseVmResource(VmInstanceSpec spec, Completion completion) {
if (spec.getDestHost() == null) {
@@ -640,6 +677,7 @@ public void releaseVmResource(VmInstanceSpec spec, Completion completion) {
syncMsg.setNvRamPath(file.getPath());
} else if (file.getType() == VmHostFileType.TpmState) {
syncMsg.setTpmStateFolder(file.getPath());
+ syncMsg.setTpmStateFallbackFolder(buildTpmStateSnapshotBackupFilePath(vmUuid));
} else {
logger.warn(String.format("unsupported vm host file type: %s, skip syncing for VM[uuid:%s] from host[uuid:%s]",
file.getType(), vmUuid, hostUuid));
@@ -699,6 +737,7 @@ public void beforeHaStartVmInstance(String vmUuid, String judgerClassName, List<
syncMsg.setNvRamPath(file.getPath());
} else if (file.getType() == VmHostFileType.TpmState) {
syncMsg.setTpmStateFolder(file.getPath());
+ syncMsg.setTpmStateFallbackFolder(buildTpmStateSnapshotBackupFilePath(vmUuid));
} else {
logger.warn(String.format(
"unsupported vm host file type: %s, skip syncing for VM[uuid:%s] from host[uuid:%s]",
diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/efi/KvmSecureBootManager.java b/plugin/kvm/src/main/java/org/zstack/kvm/efi/KvmSecureBootManager.java
index e34ec11d3b..4a71991548 100644
--- a/plugin/kvm/src/main/java/org/zstack/kvm/efi/KvmSecureBootManager.java
+++ b/plugin/kvm/src/main/java/org/zstack/kvm/efi/KvmSecureBootManager.java
@@ -279,60 +279,287 @@ static class CloneVmHostFileContext {
List syncContexts = new ArrayList<>();
}
- private void handle(SyncVmHostFilesFromHostMsg msg) {
- KvmCommandSender sender = new KvmCommandSender(msg.getHostUuid())
- .disableHostStatusCheck();
+ private static class SyncVmHostFilesContext {
+ KVMAgentCommands.ReadVmHostFileContentCmd cmd;
+ KVMAgentCommands.ReadVmHostFileContentResponse readRsp;
+ KVMAgentCommands.ReadVmHostFileContentResponse fallbackRsp;
+ long timeBeforeSync;
+ boolean needTpmFallback;
+ boolean tpmFallbackUsed;
+ }
- KVMAgentCommands.ReadVmHostFileContentCmd cmd = new KVMAgentCommands.ReadVmHostFileContentCmd();
- cmd.setHostFiles(new ArrayList<>());
+ private void handle(SyncVmHostFilesFromHostMsg msg) {
+ SyncVmHostFilesContext ctx = new SyncVmHostFilesContext();
+ ctx.cmd = new KVMAgentCommands.ReadVmHostFileContentCmd();
+ ctx.cmd.setHostFiles(new ArrayList<>());
if (msg.getTpmStateFolder() != null) {
KVMAgentCommands.VmHostFileTO to = new KVMAgentCommands.VmHostFileTO();
to.setPath(msg.getTpmStateFolder());
to.setType(VmHostFileType.TpmState.toString());
- cmd.getHostFiles().add(to);
+ ctx.cmd.getHostFiles().add(to);
}
if (msg.getNvRamPath() != null) {
KVMAgentCommands.VmHostFileTO to = new KVMAgentCommands.VmHostFileTO();
to.setPath(msg.getNvRamPath());
to.setType(VmHostFileType.NvRam.toString());
- cmd.getHostFiles().add(to);
+ ctx.cmd.getHostFiles().add(to);
}
- long now = timeHelper.getCurrentTimeMillis();
+ ctx.timeBeforeSync = timeHelper.getCurrentTimeMillis();
SyncVmHostFilesFromHostReply reply = new SyncVmHostFilesFromHostReply();
- sender.send(cmd, READ_VM_HOST_FILE_PATH, wrapper -> {
- KVMAgentCommands.ReadVmHostFileContentResponse readRsp = wrapper.getResponse(KVMAgentCommands.ReadVmHostFileContentResponse.class);
- return readRsp.isSuccess() ? null :
- operr("failed to read file content response").withException(readRsp.getError());
- }, new ReturnValueCompletion(msg) {
- @Override
- public void success(KvmResponseWrapper wrapper) {
- KVMAgentCommands.ReadVmHostFileContentResponse readRsp = wrapper.getResponse(KVMAgentCommands.ReadVmHostFileContentResponse.class);
- if (!readRsp.isSuccess()) {
- reply.setError(operr("failed to read file content response").withException(readRsp.getError()));
+
+ SimpleFlowChain.of("sync-vm-host-files-" + msg.getVmUuid())
+ .then("read-from-primary-path", trigger -> {
+ KvmCommandSender sender = new KvmCommandSender(msg.getHostUuid())
+ .disableHostStatusCheck();
+ sender.send(ctx.cmd, READ_VM_HOST_FILE_PATH, wrapper -> {
+ KVMAgentCommands.ReadVmHostFileContentResponse rsp =
+ wrapper.getResponse(KVMAgentCommands.ReadVmHostFileContentResponse.class);
+ return rsp.isSuccess() ? null :
+ operr("failed to read file content response").withException(rsp.getError());
+ }, new ReturnValueCompletion(trigger) {
+ @Override
+ public void success(KvmResponseWrapper wrapper) {
+ ctx.readRsp = wrapper.getResponse(
+ KVMAgentCommands.ReadVmHostFileContentResponse.class);
+ trigger.next();
+ }
+
+ @Override
+ public void fail(ErrorCode errorCode) {
+ boolean hasTpmFallback = msg.getTpmStateFallbackFolder() != null
+ && msg.getTpmStateFolder() != null;
+ if (hasTpmFallback) {
+ logger.info(String.format(
+ "read from host failed entirely for VM[uuid=%s], " +
+ "will try TPM fallback path [%s]",
+ msg.getVmUuid(), msg.getTpmStateFallbackFolder()));
+ ctx.readRsp = null;
+ trigger.next();
+ } else {
+ trigger.fail(errorCode);
+ }
+ }
+ });
+ })
+ .then("fallback-read-tpm-from-backup", trigger -> {
+ if (msg.getTpmStateFallbackFolder() == null || msg.getTpmStateFolder() == null) {
+ trigger.next();
+ return;
+ }
+
+ // check whether TPM part specifically failed
+ boolean tpmReadFailed;
+ if (ctx.readRsp == null || !ctx.readRsp.isSuccess()) {
+ tpmReadFailed = true;
+ } else {
+ KVMAgentCommands.VmHostFileTO tpmResult = findOneOrNull(
+ ctx.readRsp.getHostFiles(),
+ item -> msg.getTpmStateFolder().equals(item.getPath()));
+ tpmReadFailed = tpmResult == null
+ || tpmResult.getError() != null
+ || tpmResult.getContentBase64() == null;
+ }
+
+ if (!tpmReadFailed) {
+ trigger.next();
+ return;
+ }
+
+ ctx.needTpmFallback = true;
+ logger.info(String.format(
+ "TPM state read failed from primary path [%s] for VM[uuid=%s], " +
+ "trying fallback path [%s]",
+ msg.getTpmStateFolder(), msg.getVmUuid(),
+ msg.getTpmStateFallbackFolder()));
+
+ KvmCommandSender sender = new KvmCommandSender(msg.getHostUuid())
+ .disableHostStatusCheck();
+ KVMAgentCommands.ReadVmHostFileContentCmd fallbackCmd =
+ new KVMAgentCommands.ReadVmHostFileContentCmd();
+ KVMAgentCommands.VmHostFileTO fallbackTo = new KVMAgentCommands.VmHostFileTO();
+ fallbackTo.setPath(msg.getTpmStateFallbackFolder());
+ fallbackTo.setType(VmHostFileType.TpmState.toString());
+ fallbackCmd.setHostFiles(Collections.singletonList(fallbackTo));
+
+ sender.send(fallbackCmd, READ_VM_HOST_FILE_PATH, wrapper -> {
+ KVMAgentCommands.ReadVmHostFileContentResponse rsp =
+ wrapper.getResponse(KVMAgentCommands.ReadVmHostFileContentResponse.class);
+ return rsp.isSuccess() ? null :
+ operr("failed to read TPM from fallback path")
+ .withException(rsp.getError());
+ }, new ReturnValueCompletion(trigger) {
+ @Override
+ public void success(KvmResponseWrapper wrapper) {
+ ctx.fallbackRsp = wrapper.getResponse(
+ KVMAgentCommands.ReadVmHostFileContentResponse.class);
+ trigger.next();
+ }
+
+ @Override
+ public void fail(ErrorCode errorCode) {
+ logger.warn(String.format(
+ "TPM fallback read also failed for VM[uuid=%s] " +
+ "from [%s]: %s",
+ msg.getVmUuid(), msg.getTpmStateFallbackFolder(),
+ errorCode.getReadableDetails()));
+ trigger.fail(errorCode);
+ }
+ });
+ })
+ .then("merge-tpm-fallback-result", trigger -> {
+ if (!ctx.needTpmFallback) {
+ trigger.next();
+ return;
+ }
+
+ if (ctx.fallbackRsp == null || !ctx.fallbackRsp.isSuccess()) {
+ trigger.fail(operr("failed to read TPM from fallback path [%s]",
+ msg.getTpmStateFallbackFolder())
+ .withException(ctx.fallbackRsp == null ? null
+ : ctx.fallbackRsp.getError()));
+ return;
+ }
+
+ // validate fallback result
+ KVMAgentCommands.VmHostFileTO fallbackResult = findOneOrNull(
+ ctx.fallbackRsp.getHostFiles(),
+ item -> VmHostFileType.TpmState.toString().equals(item.getType()));
+ if (fallbackResult == null
+ || fallbackResult.getError() != null
+ || fallbackResult.getContentBase64() == null) {
+ String detail = fallbackResult == null ? "no result" :
+ (fallbackResult.getError() != null
+ ? fallbackResult.getError() : "empty content");
+ trigger.fail(operr(
+ "TPM fallback read from [%s] returned no valid content: %s",
+ msg.getTpmStateFallbackFolder(), detail));
+ return;
+ }
+
+ logger.info(String.format(
+ "successfully read TPM state from fallback [%s] for VM[uuid=%s]",
+ msg.getTpmStateFallbackFolder(), msg.getVmUuid()));
+
+ // rewrite path to canonical so DB stores the correct path
+ fallbackResult.setPath(msg.getTpmStateFolder());
+
+ // merge: NvRam from original response + TPM from fallback
+ KVMAgentCommands.ReadVmHostFileContentResponse mergedRsp =
+ new KVMAgentCommands.ReadVmHostFileContentResponse();
+ mergedRsp.setSuccess(true);
+ List mergedFiles = new ArrayList<>();
+ if (ctx.readRsp != null && ctx.readRsp.getHostFiles() != null) {
+ for (KVMAgentCommands.VmHostFileTO f : ctx.readRsp.getHostFiles()) {
+ if (!VmHostFileType.TpmState.toString().equals(f.getType())) {
+ mergedFiles.add(f);
+ }
+ }
+ }
+ mergedFiles.add(fallbackResult);
+ mergedRsp.setHostFiles(mergedFiles);
+ ctx.readRsp = mergedRsp;
+
+ // rebuild cmd to match merged response
+ KVMAgentCommands.ReadVmHostFileContentCmd mergedCmd =
+ new KVMAgentCommands.ReadVmHostFileContentCmd();
+ List cmdFiles = new ArrayList<>();
+ if (msg.getNvRamPath() != null) {
+ KVMAgentCommands.VmHostFileTO nvTo = new KVMAgentCommands.VmHostFileTO();
+ nvTo.setPath(msg.getNvRamPath());
+ nvTo.setType(VmHostFileType.NvRam.toString());
+ cmdFiles.add(nvTo);
+ }
+ KVMAgentCommands.VmHostFileTO tpmTo = new KVMAgentCommands.VmHostFileTO();
+ tpmTo.setPath(msg.getTpmStateFolder());
+ tpmTo.setType(VmHostFileType.TpmState.toString());
+ cmdFiles.add(tpmTo);
+ mergedCmd.setHostFiles(cmdFiles);
+ ctx.cmd = mergedCmd;
+
+ ctx.tpmFallbackUsed = true;
+ trigger.next();
+ })
+ .then("persist-to-db", trigger -> {
+ if (ctx.readRsp == null || !ctx.readRsp.isSuccess()) {
+ reply.setError(operr(
+ "no valid read response to persist for VM[uuid=%s]",
+ msg.getVmUuid()));
+ trigger.next();
+ return;
+ }
+
+ ErrorCode error;
+ if (msg.isSyncToBackup()) {
+ error = syncToBackupFiles(msg, ctx.readRsp);
+ } else {
+ error = syncToHostFiles(msg, ctx.cmd, ctx.readRsp,
+ ctx.timeBeforeSync);
+ }
+
+ if (error != null) {
+ reply.setError(error);
+ }
+ trigger.next();
+ })
+ .then("cleanup-tpm-backup", trigger -> {
+ if (!ctx.tpmFallbackUsed) {
+ trigger.next();
+ return;
+ }
+
+ cleanupTpmBackupOnHost(msg.getHostUuid(), msg.getVmUuid(),
+ msg.getTpmStateFallbackFolder());
+ trigger.next();
+ })
+ .propagateExceptionTo(msg)
+ .done(() -> bus.reply(msg, reply))
+ .error(errorCode -> {
+ reply.setError(errorCode);
bus.reply(msg, reply);
- return;
- }
+ })
+ .start();
+ }
- ErrorCode error;
- if (msg.isSyncToBackup()) {
- error = syncToBackupFiles(msg, readRsp);
- } else {
- error = syncToHostFiles(msg, cmd, readRsp, now);
+ private void cleanupTpmBackupOnHost(String hostUuid, String vmUuid, String backupPath) {
+ try {
+ KvmCommandSender sender = new KvmCommandSender(hostUuid)
+ .disableHostStatusCheck();
+
+ KVMAgentCommands.VmHostFileTO deleteTo = new KVMAgentCommands.VmHostFileTO();
+ deleteTo.setPath(backupPath);
+ deleteTo.setType(VmHostFileType.TpmState.toString());
+ deleteTo.setOperation(VmHostFileOperation.Delete.toString());
+
+ KVMAgentCommands.WriteVmHostFileContentCmd deleteCmd =
+ new KVMAgentCommands.WriteVmHostFileContentCmd();
+ deleteCmd.setHostFiles(Collections.singletonList(deleteTo));
+
+ sender.send(deleteCmd, WRITE_VM_HOST_FILE_PATH, wrapper -> {
+ KVMAgentCommands.WriteVmHostFileContentResponse rsp =
+ wrapper.getResponse(KVMAgentCommands.WriteVmHostFileContentResponse.class);
+ return rsp.isSuccess() ? null :
+ operr("failed to delete TPM backup").withException(rsp.getError());
+ }, new ReturnValueCompletion(null) {
+ @Override
+ public void success(KvmResponseWrapper wrapper) {
+ logger.debug(String.format(
+ "cleaned up TPM backup at [%s] on host[uuid=%s] for VM[uuid=%s]",
+ backupPath, hostUuid, vmUuid));
}
- if (error != null) {
- reply.setError(error);
+ @Override
+ public void fail(ErrorCode errorCode) {
+ logger.warn(String.format(
+ "failed to clean up TPM backup at [%s] on host[uuid=%s] " +
+ "for VM[uuid=%s]: %s. Will be overwritten on next stop.",
+ backupPath, hostUuid, vmUuid, errorCode.getReadableDetails()));
}
- bus.reply(msg, reply);
- }
-
- @Override
- public void fail(ErrorCode errorCode) {
- reply.setError(errorCode);
- bus.reply(msg, reply);
- }
- });
+ });
+ } catch (Exception e) {
+ logger.warn(String.format("unexpected error cleaning up TPM backup for VM[uuid=%s]: %s",
+ vmUuid, e.getMessage()), e);
+ }
}
private ErrorCode syncToHostFiles(SyncVmHostFilesFromHostMsg msg,
diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/SyncVmHostFilesFromHostMsg.java b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/SyncVmHostFilesFromHostMsg.java
index 93b21797f1..424fe90c07 100644
--- a/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/SyncVmHostFilesFromHostMsg.java
+++ b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/SyncVmHostFilesFromHostMsg.java
@@ -7,6 +7,7 @@ public class SyncVmHostFilesFromHostMsg extends NeedReplyMessage {
private String vmUuid;
private String nvRamPath;
private String tpmStateFolder;
+ private String tpmStateFallbackFolder;
private String syncReason;
private boolean syncToBackup;
private String backupResourceUuid;
@@ -43,6 +44,14 @@ public void setTpmStateFolder(String tpmStateFolder) {
this.tpmStateFolder = tpmStateFolder;
}
+ public String getTpmStateFallbackFolder() {
+ return tpmStateFallbackFolder;
+ }
+
+ public void setTpmStateFallbackFolder(String tpmStateFallbackFolder) {
+ this.tpmStateFallbackFolder = tpmStateFallbackFolder;
+ }
+
public String getSyncReason() {
return syncReason;
}
diff --git a/test/src/test/resources/springConfigXml/Kvm.xml b/test/src/test/resources/springConfigXml/Kvm.xml
index bd33bd2d81..e5fbcbc122 100755
--- a/test/src/test/resources/springConfigXml/Kvm.xml
+++ b/test/src/test/resources/springConfigXml/Kvm.xml
@@ -287,6 +287,7 @@
+