From 92d9d9c7a7e7815a42a4cd973bd80db6fbf84d9a Mon Sep 17 00:00:00 2001 From: Zhang Wenhao Date: Fri, 24 Apr 2026 07:17:21 +0800 Subject: [PATCH] [kvm]: backup TPM state before virsh undefine virsh undefine deletes /var/lib/libvirt/swtpm/{uuid}/ before releaseVmResource() can sync TPM state to DB. Add tpmBackupJobs to StopVmCmd so the agent backs up TPM files between virsh shutdown and virsh undefine. On sync, if the primary path read fails, fall back to the backup path, merge with NvRam results, persist to DB, then clean up the backup directory. KvmSecureBootExtensions now implements KVMStopVmExtensionPoint to set backup jobs and fallback folder. KvmSecureBootManager.handle( SyncVmHostFilesFromHostMsg) is refactored into a SimpleFlowChain with named steps for readability. Related: ZSV-11310 Resolves: ZSV-12030 Change-Id: I66787076646270756c7971707561687462627772 --- conf/springConfigXml/Kvm.xml | 1 + .../java/org/zstack/kvm/KVMAgentCommands.java | 9 + .../kvm/efi/KvmSecureBootExtensions.java | 41 ++- .../zstack/kvm/efi/KvmSecureBootManager.java | 299 +++++++++++++++--- .../message/SyncVmHostFilesFromHostMsg.java | 9 + .../test/resources/springConfigXml/Kvm.xml | 1 + 6 files changed, 323 insertions(+), 37 deletions(-) diff --git a/conf/springConfigXml/Kvm.xml b/conf/springConfigXml/Kvm.xml index 5a9f5e18a58..2b88e17af4f 100755 --- a/conf/springConfigXml/Kvm.xml +++ b/conf/springConfigXml/Kvm.xml @@ -288,6 +288,7 @@ + diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java index cc7c9083a7a..f5b4f03de2e 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMAgentCommands.java @@ -3421,6 +3421,7 @@ public static class StopVmCmd extends AgentCommand { private long timeout; private boolean forceStopIfNoOperatingSystemDetected; private List vmNics; + private List tpmBackupJobs; public String getUuid() { return uuid; @@ -3461,6 +3462,14 @@ public List getVmNics() { public void setVmNics(List vmNics) { this.vmNics = vmNics; } + + public List getTpmBackupJobs() { + return tpmBackupJobs; + } + + public void setTpmBackupJobs(List tpmBackupJobs) { + this.tpmBackupJobs = tpmBackupJobs; + } } public static class StopVmResponse extends AgentResponse { diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/efi/KvmSecureBootExtensions.java b/plugin/kvm/src/main/java/org/zstack/kvm/efi/KvmSecureBootExtensions.java index 27671066e2c..092f20ff96f 100644 --- a/plugin/kvm/src/main/java/org/zstack/kvm/efi/KvmSecureBootExtensions.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/efi/KvmSecureBootExtensions.java @@ -43,6 +43,7 @@ import org.zstack.header.vm.additions.VmHostBackupFileVO_; import org.zstack.header.vm.additions.VmHostFileContentVO; import org.zstack.header.vm.additions.VmHostFileContentVO_; +import org.zstack.header.vm.additions.VmHostFileBackupJob; import org.zstack.header.vm.additions.VmHostFileOperation; import org.zstack.header.vm.additions.VmHostFileType; import org.zstack.header.vm.additions.VmHostFileVO; @@ -57,9 +58,11 @@ import org.zstack.header.volume.VolumeInventory; import org.zstack.kvm.KVMAgentCommands; import org.zstack.kvm.KVMAgentCommands.*; +import org.zstack.kvm.KVMException; import org.zstack.kvm.KVMGlobalConfig; import org.zstack.kvm.KVMHostInventory; import org.zstack.kvm.KVMStartVmExtensionPoint; +import org.zstack.kvm.KVMStopVmExtensionPoint; import org.zstack.kvm.KvmCommandSender; import org.zstack.kvm.KvmResponseWrapper; import org.zstack.kvm.VolumeTO; @@ -93,7 +96,8 @@ public class KvmSecureBootExtensions implements KVMStartVmExtensionPoint, VmInstanceMigrateExtensionPoint, VolumeSnapshotCreationExtensionPoint, BeforeHaStartVmInstanceExtensionPoint, - ConvertVmInstanceToTemplatedVmExtensionPoint { + ConvertVmInstanceToTemplatedVmExtensionPoint, + KVMStopVmExtensionPoint { private static final CLogger logger = Utils.getLogger(KvmSecureBootExtensions.class); @Autowired @@ -611,6 +615,39 @@ public void afterReimageVmInstance(VolumeInventory inventory) { "deleted all VmHostFileVO and VmHostBackupFileVO records", vmUuid)); } + @Override + public void beforeStopVmOnKvm(KVMHostInventory host, VmInstanceInventory vm, + KVMAgentCommands.StopVmCmd cmd) throws KVMException { + String vmUuid = vm.getUuid(); + String hostUuid = host.getUuid(); + + VmHostFileVO tpmFile = Q.New(VmHostFileVO.class) + .eq(VmHostFileVO_.vmInstanceUuid, vmUuid) + .eq(VmHostFileVO_.hostUuid, hostUuid) + .eq(VmHostFileVO_.type, VmHostFileType.TpmState) + .find(); + if (tpmFile == null) { + return; + } + + VmHostFileBackupJob job = new VmHostFileBackupJob(); + job.setSrcPath(tpmFile.getPath()); + job.setDestPath(buildTpmStateSnapshotBackupFilePath(vmUuid)); + job.setType(VmHostFileType.TpmState.toString()); + cmd.setTpmBackupJobs(Collections.singletonList(job)); + + logger.debug(String.format("set TPM backup jobs on StopVmCmd for VM[uuid:%s]: %s -> %s", + vmUuid, job.getSrcPath(), job.getDestPath())); + } + + @Override + public void stopVmOnKvmSuccess(KVMHostInventory host, VmInstanceInventory vm) { + } + + @Override + public void stopVmOnKvmFailed(KVMHostInventory host, VmInstanceInventory vm, ErrorCode err) { + } + @Override public void releaseVmResource(VmInstanceSpec spec, Completion completion) { if (spec.getDestHost() == null) { @@ -640,6 +677,7 @@ public void releaseVmResource(VmInstanceSpec spec, Completion completion) { syncMsg.setNvRamPath(file.getPath()); } else if (file.getType() == VmHostFileType.TpmState) { syncMsg.setTpmStateFolder(file.getPath()); + syncMsg.setTpmStateFallbackFolder(buildTpmStateSnapshotBackupFilePath(vmUuid)); } else { logger.warn(String.format("unsupported vm host file type: %s, skip syncing for VM[uuid:%s] from host[uuid:%s]", file.getType(), vmUuid, hostUuid)); @@ -699,6 +737,7 @@ public void beforeHaStartVmInstance(String vmUuid, String judgerClassName, List< syncMsg.setNvRamPath(file.getPath()); } else if (file.getType() == VmHostFileType.TpmState) { syncMsg.setTpmStateFolder(file.getPath()); + syncMsg.setTpmStateFallbackFolder(buildTpmStateSnapshotBackupFilePath(vmUuid)); } else { logger.warn(String.format( "unsupported vm host file type: %s, skip syncing for VM[uuid:%s] from host[uuid:%s]", diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/efi/KvmSecureBootManager.java b/plugin/kvm/src/main/java/org/zstack/kvm/efi/KvmSecureBootManager.java index e34ec11d3b5..4a71991548a 100644 --- a/plugin/kvm/src/main/java/org/zstack/kvm/efi/KvmSecureBootManager.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/efi/KvmSecureBootManager.java @@ -279,60 +279,287 @@ static class CloneVmHostFileContext { List syncContexts = new ArrayList<>(); } - private void handle(SyncVmHostFilesFromHostMsg msg) { - KvmCommandSender sender = new KvmCommandSender(msg.getHostUuid()) - .disableHostStatusCheck(); + private static class SyncVmHostFilesContext { + KVMAgentCommands.ReadVmHostFileContentCmd cmd; + KVMAgentCommands.ReadVmHostFileContentResponse readRsp; + KVMAgentCommands.ReadVmHostFileContentResponse fallbackRsp; + long timeBeforeSync; + boolean needTpmFallback; + boolean tpmFallbackUsed; + } - KVMAgentCommands.ReadVmHostFileContentCmd cmd = new KVMAgentCommands.ReadVmHostFileContentCmd(); - cmd.setHostFiles(new ArrayList<>()); + private void handle(SyncVmHostFilesFromHostMsg msg) { + SyncVmHostFilesContext ctx = new SyncVmHostFilesContext(); + ctx.cmd = new KVMAgentCommands.ReadVmHostFileContentCmd(); + ctx.cmd.setHostFiles(new ArrayList<>()); if (msg.getTpmStateFolder() != null) { KVMAgentCommands.VmHostFileTO to = new KVMAgentCommands.VmHostFileTO(); to.setPath(msg.getTpmStateFolder()); to.setType(VmHostFileType.TpmState.toString()); - cmd.getHostFiles().add(to); + ctx.cmd.getHostFiles().add(to); } if (msg.getNvRamPath() != null) { KVMAgentCommands.VmHostFileTO to = new KVMAgentCommands.VmHostFileTO(); to.setPath(msg.getNvRamPath()); to.setType(VmHostFileType.NvRam.toString()); - cmd.getHostFiles().add(to); + ctx.cmd.getHostFiles().add(to); } - long now = timeHelper.getCurrentTimeMillis(); + ctx.timeBeforeSync = timeHelper.getCurrentTimeMillis(); SyncVmHostFilesFromHostReply reply = new SyncVmHostFilesFromHostReply(); - sender.send(cmd, READ_VM_HOST_FILE_PATH, wrapper -> { - KVMAgentCommands.ReadVmHostFileContentResponse readRsp = wrapper.getResponse(KVMAgentCommands.ReadVmHostFileContentResponse.class); - return readRsp.isSuccess() ? null : - operr("failed to read file content response").withException(readRsp.getError()); - }, new ReturnValueCompletion(msg) { - @Override - public void success(KvmResponseWrapper wrapper) { - KVMAgentCommands.ReadVmHostFileContentResponse readRsp = wrapper.getResponse(KVMAgentCommands.ReadVmHostFileContentResponse.class); - if (!readRsp.isSuccess()) { - reply.setError(operr("failed to read file content response").withException(readRsp.getError())); + + SimpleFlowChain.of("sync-vm-host-files-" + msg.getVmUuid()) + .then("read-from-primary-path", trigger -> { + KvmCommandSender sender = new KvmCommandSender(msg.getHostUuid()) + .disableHostStatusCheck(); + sender.send(ctx.cmd, READ_VM_HOST_FILE_PATH, wrapper -> { + KVMAgentCommands.ReadVmHostFileContentResponse rsp = + wrapper.getResponse(KVMAgentCommands.ReadVmHostFileContentResponse.class); + return rsp.isSuccess() ? null : + operr("failed to read file content response").withException(rsp.getError()); + }, new ReturnValueCompletion(trigger) { + @Override + public void success(KvmResponseWrapper wrapper) { + ctx.readRsp = wrapper.getResponse( + KVMAgentCommands.ReadVmHostFileContentResponse.class); + trigger.next(); + } + + @Override + public void fail(ErrorCode errorCode) { + boolean hasTpmFallback = msg.getTpmStateFallbackFolder() != null + && msg.getTpmStateFolder() != null; + if (hasTpmFallback) { + logger.info(String.format( + "read from host failed entirely for VM[uuid=%s], " + + "will try TPM fallback path [%s]", + msg.getVmUuid(), msg.getTpmStateFallbackFolder())); + ctx.readRsp = null; + trigger.next(); + } else { + trigger.fail(errorCode); + } + } + }); + }) + .then("fallback-read-tpm-from-backup", trigger -> { + if (msg.getTpmStateFallbackFolder() == null || msg.getTpmStateFolder() == null) { + trigger.next(); + return; + } + + // check whether TPM part specifically failed + boolean tpmReadFailed; + if (ctx.readRsp == null || !ctx.readRsp.isSuccess()) { + tpmReadFailed = true; + } else { + KVMAgentCommands.VmHostFileTO tpmResult = findOneOrNull( + ctx.readRsp.getHostFiles(), + item -> msg.getTpmStateFolder().equals(item.getPath())); + tpmReadFailed = tpmResult == null + || tpmResult.getError() != null + || tpmResult.getContentBase64() == null; + } + + if (!tpmReadFailed) { + trigger.next(); + return; + } + + ctx.needTpmFallback = true; + logger.info(String.format( + "TPM state read failed from primary path [%s] for VM[uuid=%s], " + + "trying fallback path [%s]", + msg.getTpmStateFolder(), msg.getVmUuid(), + msg.getTpmStateFallbackFolder())); + + KvmCommandSender sender = new KvmCommandSender(msg.getHostUuid()) + .disableHostStatusCheck(); + KVMAgentCommands.ReadVmHostFileContentCmd fallbackCmd = + new KVMAgentCommands.ReadVmHostFileContentCmd(); + KVMAgentCommands.VmHostFileTO fallbackTo = new KVMAgentCommands.VmHostFileTO(); + fallbackTo.setPath(msg.getTpmStateFallbackFolder()); + fallbackTo.setType(VmHostFileType.TpmState.toString()); + fallbackCmd.setHostFiles(Collections.singletonList(fallbackTo)); + + sender.send(fallbackCmd, READ_VM_HOST_FILE_PATH, wrapper -> { + KVMAgentCommands.ReadVmHostFileContentResponse rsp = + wrapper.getResponse(KVMAgentCommands.ReadVmHostFileContentResponse.class); + return rsp.isSuccess() ? null : + operr("failed to read TPM from fallback path") + .withException(rsp.getError()); + }, new ReturnValueCompletion(trigger) { + @Override + public void success(KvmResponseWrapper wrapper) { + ctx.fallbackRsp = wrapper.getResponse( + KVMAgentCommands.ReadVmHostFileContentResponse.class); + trigger.next(); + } + + @Override + public void fail(ErrorCode errorCode) { + logger.warn(String.format( + "TPM fallback read also failed for VM[uuid=%s] " + + "from [%s]: %s", + msg.getVmUuid(), msg.getTpmStateFallbackFolder(), + errorCode.getReadableDetails())); + trigger.fail(errorCode); + } + }); + }) + .then("merge-tpm-fallback-result", trigger -> { + if (!ctx.needTpmFallback) { + trigger.next(); + return; + } + + if (ctx.fallbackRsp == null || !ctx.fallbackRsp.isSuccess()) { + trigger.fail(operr("failed to read TPM from fallback path [%s]", + msg.getTpmStateFallbackFolder()) + .withException(ctx.fallbackRsp == null ? null + : ctx.fallbackRsp.getError())); + return; + } + + // validate fallback result + KVMAgentCommands.VmHostFileTO fallbackResult = findOneOrNull( + ctx.fallbackRsp.getHostFiles(), + item -> VmHostFileType.TpmState.toString().equals(item.getType())); + if (fallbackResult == null + || fallbackResult.getError() != null + || fallbackResult.getContentBase64() == null) { + String detail = fallbackResult == null ? "no result" : + (fallbackResult.getError() != null + ? fallbackResult.getError() : "empty content"); + trigger.fail(operr( + "TPM fallback read from [%s] returned no valid content: %s", + msg.getTpmStateFallbackFolder(), detail)); + return; + } + + logger.info(String.format( + "successfully read TPM state from fallback [%s] for VM[uuid=%s]", + msg.getTpmStateFallbackFolder(), msg.getVmUuid())); + + // rewrite path to canonical so DB stores the correct path + fallbackResult.setPath(msg.getTpmStateFolder()); + + // merge: NvRam from original response + TPM from fallback + KVMAgentCommands.ReadVmHostFileContentResponse mergedRsp = + new KVMAgentCommands.ReadVmHostFileContentResponse(); + mergedRsp.setSuccess(true); + List mergedFiles = new ArrayList<>(); + if (ctx.readRsp != null && ctx.readRsp.getHostFiles() != null) { + for (KVMAgentCommands.VmHostFileTO f : ctx.readRsp.getHostFiles()) { + if (!VmHostFileType.TpmState.toString().equals(f.getType())) { + mergedFiles.add(f); + } + } + } + mergedFiles.add(fallbackResult); + mergedRsp.setHostFiles(mergedFiles); + ctx.readRsp = mergedRsp; + + // rebuild cmd to match merged response + KVMAgentCommands.ReadVmHostFileContentCmd mergedCmd = + new KVMAgentCommands.ReadVmHostFileContentCmd(); + List cmdFiles = new ArrayList<>(); + if (msg.getNvRamPath() != null) { + KVMAgentCommands.VmHostFileTO nvTo = new KVMAgentCommands.VmHostFileTO(); + nvTo.setPath(msg.getNvRamPath()); + nvTo.setType(VmHostFileType.NvRam.toString()); + cmdFiles.add(nvTo); + } + KVMAgentCommands.VmHostFileTO tpmTo = new KVMAgentCommands.VmHostFileTO(); + tpmTo.setPath(msg.getTpmStateFolder()); + tpmTo.setType(VmHostFileType.TpmState.toString()); + cmdFiles.add(tpmTo); + mergedCmd.setHostFiles(cmdFiles); + ctx.cmd = mergedCmd; + + ctx.tpmFallbackUsed = true; + trigger.next(); + }) + .then("persist-to-db", trigger -> { + if (ctx.readRsp == null || !ctx.readRsp.isSuccess()) { + reply.setError(operr( + "no valid read response to persist for VM[uuid=%s]", + msg.getVmUuid())); + trigger.next(); + return; + } + + ErrorCode error; + if (msg.isSyncToBackup()) { + error = syncToBackupFiles(msg, ctx.readRsp); + } else { + error = syncToHostFiles(msg, ctx.cmd, ctx.readRsp, + ctx.timeBeforeSync); + } + + if (error != null) { + reply.setError(error); + } + trigger.next(); + }) + .then("cleanup-tpm-backup", trigger -> { + if (!ctx.tpmFallbackUsed) { + trigger.next(); + return; + } + + cleanupTpmBackupOnHost(msg.getHostUuid(), msg.getVmUuid(), + msg.getTpmStateFallbackFolder()); + trigger.next(); + }) + .propagateExceptionTo(msg) + .done(() -> bus.reply(msg, reply)) + .error(errorCode -> { + reply.setError(errorCode); bus.reply(msg, reply); - return; - } + }) + .start(); + } - ErrorCode error; - if (msg.isSyncToBackup()) { - error = syncToBackupFiles(msg, readRsp); - } else { - error = syncToHostFiles(msg, cmd, readRsp, now); + private void cleanupTpmBackupOnHost(String hostUuid, String vmUuid, String backupPath) { + try { + KvmCommandSender sender = new KvmCommandSender(hostUuid) + .disableHostStatusCheck(); + + KVMAgentCommands.VmHostFileTO deleteTo = new KVMAgentCommands.VmHostFileTO(); + deleteTo.setPath(backupPath); + deleteTo.setType(VmHostFileType.TpmState.toString()); + deleteTo.setOperation(VmHostFileOperation.Delete.toString()); + + KVMAgentCommands.WriteVmHostFileContentCmd deleteCmd = + new KVMAgentCommands.WriteVmHostFileContentCmd(); + deleteCmd.setHostFiles(Collections.singletonList(deleteTo)); + + sender.send(deleteCmd, WRITE_VM_HOST_FILE_PATH, wrapper -> { + KVMAgentCommands.WriteVmHostFileContentResponse rsp = + wrapper.getResponse(KVMAgentCommands.WriteVmHostFileContentResponse.class); + return rsp.isSuccess() ? null : + operr("failed to delete TPM backup").withException(rsp.getError()); + }, new ReturnValueCompletion(null) { + @Override + public void success(KvmResponseWrapper wrapper) { + logger.debug(String.format( + "cleaned up TPM backup at [%s] on host[uuid=%s] for VM[uuid=%s]", + backupPath, hostUuid, vmUuid)); } - if (error != null) { - reply.setError(error); + @Override + public void fail(ErrorCode errorCode) { + logger.warn(String.format( + "failed to clean up TPM backup at [%s] on host[uuid=%s] " + + "for VM[uuid=%s]: %s. Will be overwritten on next stop.", + backupPath, hostUuid, vmUuid, errorCode.getReadableDetails())); } - bus.reply(msg, reply); - } - - @Override - public void fail(ErrorCode errorCode) { - reply.setError(errorCode); - bus.reply(msg, reply); - } - }); + }); + } catch (Exception e) { + logger.warn(String.format("unexpected error cleaning up TPM backup for VM[uuid=%s]: %s", + vmUuid, e.getMessage()), e); + } } private ErrorCode syncToHostFiles(SyncVmHostFilesFromHostMsg msg, diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/SyncVmHostFilesFromHostMsg.java b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/SyncVmHostFilesFromHostMsg.java index 93b21797f14..424fe90c075 100644 --- a/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/SyncVmHostFilesFromHostMsg.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/vmfiles/message/SyncVmHostFilesFromHostMsg.java @@ -7,6 +7,7 @@ public class SyncVmHostFilesFromHostMsg extends NeedReplyMessage { private String vmUuid; private String nvRamPath; private String tpmStateFolder; + private String tpmStateFallbackFolder; private String syncReason; private boolean syncToBackup; private String backupResourceUuid; @@ -43,6 +44,14 @@ public void setTpmStateFolder(String tpmStateFolder) { this.tpmStateFolder = tpmStateFolder; } + public String getTpmStateFallbackFolder() { + return tpmStateFallbackFolder; + } + + public void setTpmStateFallbackFolder(String tpmStateFallbackFolder) { + this.tpmStateFallbackFolder = tpmStateFallbackFolder; + } + public String getSyncReason() { return syncReason; } diff --git a/test/src/test/resources/springConfigXml/Kvm.xml b/test/src/test/resources/springConfigXml/Kvm.xml index bd33bd2d81e..e5fbcbc1223 100755 --- a/test/src/test/resources/springConfigXml/Kvm.xml +++ b/test/src/test/resources/springConfigXml/Kvm.xml @@ -287,6 +287,7 @@ +