Skip to content

Commit dd4d148

Browse files
weizhouapacheweizhouapache
committed
kvm: reload VNC TLS certificate on running VMs after cert renewal
Fixes VM console access breaking after the VNC certificate expires and is renewed (issue #9718). On KVM, the VNC TLS certificate is the host's agent certificate, applied host-wide via libvirtd's vnc_tls_x509_cert_dir setting. Restarting libvirtd after a renewal does not affect VMs already running, since QEMU only loads that certificate once, at VM start. Extend the existing PostCertificateRenewalCommand handling to, after restarting libvirtd, reload the VNC TLS certificate live on every currently running VM via the QMP display-reload command (added in QEMU 6.0, commit 9cc07651655ee86eca41059f5ead8c4e5607c734), sent through libvirt's qemu-monitor-command passthrough for compatibility with older libvirt. Hosts running QEMU < 6.0 are detected and skipped with a warning, keeping the previous stop/start-or-migrate behavior.
1 parent a4d3c66 commit dd4d148

2 files changed

Lines changed: 219 additions & 5 deletions

File tree

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapper.java‎

Lines changed: 72 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@
1919

2020
import org.apache.cloudstack.ca.PostCertificateRenewalCommand;
2121
import org.apache.cloudstack.ca.SetupCertificateAnswer;
22+
import org.libvirt.Connect;
23+
import org.libvirt.Domain;
24+
import org.libvirt.LibvirtException;
2225

2326
import com.cloud.agent.api.Answer;
2427
import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
@@ -29,18 +32,82 @@
2932
@ResourceWrapper(handles = PostCertificateRenewalCommand.class)
3033
public final class LibvirtPostCertificateRenewalCommandWrapper extends CommandWrapper<PostCertificateRenewalCommand, Answer, LibvirtComputingResource> {
3134

35+
/**
36+
* QMP {@code display-reload} command asking QEMU to reload the VNC display's TLS credentials off disk.
37+
* {@code tls-certs: true} is required, otherwise QEMU reloads the display without touching the certificates.
38+
* Added in QEMU 6.0 by commit 9cc07651655ee86eca41059f5ead8c4e5607c734 ("qmp: add new qmp display-reload",
39+
* merged 2021-03-23) - https://github.com/qemu/qemu/commit/9cc07651655ee86eca41059f5ead8c4e5607c734
40+
*/
41+
private static final String QEMU_MONITOR_DISPLAY_RELOAD_VNC_TLS_CERTS_COMMAND =
42+
"{\"execute\":\"display-reload\",\"arguments\":{\"type\":\"vnc\",\"tls-certs\":true}}";
43+
44+
/** Minimum QEMU version supporting {@link #QEMU_MONITOR_DISPLAY_RELOAD_VNC_TLS_CERTS_COMMAND}. */
45+
private static final long MIN_QEMU_VERSION_FOR_VNC_TLS_CERT_RELOAD = 6000000L;
3246

3347
@Override
3448
public Answer execute(final PostCertificateRenewalCommand command, final LibvirtComputingResource serverResource) {
3549
logger.info("Restarting libvirt after certificate provisioning/renewal");
3650
if (command != null) {
37-
final int timeout = 30000;
38-
Script script = new Script(true, "service", timeout, logger);
39-
script.add("libvirtd");
40-
script.add("restart");
41-
script.execute();
51+
restartLibvirtd();
52+
pushRenewedVncCertificateToRunningVms(serverResource);
4253
return new SetupCertificateAnswer(true);
4354
}
4455
return new SetupCertificateAnswer(false);
4556
}
57+
58+
private void restartLibvirtd() {
59+
final int timeout = 30000;
60+
Script script = new Script(true, "service", timeout, logger);
61+
script.add("libvirtd");
62+
script.add("restart");
63+
script.execute();
64+
}
65+
66+
/**
67+
* The VNC TLS certificate on KVM is the host's agent certificate, applied host-wide via libvirtd's
68+
* {@code vnc_tls_x509_cert_dir} setting. Restarting libvirtd does not affect VMs already running, since QEMU
69+
* only loads that certificate once, at VM start - so reload it live on every running VM here, instead of
70+
* leaving them on the previous (possibly expired) certificate until stopped/started or migrated.
71+
*/
72+
private void pushRenewedVncCertificateToRunningVms(final LibvirtComputingResource serverResource) {
73+
final long qemuVersion = serverResource.getHypervisorQemuVersion();
74+
if (qemuVersion < MIN_QEMU_VERSION_FOR_VNC_TLS_CERT_RELOAD) {
75+
logger.warn("QEMU {} on this host does not support reloading the VNC TLS certificate of a running VM (QEMU >= {} required), " +
76+
"running VMs will keep using the previous certificate until they are stopped/started or migrated",
77+
qemuVersion, MIN_QEMU_VERSION_FOR_VNC_TLS_CERT_RELOAD);
78+
return;
79+
}
80+
81+
final Connect conn;
82+
final int[] domainIds;
83+
try {
84+
conn = serverResource.getLibvirtUtilitiesHelper().getConnection();
85+
domainIds = conn.listDomains();
86+
} catch (final LibvirtException e) {
87+
logger.warn("Unable to list running VMs to reload their renewed VNC certificate", e);
88+
return;
89+
}
90+
91+
for (final int domainId : domainIds) {
92+
Domain vm = null;
93+
String vmName = null;
94+
try {
95+
vm = conn.domainLookupByID(domainId);
96+
vmName = vm.getName();
97+
vm.qemuMonitorCommand(QEMU_MONITOR_DISPLAY_RELOAD_VNC_TLS_CERTS_COMMAND, 0);
98+
logger.debug("Reloaded VNC TLS certificate for VM [{}]", vmName);
99+
} catch (final Exception e) {
100+
logger.warn("Failed to reload the renewed VNC certificate for VM [{}], it will keep using the previous " +
101+
"certificate until it is stopped/started or migrated", vmName, e);
102+
} finally {
103+
if (vm != null) {
104+
try {
105+
vm.free();
106+
} catch (final LibvirtException e) {
107+
logger.trace("Ignoring libvirt error.", e);
108+
}
109+
}
110+
}
111+
}
112+
}
46113
}

‎plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapperTest.java‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
package com.cloud.hypervisor.kvm.resource.wrapper;
18+
19+
import static org.junit.Assert.assertTrue;
20+
import static org.mockito.Mockito.mock;
21+
import static org.mockito.Mockito.times;
22+
import static org.mockito.Mockito.verify;
23+
import static org.mockito.Mockito.verifyNoInteractions;
24+
import static org.mockito.Mockito.when;
25+
26+
import org.apache.cloudstack.ca.PostCertificateRenewalCommand;
27+
import org.junit.Before;
28+
import org.junit.Test;
29+
import org.junit.runner.RunWith;
30+
import org.libvirt.Connect;
31+
import org.libvirt.Domain;
32+
import org.libvirt.LibvirtException;
33+
import org.mockito.ArgumentCaptor;
34+
import org.mockito.Mock;
35+
import org.mockito.Mockito;
36+
import org.mockito.MockedConstruction;
37+
import org.mockito.junit.MockitoJUnitRunner;
38+
39+
import com.cloud.agent.api.Answer;
40+
import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
41+
import com.cloud.utils.script.Script;
42+
43+
@RunWith(MockitoJUnitRunner.class)
44+
public class LibvirtPostCertificateRenewalCommandWrapperTest {
45+
46+
private static final long SUPPORTED_QEMU_VERSION = 6000000L;
47+
48+
@Mock
49+
private LibvirtComputingResource libvirtComputingResource;
50+
@Mock
51+
private LibvirtUtilitiesHelper libvirtUtilitiesHelper;
52+
@Mock
53+
private Connect connect;
54+
55+
private final LibvirtPostCertificateRenewalCommandWrapper wrapper = new LibvirtPostCertificateRenewalCommandWrapper();
56+
57+
@Before
58+
public void setUp() {
59+
when(libvirtComputingResource.getHypervisorQemuVersion()).thenReturn(SUPPORTED_QEMU_VERSION);
60+
}
61+
62+
private Answer executeWithScriptMocked() {
63+
try (MockedConstruction<Script> ignored = Mockito.mockConstruction(Script.class)) {
64+
return wrapper.execute(new PostCertificateRenewalCommand(), libvirtComputingResource);
65+
}
66+
}
67+
68+
@Test
69+
public void testExecuteReloadsVncTlsCertificateForEachRunningVm() throws Exception {
70+
final Domain vm1 = mock(Domain.class);
71+
when(vm1.getName()).thenReturn("i-2-3-VM");
72+
final Domain vm2 = mock(Domain.class);
73+
when(vm2.getName()).thenReturn("i-4-5-VM");
74+
75+
when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper);
76+
when(libvirtUtilitiesHelper.getConnection()).thenReturn(connect);
77+
when(connect.listDomains()).thenReturn(new int[]{1, 2});
78+
when(connect.domainLookupByID(1)).thenReturn(vm1);
79+
when(connect.domainLookupByID(2)).thenReturn(vm2);
80+
81+
final Answer answer = executeWithScriptMocked();
82+
83+
assertTrue(answer.getResult());
84+
final ArgumentCaptor<String> monitorCommandCaptor = ArgumentCaptor.forClass(String.class);
85+
verify(vm1, times(1)).qemuMonitorCommand(monitorCommandCaptor.capture(), Mockito.eq(0));
86+
verify(vm2, times(1)).qemuMonitorCommand(Mockito.anyString(), Mockito.eq(0));
87+
final String capturedCommand = monitorCommandCaptor.getValue();
88+
assertTrue(capturedCommand.contains("display-reload"));
89+
assertTrue(capturedCommand.contains("\"type\":\"vnc\""));
90+
assertTrue(capturedCommand.contains("\"tls-certs\":true"));
91+
verify(vm1, times(1)).free();
92+
verify(vm2, times(1)).free();
93+
}
94+
95+
@Test
96+
public void testExecuteContinuesWithOtherVmsWhenOneReloadFails() throws Exception {
97+
final Domain failingVm = mock(Domain.class);
98+
when(failingVm.getName()).thenReturn("i-2-3-VM");
99+
when(failingVm.qemuMonitorCommand(Mockito.anyString(), Mockito.eq(0))).thenThrow(mock(LibvirtException.class));
100+
final Domain workingVm = mock(Domain.class);
101+
when(workingVm.getName()).thenReturn("i-4-5-VM");
102+
103+
when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper);
104+
when(libvirtUtilitiesHelper.getConnection()).thenReturn(connect);
105+
when(connect.listDomains()).thenReturn(new int[]{1, 2});
106+
when(connect.domainLookupByID(1)).thenReturn(failingVm);
107+
when(connect.domainLookupByID(2)).thenReturn(workingVm);
108+
109+
final Answer answer = executeWithScriptMocked();
110+
111+
assertTrue(answer.getResult());
112+
verify(workingVm, times(1)).qemuMonitorCommand(Mockito.anyString(), Mockito.eq(0));
113+
verify(failingVm, times(1)).free();
114+
verify(workingVm, times(1)).free();
115+
}
116+
117+
@Test
118+
public void testExecuteContinuesWhenNoRunningVms() throws Exception {
119+
when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper);
120+
when(libvirtUtilitiesHelper.getConnection()).thenReturn(connect);
121+
when(connect.listDomains()).thenReturn(new int[]{});
122+
123+
final Answer answer = executeWithScriptMocked();
124+
125+
assertTrue(answer.getResult());
126+
}
127+
128+
@Test
129+
public void testExecuteHandlesUnableToListRunningVms() throws Exception {
130+
when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper);
131+
when(libvirtUtilitiesHelper.getConnection()).thenThrow(mock(LibvirtException.class));
132+
133+
final Answer answer = executeWithScriptMocked();
134+
135+
assertTrue(answer.getResult());
136+
}
137+
138+
@Test
139+
public void testExecuteSkipsVncCertReloadWhenQemuVersionTooOld() throws Exception {
140+
when(libvirtComputingResource.getHypervisorQemuVersion()).thenReturn(5002000L);
141+
142+
final Answer answer = executeWithScriptMocked();
143+
144+
assertTrue(answer.getResult());
145+
verifyNoInteractions(libvirtUtilitiesHelper);
146+
}
147+
}

0 commit comments

Comments
 (0)