Skip to content

[KVM] Skip volumes on shared storage when live migrating VM with local volumes (root / datadisk) #6509

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
GabrielBrascher wants to merge 2 commits into from
Closed

[KVM] Skip volumes on shared storage when live migrating VM with local volumes (root / datadisk) #6509

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
80c3007
Add virDomainMigrate3 into Libvirt migration.
GabrielBrascher Jun 26, 2022
55fd42d
Manage multiple local disk devices labels for VM live-migration.
GabrielBrascher Jul 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -573,7 +573,7 @@ public String toString() {
}
}

enum DiskType {
public enum DiskType {
FILE("file"), BLOCK("block"), DIRECTROY("dir"), NETWORK("network");
String _diskType;

Expand Down
32 changes: 30 additions & 2 deletions plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/MigrateKVMAsync.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,13 @@

import java.util.concurrent.Callable;

import org.apache.commons.lang3.StringUtils;
import org.libvirt.Connect;
import org.libvirt.Domain;
import org.libvirt.LibvirtException;
import org.libvirt.TypedParameter;
import org.libvirt.TypedStringParameter;
import org.libvirt.TypedUlongParameter;

public class MigrateKVMAsync implements Callable<Domain> {

Expand All @@ -33,6 +37,7 @@ public class MigrateKVMAsync implements Callable<Domain> {
private String dxml = "";
private String vmName = "";
private String destIp = "";
private String rootDiskDiskDevicesLabels = null;
private boolean migrateStorage;
private boolean migrateNonSharedInc;
private boolean autoConvergence;
Expand Down Expand Up @@ -85,9 +90,11 @@ public class MigrateKVMAsync implements Callable<Domain> {

// Libvirt 1.2.3 supports auto converge.
private static final int LIBVIRT_VERSION_SUPPORTS_AUTO_CONVERGE = 1002003;
private static final String TCP_URI = "tcp:";

public MigrateKVMAsync(final LibvirtComputingResource libvirtComputingResource, final Domain dm, final Connect dconn, final String dxml,
final boolean migrateStorage, final boolean migrateNonSharedInc, final boolean autoConvergence, final String vmName, final String destIp) {
final boolean migrateStorage, final boolean migrateNonSharedInc, final boolean autoConvergence, final String vmName, final String destIp,
final String rootDiskDiskDevicesLabels) {
this.libvirtComputingResource = libvirtComputingResource;

this.dm = dm;
Expand All @@ -98,11 +105,13 @@ public MigrateKVMAsync(final LibvirtComputingResource libvirtComputingResource,
this.autoConvergence = autoConvergence;
this.vmName = vmName;
this.destIp = destIp;
this.rootDiskDiskDevicesLabels = rootDiskDiskDevicesLabels;
}

@Override
public Domain call() throws LibvirtException {
long flags = VIR_MIGRATE_LIVE;
String destUri = TCP_URI + destIp;

if (dconn.getLibVirVersion() >= LIBVIRT_VERSION_SUPPORTS_MIGRATE_COMPRESSED) {
flags |= VIR_MIGRATE_COMPRESSED;
Expand All @@ -121,6 +130,25 @@ public Domain call() throws LibvirtException {
flags |= VIR_MIGRATE_AUTO_CONVERGE;
}

return dm.migrate(dconn, flags, dxml, vmName, "tcp:" + destIp, libvirtComputingResource.getMigrateSpeed());
if (StringUtils.isNotBlank(rootDiskDiskDevicesLabels) && migrateStorage && !migrateNonSharedInc) {
return migrateRootDisk(flags, rootDiskDiskDevicesLabels, destUri);
}
return dm.migrate(dconn, flags, dxml, vmName, destUri, libvirtComputingResource.getMigrateSpeed());
}

/**
* Uses libvirt.virDomainMigrate3 in order to map the root volume to be copied when using the flag VIR_MIGRATE_NON_SHARED_DISK.
* VIR_MIGRATE_NON_SHARED_DISK is analagous to "--copy-all" in "virsh".
* Binding into: "virsh migrate i-2-13456-VM qemu://<DestHostAddr>/system --live --compressed --copy-storage-all --migrate-disks vda --xml i-2-13456-VM.xml"
*/
private Domain migrateRootDisk(long flags, String diskDevicesName, String uri) throws LibvirtException {
int nParams = 5;
TypedParameter[] params = new TypedParameter[nParams];
params[0] = new TypedStringParameter(Domain.DomainMigrateParameters.VIR_MIGRATE_PARAM_DEST_XML, dxml);
params[1] = new TypedStringParameter(Domain.DomainMigrateParameters.VIR_MIGRATE_PARAM_DEST_NAME, vmName);
params[2] = new TypedStringParameter(Domain.DomainMigrateParameters.VIR_MIGRATE_PARAM_URI, uri);
params[3] = new TypedUlongParameter(Domain.DomainMigrateParameters.VIR_MIGRATE_PARAM_BANDWIDTH, (long)libvirtComputingResource.getMigrateSpeed());
params[4] = new TypedStringParameter(Domain.DomainMigrateParameters.VIR_MIGRATE_PARAM_MIGRATE_DISKS, diskDevicesName);
return dm.migrate(dconn, params, flags);
}
}
28 changes: 27 additions & 1 deletion ...src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
Expand All @@ -45,6 +46,7 @@

import com.cloud.agent.api.to.DiskTO;
import com.cloud.agent.api.to.DpdkTO;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
Expand Down Expand Up @@ -188,6 +190,7 @@ Use VIR_DOMAIN_XML_SECURE (value = 1) prior to v1.0.0.
final boolean migrateStorage = MapUtils.isNotEmpty(mapMigrateStorage);
final boolean migrateStorageManaged = command.isMigrateStorageManaged();

String rootDiskDiskDevicesLabels = null;
if (migrateStorage) {
if (s_logger.isDebugEnabled()) {
s_logger.debug(String.format("Changing VM [%s] volumes during migration to host: [%s].", vmName, target));
Expand All @@ -196,6 +199,7 @@ Use VIR_DOMAIN_XML_SECURE (value = 1) prior to v1.0.0.
if (s_logger.isDebugEnabled()) {
s_logger.debug(String.format("Changed VM [%s] XML configuration of used storage. New XML configuration is [%s].", vmName, xmlDesc));
}
rootDiskDiskDevicesLabels = retrieveLocalDiskDevicesLabels(disks);
}

Map<String, DpdkTO> dpdkPortsMapping = command.getDpdkInterfaceMapping();
Expand All @@ -222,7 +226,7 @@ Use VIR_DOMAIN_XML_SECURE (value = 1) prior to v1.0.0.

final Callable<Domain> worker = new MigrateKVMAsync(libvirtComputingResource, dm, dconn, xmlDesc,
migrateStorage, migrateNonSharedInc,
command.isAutoConvergence(), vmName, command.getDestinationIp());
command.isAutoConvergence(), vmName, command.getDestinationIp(), rootDiskDiskDevicesLabels);
final Future<Domain> migrateThread = executor.submit(worker);
executor.shutdown();
long sleeptime = 0;
Expand Down Expand Up @@ -359,6 +363,28 @@ Use VIR_DOMAIN_XML_SECURE (value = 1) prior to v1.0.0.
return new MigrateAnswer(command, result == null, result, null);
}

/**
* It returns the label of the Root disk in case it is placed in a local storage.
* In such cases, the disk device is of type "File".
* If the Root volume is not in local storage then it returns null.
* Example of output: "sda" for one Root local.
* In case of multiple devices, then comma separated; example: "sda,sdb".
*/
protected String retrieveLocalDiskDevicesLabels(List<DiskDef> disks) {
List<String> labels = new ArrayList<>();
for (DiskDef diskDef : disks) {
if (DiskDef.DiskType.FILE == diskDef.getDiskType() && DiskDef.DeviceType.DISK == diskDef.getDeviceType()) {
labels.add(diskDef.getDiskLabel());
}
}

String localDiskDevicesLabels = null;
if (CollectionUtils.isNotEmpty(labels))
localDiskDevicesLabels = String.join(",", labels);

return localDiskDevicesLabels;
}

/**
* Replace DPDK source path and target before migrations
*/
Expand Down
53 changes: 53 additions & 0 deletions ...test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -791,4 +791,57 @@ public void testReplaceDPDKPorts() throws ParserConfigurationException, IOExcept
Assert.assertTrue(replaced.contains("csdpdk-7"));
Assert.assertFalse(replaced.contains("csdpdk-1"));
}

@Test
public void discoverLocalRootDiskDeviceLabelTestRootDiskFile() {
String rootLabel = "rootLabel";
DiskDef rootDisk = mockDisk(DiskDef.DiskType.FILE, DiskDef.DeviceType.DISK, rootLabel);
Mockito.when(rootDisk.getDiskLabel()).thenReturn(rootLabel);
List<DiskDef> disks = new ArrayList<>();
disks.add(rootDisk);
prepareAndTestDiscoverLocalRootDiskDeviceLabel(disks, rootLabel);
}

@Test
public void discoverLocalRootDiskDeviceLabelTestRootDiskFileNotDisk() {
String rootLabel = "rootLabel";
String expectedLabel = null;
for (DiskDef.DeviceType deviceType : DiskDef.DeviceType.values()) {
if (DiskDef.DeviceType.DISK != deviceType) {
List<DiskDef> disks = new ArrayList<>();
DiskDef rootDisk = mockDisk(DiskDef.DiskType.FILE, deviceType, rootLabel);
Mockito.when(rootDisk.getDiskLabel()).thenReturn(rootLabel);
disks.add(rootDisk);
prepareAndTestDiscoverLocalRootDiskDeviceLabel(disks, expectedLabel);
}
}
}

@Test
public void TestMultipleDeviceLabel() {
List<DiskDef> disks = new ArrayList<>();
disks.add(mockDisk(DiskDef.DiskType.FILE, DiskDef.DeviceType.DISK, "label1"));
disks.add(mockDisk(DiskDef.DiskType.FILE, DiskDef.DeviceType.DISK, "label2"));
disks.add(mockDisk(DiskDef.DiskType.NETWORK, DiskDef.DeviceType.DISK, "label3"));
disks.add(mockDisk(DiskDef.DiskType.BLOCK, DiskDef.DeviceType.DISK, "label4"));
String diskLabel = libvirtMigrateCmdWrapper.retrieveLocalDiskDevicesLabels(disks);
String expected = "label1,label2";
Assert.assertEquals(expected, diskLabel);
}

private void prepareAndTestDiscoverLocalRootDiskDeviceLabel(List<DiskDef> disks, String expected) {
disks.add(mockDisk(DiskDef.DiskType.NETWORK, DiskDef.DeviceType.DISK, "label1"));
disks.add(mockDisk(DiskDef.DiskType.NETWORK, DiskDef.DeviceType.DISK, "label2"));
disks.add(mockDisk(DiskDef.DiskType.BLOCK, DiskDef.DeviceType.DISK,"label3"));
String diskLabel = libvirtMigrateCmdWrapper.retrieveLocalDiskDevicesLabels(disks);
Assert.assertEquals(expected, diskLabel);
}

private DiskDef mockDisk(DiskDef.DiskType file, DiskDef.DeviceType diskType, String label) {
DiskDef mockedDisk = Mockito.mock(DiskDef.class);
Mockito.when(mockedDisk.getDiskType()).thenReturn(file);
Mockito.when(mockedDisk.getDeviceType()).thenReturn(diskType);
Mockito.when(mockedDisk.getDiskLabel()).thenReturn(label);
return mockedDisk;
}
}