Skip to content

Commit eb97acf

Browse files
Pearl1594Pearl1594
committed
deliver multi-VLAN trunk nics via libvirt vlan XML and bridge fallback
1 parent 219f9cb commit eb97acf

9 files changed

Lines changed: 468 additions & 2 deletions

File tree

‎api/src/main/java/com/cloud/agent/api/to/NicTO.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@ public class NicTO extends NetworkTO {
3737

3838
String networkSegmentName;
3939

40+
boolean trunkVlan;
41+
List<NetworkTO> associatedNetworks;
42+
4043
public NicTO() {
4144
super();
4245
}
@@ -163,4 +166,21 @@ public boolean isEnabled() {
163166
public void setEnabled(boolean enabled) {
164167
this.enabled = enabled;
165168
}
169+
170+
// trunkVlan/associatedNetworks carry the additional networks for a multi-VLAN trunk nic; unset for ordinary nics
171+
public boolean isTrunkVlan() {
172+
return trunkVlan;
173+
}
174+
175+
public void setTrunkVlan(boolean trunkVlan) {
176+
this.trunkVlan = trunkVlan;
177+
}
178+
179+
public List<NetworkTO> getAssociatedNetworks() {
180+
return associatedNetworks;
181+
}
182+
183+
public void setAssociatedNetworks(List<NetworkTO> associatedNetworks) {
184+
this.associatedNetworks = associatedNetworks;
185+
}
166186
}

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java‎

Lines changed: 107 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,13 @@
2020
package com.cloud.hypervisor.kvm.resource;
2121

2222
import java.io.File;
23+
import java.net.URI;
2324
import java.util.ArrayList;
25+
import java.util.LinkedHashSet;
2426
import java.util.List;
2527
import java.util.Map;
28+
import java.util.Set;
29+
import java.util.concurrent.ConcurrentHashMap;
2630
import java.util.regex.Matcher;
2731
import java.util.regex.Pattern;
2832

@@ -33,6 +37,7 @@
3337
import org.apache.commons.lang3.StringUtils;
3438
import org.libvirt.LibvirtException;
3539

40+
import com.cloud.agent.api.to.NetworkTO;
3641
import com.cloud.agent.api.to.NicTO;
3742
import com.cloud.agent.properties.AgentProperties;
3843
import com.cloud.agent.properties.AgentPropertiesFileHandler;
@@ -43,6 +48,8 @@
4348

4449
public class BridgeVifDriver extends VifDriverBase {
4550

51+
private static final String GUEST_UPLINK_TRUNK_VLAN_RANGE = "2-4094";
52+
4653
private int _timeout;
4754

4855
private final Object _vnetBridgeMonitor = new Object();
@@ -51,6 +58,7 @@ public class BridgeVifDriver extends VifDriverBase {
5158
private String _macIpScriptPath;
5259
private String _controlCidr = NetUtils.getLinkLocalCIDR();
5360
private Long libvirtVersion;
61+
private final Set<String> uplinkVlanTrunkEnsuredBridges = ConcurrentHashMap.newKeySet();
5462

5563
private static boolean isVxlanOrNetris(String protocol) {
5664
return protocol.equals(Networks.BroadcastDomainType.Vxlan.scheme()) || protocol.equals(Networks.BroadcastDomainType.Netris.scheme());
@@ -199,6 +207,102 @@ protected boolean isValidProtocolAndVnetId(final String vNetId, final String pro
199207
return vNetId != null && protocol != null && !vNetId.equalsIgnoreCase("untagged");
200208
}
201209

210+
protected void plugTrunkVlanNic(LibvirtVMDef.InterfaceDef intf, NicTO nic, String trafficLabel, String guestOsType, String nicAdapter,
211+
Integer networkRateKBps) throws InternalErrorException {
212+
if (nic.getBroadcastType() != Networks.BroadcastDomainType.Vlan) {
213+
throw new InternalErrorException("Multi-VLAN trunk nics are only supported on VLAN-isolated guest networks");
214+
}
215+
if (!_libvirtComputingResource.hostSupportsVlanFiltering()) {
216+
throw new InternalErrorException("vlan_filtering is not enabled on this host's guest bridge; "
217+
+ "this host cannot accept a multi-VLAN trunk nic");
218+
}
219+
220+
String brName = trafficLabel != null && !trafficLabel.isEmpty() ? trafficLabel : _bridges.get("guest");
221+
222+
ensureUplinkAllowsAllVlans(brName);
223+
224+
List<Integer> vlanTags = collectTrunkVlanTags(nic);
225+
226+
logger.debug("plugging trunk nic " + nic.getMac() + " onto guest bridge " + brName + " with vlan tags " + vlanTags);
227+
intf.defBridgeNet(brName, null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), networkRateKBps);
228+
229+
if (_libvirtComputingResource.hostSupportsVlanTrunkXml()) {
230+
intf.setTrunkVlanTags(vlanTags);
231+
}
232+
// else: older libvirt can't express trunk membership; ensureVlanTrunkMembership() applies it manually once the tap exists
233+
}
234+
235+
private List<Integer> collectTrunkVlanTags(NicTO nic) throws InternalErrorException {
236+
Set<Integer> vlanTags = new LinkedHashSet<>();
237+
vlanTags.add(parseVlanTag(nic.getBroadcastUri(), "primary network of nic " + nic.getMac()));
238+
if (nic.getAssociatedNetworks() != null) {
239+
for (NetworkTO associatedNetwork : nic.getAssociatedNetworks()) {
240+
if (associatedNetwork.getBroadcastType() != Networks.BroadcastDomainType.Vlan) {
241+
throw new InternalErrorException("Multi-VLAN trunk nics only support VLAN-isolated associated networks");
242+
}
243+
vlanTags.add(parseVlanTag(associatedNetwork.getBroadcastUri(), "associated network " + associatedNetwork.getUuid()));
244+
}
245+
}
246+
return new ArrayList<>(vlanTags);
247+
}
248+
249+
private Integer parseVlanTag(URI broadcastUri, String description) throws InternalErrorException {
250+
String vlanValue = broadcastUri == null ? null : Networks.BroadcastDomainType.getValue(broadcastUri);
251+
if (StringUtils.isBlank(vlanValue)) {
252+
throw new InternalErrorException("Cannot determine VLAN for " + description
253+
+ ": no VLAN has been assigned yet (is the network implemented?). Refusing to plug this multi-VLAN trunk nic.");
254+
}
255+
try {
256+
return Integer.valueOf(vlanValue);
257+
} catch (NumberFormatException e) {
258+
throw new InternalErrorException("Invalid VLAN value '" + vlanValue + "' for " + description);
259+
}
260+
}
261+
262+
private void ensureUplinkAllowsAllVlans(String brName) throws InternalErrorException {
263+
if (uplinkVlanTrunkEnsuredBridges.contains(brName)) {
264+
return;
265+
}
266+
synchronized (_vnetBridgeMonitor) {
267+
if (uplinkVlanTrunkEnsuredBridges.contains(brName)) {
268+
return;
269+
}
270+
String uplinkPif = _pifs.get(brName);
271+
if (StringUtils.isBlank(uplinkPif)) {
272+
throw new InternalErrorException("Cannot determine the uplink interface for guest bridge " + brName
273+
+ "; refusing to plug a multi-VLAN trunk nic");
274+
}
275+
runBridgeVlanCommand("add", uplinkPif, GUEST_UPLINK_TRUNK_VLAN_RANGE);
276+
uplinkVlanTrunkEnsuredBridges.add(brName);
277+
}
278+
}
279+
280+
@Override
281+
public void ensureVlanTrunkMembership(LibvirtVMDef.InterfaceDef iface, NicTO nic) throws InternalErrorException {
282+
if (!nic.isTrunkVlan() || _libvirtComputingResource.hostSupportsVlanTrunkXml()) {
283+
return;
284+
}
285+
String tapName = iface.getDevName();
286+
if (StringUtils.isBlank(tapName)) {
287+
throw new InternalErrorException("Cannot apply manual VLAN trunk membership: tap device name unknown for nic " + nic.getMac());
288+
}
289+
for (Integer vlanTag : collectTrunkVlanTags(nic)) {
290+
runBridgeVlanCommand("add", tapName, String.valueOf(vlanTag));
291+
}
292+
}
293+
294+
protected void runBridgeVlanCommand(String operation, String dev, String vid) throws InternalErrorException {
295+
final Script command = new Script("bridge", _timeout, logger);
296+
command.add("vlan");
297+
command.add(operation);
298+
command.add("dev", dev);
299+
command.add("vid", vid);
300+
final String result = command.execute();
301+
if (result != null) {
302+
throw new InternalErrorException("Failed to " + operation + " VLAN " + vid + " membership on " + dev + ": " + result);
303+
}
304+
}
305+
202306
protected String createStorageVnetBridgeIfNeeded(NicTO nic, String trafficLabel,
203307
String storageBrName) throws InternalErrorException {
204308
if (nic.getBroadcastUri() == null) {
@@ -248,7 +352,9 @@ public LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicA
248352
}
249353

250354
if (nic.getType() == Networks.TrafficType.Guest) {
251-
if (isBroadcastTypeVlanOrVxlan(nic) && isValidProtocolAndVnetId(vNetId, protocol)) {
355+
if (nic.isTrunkVlan()) {
356+
plugTrunkVlanNic(intf, nic, trafficLabel, guestOsType, nicAdapter, networkRateKBps);
357+
} else if (isBroadcastTypeVlanOrVxlan(nic) && isValidProtocolAndVnetId(vNetId, protocol)) {
252358
if (trafficLabel != null && !trafficLabel.isEmpty()) {
253359
logger.debug("creating a vNet dev and bridge for guest traffic per traffic label " + trafficLabel);
254360
String brName = createVnetBr(vNetId, trafficLabel, protocol);

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import com.cloud.cpu.CPU;
3131
import org.apache.cloudstack.api.ApiConstants.IoDriverPolicy;
3232
import org.apache.cloudstack.utils.qemu.QemuObject;
33+
import org.apache.commons.collections.CollectionUtils;
3334
import org.apache.commons.lang.StringEscapeUtils;
3435
import org.apache.commons.lang3.StringUtils;
3536
import org.apache.logging.log4j.Logger;
@@ -1602,6 +1603,8 @@ enum HostNicType {
16021603
private String _virtualPortType;
16031604
private String _virtualPortInterfaceId;
16041605
private int _vlanTag = -1;
1606+
private boolean _vlanTrunk = false;
1607+
private List<Integer> _vlanTrunkTags;
16051608
private boolean _pxeDisable = false;
16061609
private boolean _linkStateUp = true;
16071610
private Integer _slot;
@@ -1762,6 +1765,19 @@ public int getVlanTag() {
17621765
return _vlanTag;
17631766
}
17641767

1768+
public void setTrunkVlanTags(List<Integer> vlanTags) {
1769+
_vlanTrunk = true;
1770+
_vlanTrunkTags = vlanTags;
1771+
}
1772+
1773+
public List<Integer> getTrunkVlanTags() {
1774+
return _vlanTrunkTags;
1775+
}
1776+
1777+
public boolean isVlanTrunk() {
1778+
return _vlanTrunk;
1779+
}
1780+
17651781
public void setSlot(Integer slot) {
17661782
_slot = slot;
17671783
}
@@ -1865,7 +1881,13 @@ public String getContent() {
18651881
}
18661882
netBuilder.append("</virtualport>\n");
18671883
}
1868-
if (_vlanTag > 0 && _vlanTag < 4095) {
1884+
if (_vlanTrunk && CollectionUtils.isNotEmpty(_vlanTrunkTags)) {
1885+
netBuilder.append("<vlan trunk='yes'>\n");
1886+
for (Integer tag : _vlanTrunkTags) {
1887+
netBuilder.append("<tag id='" + tag + "'/>\n");
1888+
}
1889+
netBuilder.append("</vlan>");
1890+
} else if (_vlanTag > 0 && _vlanTag < 4095) {
18691891
netBuilder.append("<vlan trunk='no'>\n<tag id='" + _vlanTag + "'/>\n</vlan>");
18701892
}
18711893

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/VifDriver.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,4 +46,8 @@ public interface VifDriver {
4646

4747
void deleteBr(NicTO nic);
4848

49+
// applies manual VLAN trunk membership to a trunk nic's live tap on hosts whose libvirt can't do it via <vlan> XML; no-op otherwise
50+
default void ensureVlanTrunkMembership(LibvirtVMDef.InterfaceDef iface, NicTO nic) throws InternalErrorException {
51+
}
52+
4953
}

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapper.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import com.cloud.hypervisor.kvm.resource.VifDriver;
3030
import com.cloud.resource.CommandWrapper;
3131
import com.cloud.resource.ResourceWrapper;
32+
import com.cloud.utils.exception.CloudRuntimeException;
3233
import com.cloud.vm.VirtualMachine;
3334
import org.libvirt.Connect;
3435
import org.libvirt.Domain;
@@ -67,6 +68,15 @@ public Answer execute(final PlugNicCommand command, final LibvirtComputingResour
6768
}
6869
vm.attachDevice(interfaceDef.toString());
6970

71+
if (nic.isTrunkVlan()) {
72+
try {
73+
final InterfaceDef liveInterfaceDef = libvirtComputingResource.getInterface(conn, vmName, nic.getMac());
74+
vifDriver.ensureVlanTrunkMembership(liveInterfaceDef, nic);
75+
} catch (CloudRuntimeException e) {
76+
throw new InternalErrorException("Failed to locate live tap for trunk nic " + nic.getMac() + ": " + e.getMessage());
77+
}
78+
}
79+
7080
// apply default network rules on new nic
7181
if (vmType == VirtualMachine.Type.User && nic.isSecurityGroupEnabled()) {
7282
final Long vmId = Long.valueOf(vmName.split("-")[2]);

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import com.cloud.network.Networks.TrafficType;
5050
import com.cloud.resource.CommandWrapper;
5151
import com.cloud.resource.ResourceWrapper;
52+
import com.cloud.utils.exception.CloudRuntimeException;
5253
import com.cloud.vm.UserVmManager;
5354
import com.cloud.vm.VirtualMachine;
5455

@@ -96,6 +97,7 @@ public Answer execute(final StartCommand command, final LibvirtComputingResource
9697
String vmFinalSpecification = performXmlTransformHook(vmInitialSpecification, libvirtComputingResource);
9798
libvirtComputingResource.startVM(conn, vmName, vmFinalSpecification);
9899
performAgentStartHook(vmName, libvirtComputingResource);
100+
applyManualVlanTrunkMembership(conn, vmName, nics, libvirtComputingResource);
99101

100102
libvirtComputingResource.applyDefaultNetworkRules(conn, vmSpec, false);
101103

@@ -179,6 +181,21 @@ public Answer execute(final StartCommand command, final LibvirtComputingResource
179181
}
180182
}
181183

184+
private void applyManualVlanTrunkMembership(Connect conn, String vmName, NicTO[] nics, LibvirtComputingResource libvirtComputingResource)
185+
throws InternalErrorException {
186+
for (NicTO nic : nics) {
187+
if (!nic.isTrunkVlan()) {
188+
continue;
189+
}
190+
try {
191+
LibvirtVMDef.InterfaceDef liveInterface = libvirtComputingResource.getInterface(conn, vmName, nic.getMac());
192+
libvirtComputingResource.getVifDriver(nic.getType(), nic.getName()).ensureVlanTrunkMembership(liveInterface, nic);
193+
} catch (CloudRuntimeException e) {
194+
throw new InternalErrorException("Failed to locate live tap for trunk nic " + nic.getMac() + ": " + e.getMessage());
195+
}
196+
}
197+
}
198+
182199
private void mountSecondaryStoragesIfNeeded(StartCommand command, LibvirtComputingResource libvirtComputingResource, List<KVMStoragePool> secondaryStorages) {
183200
if (CollectionUtils.isNotEmpty(command.getSecondaryStorages())) {
184201
for (String secondaryStorageUrl : command.getSecondaryStorages()) {

0 commit comments

Comments
 (0)