From 7acfaaeb4bfb35900c78bd0f2aa32f2f5fee92d4 Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Tue, 15 Sep 2026 08:33:56 +0000 Subject: [PATCH 1/8] Direct Routed (L3) guest networks: route public IPv4/IPv6 directly to Instances Add a guest network type in which the hypervisor performs L3 routing for the Instance: no Virtual Router, no NAT and no DHCP. Each Instance receives a public IPv4 address as a /32 and/or an IPv6 address as a /128, with a shared, host-independent link-local gateway (169.254.0.1 and fe80::1) that every hypervisor carries. All addressing reaches the Instance via ConfigDrive/cloud-init; a routing daemon on the host (FRR, BIRD, ...) advertises the addresses to the fabric and is deliberately out of scope for CloudStack. The networks live on a dedicated physical network with the new ROUTED isolation method, so the feature can be added to existing zones without touching anything already running there. Each network carries a routed:// broadcast domain, the id allocated from the physical network's vnet range or chosen by the operator, which names the network's bridge on every host. Routed ids are validated and canonicalised: a positive integer of at most ten digits, given bare or as routed://. A routed public range may not take an id that a guest network holds or that lies inside a ROUTED physical network's vnet range. Both address families are optional and IPv6-only networks are supported. IPv4 is a subnet (cidr=..., or netmask plus start/end IP), IPv6 is an ip6cidr alone: addresses derive from the subnet and the NIC MAC with EUI-64, so no range exists. No gateways are declared or stored. createNetwork gains an optional cidr parameter (L3 only, additive); an explicit startip/endip must lie inside the given subnet. The zone-wide IPv4 overlap check for L3 ranges runs in the range-creation path that commitNetwork() uses, and the vlan overlap check treats a subnet overlap with an L3 network as a conflict, so both directions are covered. Secondary IPs on L3 NICs follow the Shared branch of allocateSecondaryGuestIP. Zone IPv6 DNS is not required for L3 networks. A DefaultL3NetworkOffering (UserData and DNS via ConfigDrive, security groups) is created on install and upgrade. General fixes the feature surfaced but that apply beyond it: auto-allocated ids are exempt from the dynamic-vlan-range check in createVlanAndPublicIpRange; canUseForDeploy() counts the real IPv4 pool and no longer hides IPv4-less networks from the deploy wizard; the zone-wide IPv6 overlap check keys on ip6_cidr rather than ip6_gateway. Implements #12210 --- .../main/java/com/cloud/network/Network.java | 4 +- .../main/java/com/cloud/network/Networks.java | 44 ++++ .../com/cloud/offering/NetworkOffering.java | 1 + .../user/network/CreateNetworkCmd.java | 14 +- .../user/network/ListNetworkOfferingsCmd.java | 2 +- .../command/user/vm/RemoveIpFromVmNicCmd.java | 12 +- .../java/com/cloud/network/NetworksTest.java | 15 ++ .../user/network/CreateNetworkCmdTest.java | 2 +- .../api/NetworkRulesVmSecondaryIpCommand.java | 16 ++ .../service/NetworkOrchestrationService.java | 2 + .../orchestration/NetworkOrchestrator.java | 59 ++++- .../ConfigurationManagerImpl.java | 157 +++++++++--- .../cloud/network/IpAddressManagerImpl.java | 3 +- .../cloud/network/Ipv6AddressManagerImpl.java | 2 +- .../com/cloud/network/NetworkModelImpl.java | 13 +- .../com/cloud/network/NetworkServiceImpl.java | 224 ++++++++++++++++-- .../network/guru/DirectRoutedNetworkGuru.java | 192 +++++++++++++++ .../cloud/network/guru/PublicNetworkGuru.java | 62 ++++- .../security/SecurityGroupManagerImpl.java | 23 +- .../java/com/cloud/vm/UserVmManagerImpl.java | 2 +- .../spring-server-network-context.xml | 3 + .../ConfigurationManagerImplTest.java | 122 ++++++++++ .../cloud/network/NetworkModelImplTest.java | 62 +++++ .../cloud/network/NetworkServiceImplTest.java | 129 ++++++++++ .../guru/DirectRoutedNetworkGuruTest.java | 197 +++++++++++++++ .../network/guru/PublicNetworkGuruTest.java | 135 +++++++++++ .../com/cloud/vpc/MockNetworkManagerImpl.java | 5 + .../java/com/cloud/utils/net/NetUtils.java | 22 +- 28 files changed, 1436 insertions(+), 88 deletions(-) create mode 100644 server/src/main/java/com/cloud/network/guru/DirectRoutedNetworkGuru.java create mode 100644 server/src/test/java/com/cloud/network/guru/DirectRoutedNetworkGuruTest.java create mode 100644 server/src/test/java/com/cloud/network/guru/PublicNetworkGuruTest.java diff --git a/api/src/main/java/com/cloud/network/Network.java b/api/src/main/java/com/cloud/network/Network.java index 2f0bcdd5ef9a..d7b467bab5a7 100644 --- a/api/src/main/java/com/cloud/network/Network.java +++ b/api/src/main/java/com/cloud/network/Network.java @@ -43,7 +43,7 @@ public interface Network extends ControlledEntity, StateObject, InternalIdentity, Identity, Serializable, Displayable { enum GuestType { - Shared, Isolated, L2; + Shared, Isolated, L2, L3; public static GuestType fromValue(String type) { if (StringUtils.isBlank(type)) { @@ -54,6 +54,8 @@ public static GuestType fromValue(String type) { return Isolated; } else if (type.equalsIgnoreCase("L2")) { return L2; + } else if (type.equalsIgnoreCase("L3")) { + return L3; } else { throw new InvalidParameterValueException("Unexpected Guest type : " + type); } diff --git a/api/src/main/java/com/cloud/network/Networks.java b/api/src/main/java/com/cloud/network/Networks.java index 61a1c820723f..2ede10894af5 100644 --- a/api/src/main/java/com/cloud/network/Networks.java +++ b/api/src/main/java/com/cloud/network/Networks.java @@ -18,6 +18,7 @@ import java.net.URI; import java.net.URISyntaxException; +import java.util.regex.Pattern; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.commons.lang3.StringUtils; @@ -131,6 +132,23 @@ public URI toUri(T value) { } } }, + /** + * Direct Routed (L3) networks: the id is a label naming the per-network bridge on the + * hypervisor (brdr-<id>), not an encapsulation — nothing appears on the wire. + */ + Routed("routed", Long.class) { + @Override + public URI toUri(T value) { + try { + if (value.toString().contains("://")) + return new URI(value.toString()); + else + return new URI("routed://" + value.toString()); + } catch (URISyntaxException e) { + throw new CloudRuntimeException("Unable to convert to broadcast URI: " + value); + } + } + }, UnDecided(null, null), OpenDaylight("opendaylight", String.class), TUNGSTEN("tf", String.class), @@ -161,6 +179,32 @@ public Class type() { return type; } + /** + * A routed id — the value of a routed://<id> broadcast domain — names a bridge on + * every hypervisor (brdr-<id>, at most 15 characters) and derives that bridge's MAC + * address from five bytes, so it is a positive integer of at most ten digits without + * leading zeros. + */ + public static final int ROUTED_ID_MAX_DIGITS = 10; + private static final Pattern ROUTED_ID_PATTERN = Pattern.compile("^[1-9][0-9]{0," + (ROUTED_ID_MAX_DIGITS - 1) + "}$"); + + /** + * Extracts the routed id from a bare number or from a routed://<id> URI string. + * + * @return the bare id, or null when the candidate is not a valid routed id + */ + public static String getRoutedId(String candidate) { + if (StringUtils.isBlank(candidate)) { + return null; + } + String id = candidate.trim(); + String prefix = Routed.scheme() + "://"; + if (id.startsWith(prefix)) { + id = id.substring(prefix.length()); + } + return ROUTED_ID_PATTERN.matcher(id).matches() ? id : null; + } + /** * The default implementation of toUri returns an uri with the scheme and value as host * diff --git a/api/src/main/java/com/cloud/offering/NetworkOffering.java b/api/src/main/java/com/cloud/offering/NetworkOffering.java index 5000a4f8c626..b3ab6961fd75 100644 --- a/api/src/main/java/com/cloud/offering/NetworkOffering.java +++ b/api/src/main/java/com/cloud/offering/NetworkOffering.java @@ -81,6 +81,7 @@ enum RoutingMode { public final static String DefaultL2NetworkOfferingVlan = "DefaultL2NetworkOfferingVlan"; public final static String DefaultL2NetworkOfferingConfigDrive = "DefaultL2NetworkOfferingConfigDrive"; public final static String DefaultL2NetworkOfferingConfigDriveVlan = "DefaultL2NetworkOfferingConfigDriveVlan"; + public final static String DefaultL3NetworkOffering = "DefaultL3NetworkOffering"; /** * @return name for the network offering. diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java index 79fb5f6d01cf..d334043d7d55 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java @@ -87,6 +87,12 @@ public class CreateNetworkCmd extends BaseCmd implements UserCmd { + "for shared networks and isolated networks when it belongs to VPC") private String netmask; + @Parameter(name = ApiConstants.CIDR, type = CommandType.STRING, since = "24.0.0", + description = "The IPv4 subnet of the Network in CIDR notation, e.g. 192.0.2.0/24. Supported for L3 (Direct Routed) " + + "networks only, as an alternative to netmask: the IP range defaults to the subnet's usable addresses, " + + "and startip/endip may narrow it") + private String cidr; + @Parameter(name = ApiConstants.START_IP, type = CommandType.STRING, description = "The beginning IP address in the Network IP range") private String startIp; @@ -231,6 +237,10 @@ public String getNetmask() { return netmask; } + public String getCidr() { + return cidr; + } + public String getStartIp() { return startIp; } @@ -340,10 +350,10 @@ public Long getPhysicalNetworkId() { } } if (physicalNetworkId != null) { - if ((offering.getGuestType() == GuestType.Shared) || (offering.getGuestType() == GuestType.L2)) { + if ((offering.getGuestType() == GuestType.Shared) || (offering.getGuestType() == GuestType.L2) || (offering.getGuestType() == GuestType.L3)) { return physicalNetworkId; } else { - throw new InvalidParameterValueException("Physical network ID can be specified for networks of guest IP type " + GuestType.Shared + " or " + GuestType.L2 + " only."); + throw new InvalidParameterValueException(String.format("Physical network ID can be specified for networks of guest IP type %s, %s or %s only.", GuestType.Shared, GuestType.L2, GuestType.L3)); } } else { if (zoneId == null) { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/network/ListNetworkOfferingsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/network/ListNetworkOfferingsCmd.java index ff3b61056be3..0238219b729d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/network/ListNetworkOfferingsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/network/ListNetworkOfferingsCmd.java @@ -82,7 +82,7 @@ public class ListNetworkOfferingsCmd extends BaseListCmd { description = "The ID of the network. Pass this in if you want to see the available network offering that a network can be changed to.") private Long networkId; - @Parameter(name = ApiConstants.GUEST_IP_TYPE, type = CommandType.STRING, description = "List network offerings by guest type: shared or isolated") + @Parameter(name = ApiConstants.GUEST_IP_TYPE, type = CommandType.STRING, description = "List network offerings by guest type: Shared, Isolated, L2 or L3") private String guestIpType; @Parameter(name = ApiConstants.SUPPORTED_SERVICES, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RemoveIpFromVmNicCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RemoveIpFromVmNicCmd.java index f4c4d82b30d4..f8bdfe718d1b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RemoveIpFromVmNicCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/RemoveIpFromVmNicCmd.java @@ -124,6 +124,16 @@ public NetworkType getNetworkType() { } + /** + * A Direct Routed (L3) network needs the agent told when a secondary IP goes away, so the + * host route and neighbour entry are removed - otherwise the host keeps routing an address + * the Instance no longer owns, and the routing daemon keeps advertising it. + */ + private boolean isDirectRoutedNetwork() { + Network ntwk = _entityMgr.findById(Network.class, getNetworkId()); + return ntwk != null && Network.GuestType.L3.equals(ntwk.getGuestType()); + } + private boolean isZoneSGEnabled() { Network ntwk = _entityMgr.findById(Network.class, getNetworkId()); DataCenter dc = _entityMgr.findById(DataCenter.class, ntwk.getDataCenterId()); @@ -144,7 +154,7 @@ public void execute() throws InvalidParameterValueException { secIp = nicSecIp.getIp6Address(); } - if (isZoneSGEnabled()) { + if (isZoneSGEnabled() || isDirectRoutedNetwork()) { //remove the security group rules for this secondary ip boolean success = false; success = _securityGroupService.securityGroupRulesForVmSecIp(nicSecIp.getNicId(), secIp, false); diff --git a/api/src/test/java/com/cloud/network/NetworksTest.java b/api/src/test/java/com/cloud/network/NetworksTest.java index 6f0f3fbd1efe..871b413ec134 100644 --- a/api/src/test/java/com/cloud/network/NetworksTest.java +++ b/api/src/test/java/com/cloud/network/NetworksTest.java @@ -123,6 +123,21 @@ public void otherTypesTest() throws URISyntaxException { Assert.assertEquals("id2 should be \"2\"", "2", id); } + @Test + public void getRoutedIdAcceptsBareAndPrefixedIds() { + Assert.assertEquals("5828", BroadcastDomainType.getRoutedId("5828")); + Assert.assertEquals("5828", BroadcastDomainType.getRoutedId("routed://5828")); + Assert.assertEquals("9999999999", BroadcastDomainType.getRoutedId("9999999999")); + Assert.assertEquals("routed://5828", BroadcastDomainType.Routed.toUri(BroadcastDomainType.getRoutedId("routed://5828")).toString()); + } + + @Test + public void getRoutedIdRejectsMalformedIds() { + for (String candidate : new String[] {null, "", "0", "0534", "abc", "routed://abc", "routed://5828;x", "vlan://5828", "10000000000", "58 28"}) { + Assert.assertNull("expected " + candidate + " to be rejected", BroadcastDomainType.getRoutedId(candidate)); + } + } + @Test public void invalidTypesTest() throws URISyntaxException { String uri1 = "https://1"; diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmdTest.java index 3f5b75828025..8b1b8618094b 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmdTest.java @@ -251,7 +251,7 @@ public void testGetPhysicalNetworkIdForNonSharedNet() { try { cmd.getPhysicalNetworkId(); } catch (Exception e) { - Assert.assertTrue(e.getMessage().startsWith("Physical network ID can be specified for networks of guest IP type Shared or L2 only.")); + Assert.assertTrue(e.getMessage().startsWith("Physical network ID can be specified for networks of guest IP type Shared, L2 or L3 only.")); } } diff --git a/core/src/main/java/com/cloud/agent/api/NetworkRulesVmSecondaryIpCommand.java b/core/src/main/java/com/cloud/agent/api/NetworkRulesVmSecondaryIpCommand.java index c0752a1e97e5..ed6949a48095 100644 --- a/core/src/main/java/com/cloud/agent/api/NetworkRulesVmSecondaryIpCommand.java +++ b/core/src/main/java/com/cloud/agent/api/NetworkRulesVmSecondaryIpCommand.java @@ -28,12 +28,28 @@ public class NetworkRulesVmSecondaryIpCommand extends Command { private String vmSecIp; private String vmMac; private String action; + private boolean directRouted; + private boolean applySecurityGroupRules = true; public NetworkRulesVmSecondaryIpCommand(String vmName, VirtualMachine.Type type) { this.vmName = vmName; this.type = type; } + public NetworkRulesVmSecondaryIpCommand(String vmName, String vmMac, String secondaryIp, boolean action, boolean directRouted, boolean applySecurityGroupRules) { + this(vmName, vmMac, secondaryIp, action); + this.directRouted = directRouted; + this.applySecurityGroupRules = applySecurityGroupRules; + } + + public boolean isDirectRouted() { + return directRouted; + } + + public boolean isApplySecurityGroupRules() { + return applySecurityGroupRules; + } + public NetworkRulesVmSecondaryIpCommand(String vmName, String vmMac, String secondaryIp, boolean action) { this.vmName = vmName; this.vmMac = vmMac; diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java index 109a44488ec4..8ebd251d5b71 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java @@ -218,6 +218,8 @@ void prepare(VirtualMachineProfile profile, DeployDestination dest, ReservationC boolean isSharedNetworkWithoutSpecifyVlan(NetworkOffering offering); + boolean isL3NetworkWithoutSpecifyVlan(NetworkOffering offering); + boolean shutdownNetwork(long networkId, ReservationContext context, boolean cleanupElements); boolean destroyNetwork(long networkId, ReservationContext context, boolean forced); diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index 8af75562b31c..ff342ad79631 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -557,6 +557,15 @@ public boolean configure(final String name, final Map params) th sgProviders.add(Provider.SecurityGroupProvider); defaultSharedSGEnabledNetworkOfferingProviders.put(Service.SecurityGroup, sgProviders); + final Map> defaultL3NetworkOfferingProviders = new HashMap<>(); + final Set configDriveProvider = new HashSet<>(); + configDriveProvider.add(Provider.ConfigDrive); + defaultL3NetworkOfferingProviders.put(Service.UserData, configDriveProvider); + defaultL3NetworkOfferingProviders.put(Service.Dns, configDriveProvider); + final Set l3SecurityGroupProvider = new HashSet<>(); + l3SecurityGroupProvider.add(Provider.SecurityGroupProvider); + defaultL3NetworkOfferingProviders.put(Service.SecurityGroup, l3SecurityGroupProvider); + tungstenProvider.add(Provider.Tungsten); final Map> defaultTungstenSharedSGEnabledNetworkOfferingProviders = new HashMap<>(); defaultTungstenSharedSGEnabledNetworkOfferingProviders.put(Service.Connectivity, tungstenProvider); @@ -620,6 +629,13 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { null, true, false, false, false, false, null, null, null, true, null, null, false); } + if (_networkOfferingDao.findByUniqueName(NetworkOffering.DefaultL3NetworkOffering) == null) { + offering = _configMgr.createNetworkOffering(NetworkOffering.DefaultL3NetworkOffering, + "Offering for Direct Routed (L3) networks - public IPs routed directly to Instances, configuration via ConfigDrive (UserData and DNS), Security Groups enabled, no Virtual Router and no DHCP", + TrafficType.Guest, null, false, Availability.Optional, null, defaultL3NetworkOfferingProviders, true, Network.GuestType.L3, false, null, true, + null, true, false, null, false, null, true, false, false, false, false, null, null, null, true, null, null, false); + } + if (_networkOfferingDao.findByUniqueName(NetworkOffering.DEFAULT_TUNGSTEN_SHARED_NETWORK_OFFERING_WITH_SGSERVICE) == null) { offering = _configMgr.createNetworkOffering(NetworkOffering.DEFAULT_TUNGSTEN_SHARED_NETWORK_OFFERING_WITH_SGSERVICE, "Offering for Tungsten Shared Security group enabled networks", TrafficType.Guest, null, true, Availability.Optional, null, defaultTungstenSharedSGEnabledNetworkOfferingProviders, true, Network.GuestType.Shared, false, null, true, @@ -2940,7 +2956,7 @@ private Network createGuestNetwork(final long networkOfferingId, final String na final boolean vlanSpecified = vlanId != null; if (vlanSpecified != ntwkOff.isSpecifyVlan()) { if (vlanSpecified) { - if (!isSharedNetworkWithoutSpecifyVlan(ntwkOff) && !isPrivateGatewayWithoutSpecifyVlan(ntwkOff)) { + if (!isSharedNetworkWithoutSpecifyVlan(ntwkOff) && !isL3NetworkWithoutSpecifyVlan(ntwkOff) && !isPrivateGatewayWithoutSpecifyVlan(ntwkOff)) { throw new InvalidParameterValueException("Can't specify vlan; corresponding offering says specifyVlan=false"); } } else { @@ -2950,13 +2966,18 @@ private Network createGuestNetwork(final long networkOfferingId, final String na if (vlanSpecified) { URI uri = encodeVlanIdIntoBroadcastUri(vlanId, pNtwk); + if (BroadcastDomainType.getSchemeValue(uri) == BroadcastDomainType.Routed + && _vlanDao.findByZoneAndVlanId(zoneId, uri.toString()) != null) { + throw new InvalidParameterValueException(String.format( + "The routed id %s is already used by a public IP range in zone %s", vlanId, zone.getName())); + } // Aux: generate secondary URI for secondary VLAN ID (if provided) for performing checks URI secondaryUri = StringUtils.isNotBlank(isolatedPvlan) ? BroadcastDomainType.fromString(isolatedPvlan) : null; - if (isSharedNetworkWithoutSpecifyVlan(ntwkOff) || isPrivateGatewayWithoutSpecifyVlan(ntwkOff)) { + if (isSharedNetworkWithoutSpecifyVlan(ntwkOff) || isL3NetworkWithoutSpecifyVlan(ntwkOff) || isPrivateGatewayWithoutSpecifyVlan(ntwkOff)) { bypassVlanOverlapCheck = true; } //don't allow to specify vlan tag used by physical network for dynamic vlan allocation - if (!(bypassVlanOverlapCheck && (ntwkOff.getGuestType() == GuestType.Shared || isPrivateNetwork)) + if (!(bypassVlanOverlapCheck && (ntwkOff.getGuestType() == GuestType.Shared || ntwkOff.getGuestType() == GuestType.L3 || isPrivateNetwork)) && _dcDao.findVnet(zoneId, pNtwk.getId(), BroadcastDomainType.getValue(uri)).size() > 0) { throw new InvalidParameterValueException("The VLAN tag to use for new guest network, " + vlanId + " is already being used for dynamic vlan allocation for the guest network in zone " + zone.getName()); @@ -3097,12 +3118,12 @@ public Network doInTransaction(final TransactionStatus status) { final NetworkVO userNetwork = new NetworkVO(); userNetwork.setNetworkDomain(networkDomainFinal); - if (cidr != null && gateway != null) { + if (cidr != null && (gateway != null || ntwkOff.getGuestType() == GuestType.L3)) { userNetwork.setCidr(cidr); userNetwork.setGateway(gateway); } - if (StringUtils.isNoneBlank(ip6Gateway, ip6Cidr)) { + if (StringUtils.isNotBlank(ip6Cidr) && (StringUtils.isNotBlank(ip6Gateway) || ntwkOff.getGuestType() == GuestType.L3)) { userNetwork.setIp6Cidr(ip6Cidr); userNetwork.setIp6Gateway(ip6Gateway); } @@ -3161,14 +3182,17 @@ public Network doInTransaction(final TransactionStatus status) { uri = encodeVlanIdIntoBroadcastUri(vlanIdFinal, pNtwk); } - if (_networksDao.listByPhysicalNetworkPvlan(physicalNetworkId, uri.toString()).size() > 0) { + final boolean isRoutedUri = uri != null && BroadcastDomainType.getSchemeValue(uri) == BroadcastDomainType.Routed; + if (!isRoutedUri && _networksDao.listByPhysicalNetworkPvlan(physicalNetworkId, uri.toString()).size() > 0) { throw new InvalidParameterValueException(String.format( "Network with vlan %s already exists or overlaps with other network pvlans in zone %s", vlanIdFinal, zone)); } userNetwork.setBroadcastUri(uri); - if (!vlanIdFinal.equalsIgnoreCase(Vlan.UNTAGGED)) { + if (uri != null && BroadcastDomainType.getSchemeValue(uri) == BroadcastDomainType.Routed) { + userNetwork.setBroadcastDomainType(BroadcastDomainType.Routed); + } else if (!vlanIdFinal.equalsIgnoreCase(Vlan.UNTAGGED)) { userNetwork.setBroadcastDomainType(BroadcastDomainType.Vlan); } else { userNetwork.setBroadcastDomainType(BroadcastDomainType.Native); @@ -3233,6 +3257,19 @@ public boolean isSharedNetworkWithoutSpecifyVlan(NetworkOffering offering) { return !offering.isSpecifyVlan(); } + /** + * An L3 (Direct Routed) network whose offering does not carry specifyVlan gets its routed id + * allocated from the physical network's vnet range at creation, the same way a Shared network + * without specifyVlan gets its VLAN — and released the same way on deletion. + */ + @Override + public boolean isL3NetworkWithoutSpecifyVlan(NetworkOffering offering) { + if (offering == null || offering.getTrafficType() != TrafficType.Guest || offering.getGuestType() != GuestType.L3) { + return false; + } + return !offering.isSpecifyVlan(); + } + private boolean isPrivateGatewayWithoutSpecifyVlan(NetworkOffering ntwkOff) { return ntwkOff.getId() == _networkOfferingDao.findByUniqueName(NetworkOffering.SystemPrivateGatewayNetworkOfferingWithoutVlan).getId(); } @@ -3250,9 +3287,13 @@ protected URI encodeVlanIdIntoBroadcastUri(String vlanId, PhysicalNetwork pNtwk) if (!pNtwk.getIsolationMethods().isEmpty() && StringUtils.isNotBlank(pNtwk.getIsolationMethods().get(0))) { String isolationMethod = pNtwk.getIsolationMethods().get(0).toLowerCase(); String vxlan = BroadcastDomainType.Vxlan.toString().toLowerCase(); + String routed = BroadcastDomainType.Routed.toString().toLowerCase(); if (isolationMethod.equals(vxlan)) { return BroadcastDomainType.encodeStringIntoBroadcastUri(vlanId, BroadcastDomainType.Vxlan); } + if (isolationMethod.equals(routed)) { + return BroadcastDomainType.encodeStringIntoBroadcastUri(vlanId, BroadcastDomainType.Routed); + } } return BroadcastDomainType.fromString(vlanId); } @@ -3668,8 +3709,8 @@ protected Pair> deleteVlansInNetwork(final NetworkVO netwo logger.debug("Deleted ip range for private network {}", network); } - // release vlans of user-shared networks without specifyvlan - if (isSharedNetworkWithoutSpecifyVlan(_networkOfferingDao.findById(network.getNetworkOfferingId()))) { + final NetworkOffering deletedNetworkOffering = _networkOfferingDao.findById(network.getNetworkOfferingId()); + if (isSharedNetworkWithoutSpecifyVlan(deletedNetworkOffering) || isL3NetworkWithoutSpecifyVlan(deletedNetworkOffering)) { logger.debug("Releasing vnet for the network {}", network); _dcDao.releaseVnet(BroadcastDomainType.getValue(network.getBroadcastUri()), network.getDataCenterId(), network.getPhysicalNetworkId(), network.getAccountId(), network.getReservationId()); diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index c68dc390df2a..a4ce7f850b89 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -5831,15 +5831,20 @@ public Vlan createVlanAndPublicIpRange(final long zoneId, final long networkId, vlanId = Vlan.UNTAGGED; } + if (vlanId != null && vlanId.startsWith(BroadcastDomainType.Routed.scheme() + "://")) { + vlanId = canonicalizeRoutedRangeId(zoneId, vlanId); + } + final VlanType vlanType = forVirtualNetwork ? VlanType.VirtualNetwork : VlanType.DirectAttached; if ((domain != null || vlanOwner != null) && zone.getNetworkType() != NetworkType.Advanced) { throw new InvalidParameterValueException("Vlan owner can be defined only in the zone of type " + NetworkType.Advanced); } + final boolean gatewaylessL3 = network.getGuestType() == GuestType.L3; if (ipv4) { // Make sure the gateway is valid - if (!NetUtils.isValidIp4(vlanGateway)) { + if (!(gatewaylessL3 && vlanGateway == null) && !NetUtils.isValidIp4(vlanGateway)) { throw new InvalidParameterValueException("Please specify a valid gateway"); } @@ -5850,7 +5855,7 @@ public Vlan createVlanAndPublicIpRange(final long zoneId, final long networkId, } if (ipv6) { - if (!NetUtils.isValidIp6(vlanIp6Gateway)) { + if (!(gatewaylessL3 && vlanIp6Gateway == null) && !NetUtils.isValidIp6(vlanIp6Gateway)) { throw new InvalidParameterValueException("Please specify a valid IPv6 gateway"); } if (!NetUtils.isValidIp6Cidr(vlanIp6Cidr)) { @@ -5858,12 +5863,14 @@ public Vlan createVlanAndPublicIpRange(final long zoneId, final long networkId, } } - boolean isSharedNetworkWithoutSpecifyVlan = _networkMgr.isSharedNetworkWithoutSpecifyVlan(_networkOfferingDao.findById(network.getNetworkOfferingId())); + final NetworkOffering networkOffering = _networkOfferingDao.findById(network.getNetworkOfferingId()); + boolean isSharedNetworkWithoutSpecifyVlan = _networkMgr.isSharedNetworkWithoutSpecifyVlan(networkOffering); + boolean isL3NetworkWithoutSpecifyVlan = _networkMgr.isL3NetworkWithoutSpecifyVlan(networkOffering); if (ipv4) { - final String newCidr = NetUtils.getCidrFromGatewayAndNetmask(vlanGateway, vlanNetmask); + final String newCidr = NetUtils.getCidrFromGatewayAndNetmask(vlanGateway != null ? vlanGateway : startIP, vlanNetmask); //Make sure start and end ips are with in the range of cidr calculated for this gateway and netmask { - if (!NetUtils.isIpWithInCidrRange(vlanGateway, newCidr) || !NetUtils.isIpWithInCidrRange(startIP, newCidr) || !NetUtils.isIpWithInCidrRange(endIP, newCidr)) { + if ((vlanGateway != null && !NetUtils.isIpWithInCidrRange(vlanGateway, newCidr)) || !NetUtils.isIpWithInCidrRange(startIP, newCidr) || !NetUtils.isIpWithInCidrRange(endIP, newCidr)) { throw new InvalidParameterValueException("Please specify a valid IP range or valid netmask or valid gateway"); } @@ -5886,6 +5893,10 @@ public Vlan createVlanAndPublicIpRange(final long zoneId, final long networkId, if (!isSharedNetworkWithoutSpecifyVlan) { checkZoneVlanIpOverlap(zone, network, newCidr, vlanId, vlanGateway, vlanNetmask, startIP, endIP); } + + if (gatewaylessL3) { + checkOverlapPublicIpRange(zoneId, startIP, endIP); + } } String ipv6Range = null; @@ -5897,7 +5908,7 @@ public Vlan createVlanAndPublicIpRange(final long zoneId, final long networkId, final List vlans = _vlanDao.listByZone(zone.getId()); for (final VlanVO vlan : vlans) { - if (vlan.getIp6Gateway() == null) { + if (vlan.getIp6Cidr() == null) { continue; } if ((StringUtils.isAllEmpty(ipv6Range, vlan.getIp6Range())) && @@ -5917,7 +5928,7 @@ public Vlan createVlanAndPublicIpRange(final long zoneId, final long networkId, } // Check if the vlan is being used - if (isSharedNetworkWithoutSpecifyVlan) { + if (isSharedNetworkWithoutSpecifyVlan || isL3NetworkWithoutSpecifyVlan) { bypassVlanOverlapCheck = true; } if (!bypassVlanOverlapCheck && !forExternalProvider && !_zoneDao.findVnet(zoneId, physicalNetworkId, BroadcastDomainType.getValue(BroadcastDomainType.fromString(vlanId))).isEmpty()) { @@ -5940,7 +5951,7 @@ public Vlan createVlanAndPublicIpRange(final long zoneId, final long networkId, if (vlan != null && network.getTrafficType() != TrafficType.Public) { if (ipv4) { - addCidrAndGatewayForIpv4(networkId, vlanGateway, vlanNetmask); + addCidrAndGatewayForIpv4(networkId, vlanGateway, vlanNetmask, startIP); } else if (ipv6) { addCidrAndGatewayForIpv6(networkId, vlanIp6Gateway, vlanIp6Cidr); } @@ -5949,10 +5960,48 @@ public Vlan createVlanAndPublicIpRange(final long zoneId, final long networkId, return vlan; } - private void addCidrAndGatewayForIpv4(final long networkId, final String vlanGateway, final String vlanNetmask) { + /** + * Validates the routed id of a public IP range created with vlan=routed://<id> (SystemVMs on + * a ROUTED physical network). The id must be a well-formed routed id, must not already name a + * guest network's bridge anywhere in the zone, and must not fall inside the routed-id range + * of any ROUTED physical network, from which L3 networks without specifyVlan draw theirs: + * guest networks and public ranges share one id space, since both name a brdr-<id> bridge on + * the hosts. + * + * @return the canonical routed://<id> form + */ + protected String canonicalizeRoutedRangeId(final long zoneId, final String vlanId) { + final String routedId = BroadcastDomainType.getRoutedId(vlanId); + if (routedId == null) { + throw new InvalidParameterValueException(String.format( + "%s is not a valid routed id: expected routed:// with a positive integer of at most %d digits", vlanId, BroadcastDomainType.ROUTED_ID_MAX_DIGITS)); + } + final String canonicalVlanId = BroadcastDomainType.Routed.toUri(routedId).toString(); + if (!_networkDao.listByZoneAndUriAndGuestType(zoneId, canonicalVlanId, null).isEmpty()) { + throw new InvalidParameterValueException(String.format("The routed id %s is already used by a guest network in zone %d", routedId, zoneId)); + } + for (final PhysicalNetworkVO physicalNetwork : _physicalNetworkDao.listByZone(zoneId)) { + if (physicalNetwork.getIsolationMethods() == null || !physicalNetwork.getIsolationMethods().contains("ROUTED")) { + continue; + } + if (!_zoneDao.findVnet(zoneId, physicalNetwork.getId(), routedId).isEmpty()) { + throw new InvalidParameterValueException(String.format( + "The routed id %s lies inside the routed id range of physical network %s, from which %s networks are allocated their ids", + routedId, physicalNetwork.getName(), GuestType.L3)); + } + } + return canonicalVlanId; + } + + /** + * Appends the new range's subnet, and its gateway when the range carries one, to the network + * row. A gateway-less range (L3 networks) derives the same subnet from its start IP, and its + * absent gateway leaves the network's gateway untouched. + */ + private void addCidrAndGatewayForIpv4(final long networkId, final String vlanGateway, final String vlanNetmask, final String startIP) { final NetworkVO networkVO = _networkDao.findById(networkId); String networkCidr = networkVO.getCidr(); - String newCidr = NetUtils.getCidrFromGatewayAndNetmask(vlanGateway, vlanNetmask); + String newCidr = NetUtils.getCidrFromGatewayAndNetmask(vlanGateway != null ? vlanGateway : startIP, vlanNetmask); String newNetworkCidr = com.cloud.utils.StringUtils.updateCommaSeparatedStringWithValue(networkCidr, newCidr, true); networkVO.setCidr(newNetworkCidr); @@ -5999,6 +6048,8 @@ private String getNetworkVlanId(Network network, boolean connectivityWithoutVlan if (network.getBroadcastDomainType() != BroadcastDomainType.Vlan) { networkVlanId = networkVlanId.split("-")[0]; } + } else if (BroadcastDomainType.getSchemeValue(uri) == BroadcastDomainType.Routed) { + networkVlanId = BroadcastDomainType.getValue(uri); } } return networkVlanId; @@ -6013,10 +6064,10 @@ private void checkZoneVlanIpOverlap(DataCenterVO zone, Network network, String n final String otherVlanGateway = vlan.getVlanGateway(); final String otherVlanNetmask = vlan.getVlanNetmask(); // Continue if it's not IPv4 - if (ObjectUtils.anyNull(otherVlanGateway, otherVlanNetmask, vlan.getNetworkId())) { + if (ObjectUtils.anyNull(otherVlanNetmask, vlan.getNetworkId()) || (otherVlanGateway == null && StringUtils.isBlank(vlan.getIpRange()))) { continue; } - final String otherCidr = NetUtils.getCidrFromGatewayAndNetmask(otherVlanGateway, otherVlanNetmask); + final String otherCidr = NetUtils.getCidrFromGatewayAndNetmask(otherVlanGateway != null ? otherVlanGateway : vlan.getIpRange().split("\\-")[0], otherVlanNetmask); if( !NetUtils.isNetworksOverlap(newCidr, otherCidr)) { continue; } @@ -6035,7 +6086,7 @@ private void checkZoneVlanIpOverlap(DataCenterVO zone, Network network, String n } // extend IP range - if (!vlanGateway.equals(otherVlanGateway) || !vlanNetmask.equals(vlan.getVlanNetmask())) { + if (!Objects.equals(vlanGateway, otherVlanGateway) || !Objects.equals(vlanNetmask, vlan.getVlanNetmask())) { throw new InvalidParameterValueException("The IP range has already been added with gateway " + otherVlanGateway + " ,and netmask " + otherVlanNetmask + ", Please specify the gateway/netmask if you want to extend ip range" ); @@ -6053,7 +6104,7 @@ private void checkZoneVlanIpOverlap(DataCenterVO zone, Network network, String n final Long nwId = vlan.getNetworkId(); if (nwId != null) { final Network nw = _networkModel.getNetwork(nwId); - if (nw != null && nw.getTrafficType() == TrafficType.Public) { + if (nw != null && (nw.getTrafficType() == TrafficType.Public || nw.getGuestType() == GuestType.L3)) { overlapped = true; } } @@ -6839,12 +6890,14 @@ private void checkPublicIpRangeErrors(final long zoneId, final String vlanId, fi throw new InvalidParameterValueException("Please ensure that your start IP and end IP are in the same subnet, as per the IP range's netmask."); } - if (!NetUtils.sameSubnet(startIP, vlanGateway, vlanNetmask)) { - throw new InvalidParameterValueException("Please ensure that your start IP is in the same subnet as your IP range's gateway, as per the IP range's netmask."); - } + if (vlanGateway != null) { + if (!NetUtils.sameSubnet(startIP, vlanGateway, vlanNetmask)) { + throw new InvalidParameterValueException("Please ensure that your start IP is in the same subnet as your IP range's gateway, as per the IP range's netmask."); + } - if (endIP != null && !NetUtils.sameSubnet(endIP, vlanGateway, vlanNetmask)) { - throw new InvalidParameterValueException("Please ensure that your end IP is in the same subnet as your IP range's gateway, as per the IP range's netmask."); + if (endIP != null && !NetUtils.sameSubnet(endIP, vlanGateway, vlanNetmask)) { + throw new InvalidParameterValueException("Please ensure that your end IP is in the same subnet as your IP range's gateway, as per the IP range's netmask."); + } } // check if the gatewayip is the part of the ip range being added. // RFC 3021 - 31-Bit Prefixes on IPv4 Point-to-Point Links @@ -6852,11 +6905,13 @@ private void checkPublicIpRangeErrors(final long zoneId, final String vlanId, fi // 192.168.24.0 - 255.255.255.254 - 192.168.24.0 - 192.168.24.1 // https://tools.ietf.org/html/rfc3021 // Added by Wilder Rodrigues - final String newCidr = NetUtils.getCidrFromGatewayAndNetmask(vlanGateway, vlanNetmask); - if (!NetUtils.is31PrefixCidr(newCidr)) { - if (NetUtils.ipRangesOverlap(startIP, endIP, vlanGateway, vlanGateway)) { - throw new InvalidParameterValueException( - "The gateway ip should not be the part of the ip range being added."); + if (vlanGateway != null) { + final String newCidr = NetUtils.getCidrFromGatewayAndNetmask(vlanGateway, vlanNetmask); + if (!NetUtils.is31PrefixCidr(newCidr)) { + if (NetUtils.ipRangesOverlap(startIP, endIP, vlanGateway, vlanGateway)) { + throw new InvalidParameterValueException( + "The gateway ip should not be the part of the ip range being added."); + } } } } @@ -7214,7 +7269,7 @@ public NetworkOffering createNetworkOffering(final NetworkOfferingBaseCmd cmd) { } if (guestType == null) { - throw new InvalidParameterValueException("Invalid \"type\" parameter is given; can have Shared and Isolated values"); + throw new InvalidParameterValueException("Invalid \"type\" parameter is given; supported values are " + Arrays.toString(Network.GuestType.values())); } if (internetProtocol != null) { @@ -7271,9 +7326,8 @@ public NetworkOffering createNetworkOffering(final NetworkOfferingBaseCmd cmd) { } if (service == Service.SecurityGroup) { - // allow security group service for Shared networks only - if (guestType != GuestType.Shared) { - throw new InvalidParameterValueException("Security group service is supported for network offerings with guest ip type " + GuestType.Shared); + if (guestType != GuestType.Shared && guestType != GuestType.L3) { + throw new InvalidParameterValueException(String.format("Security group service is supported for network offerings with guest ip type %s or %s", GuestType.Shared, GuestType.L3)); } final Set sgProviders = new HashSet<>(); sgProviders.add(Provider.SecurityGroupProvider); @@ -7369,6 +7423,10 @@ public NetworkOffering createNetworkOffering(final NetworkOfferingBaseCmd cmd) { // validate providers combination here _networkModel.canProviderSupportServices(providerCombinationToVerify); + if (guestType == GuestType.L3) { + validateL3NetworkOffering(serviceProviderMap, networkMode, specifyVlan, specifyIpRanges, forVpc); + } + // validate the LB service capabilities specified in the network // offering final Map lbServiceCapabilityMap = cmd.getServiceCapabilities(Service.Lb); @@ -7471,6 +7529,49 @@ public NetworkOffering createNetworkOffering(final NetworkOfferingBaseCmd cmd) { return offering; } + /** + * Validates a network offering for the L3 (Direct Routed) guest type. There is no Virtual + * Router and no DHCP on these networks: the Instance learns its /32 (and /128) address, + * on-link gateway and routes exclusively from ConfigDrive. UserData via ConfigDrive is + * therefore mandatory. Dns is optional (a template may carry its own resolvers) but must be + * provided by ConfigDrive when present. SecurityGroup is the only other permitted service. + * specifyVlan is a free choice: with it the operator supplies the routed id (routed://<id>, + * naming the per-network bridge) at network creation via the vlan parameter; without it + * CloudStack allocates one from the ROUTED physical network's vnet range. + */ + protected void validateL3NetworkOffering(final Map> serviceProviderMap, final NetworkOffering.NetworkMode networkMode, + final boolean specifyVlan, final boolean specifyIpRanges, final Boolean forVpc) { + if (Boolean.TRUE.equals(forVpc)) { + throw new InvalidParameterValueException(String.format("VPC is not supported for network offerings with guest type %s", GuestType.L3)); + } + if (networkMode != null) { + throw new InvalidParameterValueException(String.format("Network mode can not be specified for network offerings with guest type %s", GuestType.L3)); + } + if (!specifyIpRanges) { + throw new InvalidParameterValueException(String.format("Network offerings with guest type %s must specify IP ranges", GuestType.L3)); + } + if (serviceProviderMap.containsKey(Service.Dhcp)) { + throw new InvalidParameterValueException(String.format("DHCP is not supported (and not needed) for network offerings with guest type %s; addressing is delivered via ConfigDrive", GuestType.L3)); + } + final Set allowedL3Services = new HashSet<>(Arrays.asList(Service.UserData, Service.Dns, Service.SecurityGroup)); + for (final Service service : serviceProviderMap.keySet()) { + if (!allowedL3Services.contains(service)) { + throw new InvalidParameterValueException(String.format("Service %s is not supported for network offerings with guest type %s; supported services are %s", + service.getName(), GuestType.L3, StringUtils.join(allowedL3Services.stream().map(Service::getName).toArray(), ", "))); + } + } + final Set configDriveOnly = Collections.singleton(Provider.ConfigDrive); + final Set userDataProviders = serviceProviderMap.get(Service.UserData); + if (!configDriveOnly.equals(userDataProviders)) { + throw new InvalidParameterValueException(String.format("UserData with provider %s is mandatory for network offerings with guest type %s; it is the only channel that carries the Instance's network configuration", + Provider.ConfigDrive.getName(), GuestType.L3)); + } + final Set dnsProviders = serviceProviderMap.get(Service.Dns); + if (dnsProviders != null && !configDriveOnly.equals(dnsProviders)) { + throw new InvalidParameterValueException(String.format("DNS on network offerings with guest type %s must use provider %s", GuestType.L3, Provider.ConfigDrive.getName())); + } + } + public static NetworkOffering.RoutingMode verifyRoutingMode(String routingModeString) { NetworkOffering.RoutingMode routingMode = null; if (routingModeString != null) { diff --git a/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java b/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java index 508d15fe3fc4..9419098f8709 100644 --- a/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java +++ b/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java @@ -2437,8 +2437,7 @@ public void allocateDirectIp(final NicProfile nic, final DataCenter dc, final Vi @Override public void doInTransactionWithoutResult(TransactionStatus status) throws InsufficientAddressCapacityException { //This method allocates direct ip for the Shared network in Advance zones - boolean ipv4 = false; - if (network.getGateway() != null) { + if (network.getGateway() != null || (GuestType.L3 == network.getGuestType() && network.getCidr() != null)) { if (nic.getIPv4Address() == null) { PublicIp ip = null; diff --git a/server/src/main/java/com/cloud/network/Ipv6AddressManagerImpl.java b/server/src/main/java/com/cloud/network/Ipv6AddressManagerImpl.java index d096b09ec0d4..f7020ea092d4 100644 --- a/server/src/main/java/com/cloud/network/Ipv6AddressManagerImpl.java +++ b/server/src/main/java/com/cloud/network/Ipv6AddressManagerImpl.java @@ -204,7 +204,7 @@ protected boolean isIp6Taken(Network network, String requestedIpv6) { */ @Override public void setNicIp6Address(final NicProfile nic, final DataCenter dc, final Network network) throws InsufficientAddressCapacityException { - if (network.getIp6Gateway() != null) { + if (network.getIp6Gateway() != null || (Network.GuestType.L3 == network.getGuestType() && network.getIp6Cidr() != null)) { if (nic.getIPv6Address() == null) { logger.debug("Found IPv6 CIDR " + network.getIp6Cidr() + " for Network " + network); nic.setIPv6Cidr(network.getIp6Cidr()); diff --git a/server/src/main/java/com/cloud/network/NetworkModelImpl.java b/server/src/main/java/com/cloud/network/NetworkModelImpl.java index f47046cdc434..7cccc9b99921 100644 --- a/server/src/main/java/com/cloud/network/NetworkModelImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkModelImpl.java @@ -749,6 +749,12 @@ public Network getNetwork(long id) { return _networksDao.findById(id); } + /** + * A Shared network's IPv4 presence is its gateway; a gateway-less L3 network's is its cidr — + * when either is present, a free address must remain in the network's IPv4 pool. IPv6 needs + * no free-IP check: addresses are computed with EUI-64 from the subnet and the NIC MAC, so an + * IPv6-only network never runs out. + */ @Override public boolean canUseForDeploy(Network network) { if (network.getTrafficType() != TrafficType.Guest) { @@ -758,8 +764,8 @@ public boolean canUseForDeploy(Network network) { return true; // do not check free IPs if there is no service in the network } boolean hasFreeIps = true; - if (network.getGuestType() == GuestType.Shared) { - if (network.getGateway() != null) { + if (network.getGuestType() == GuestType.Shared || network.getGuestType() == GuestType.L3) { + if (network.getGateway() != null || (network.getGuestType() == GuestType.L3 && network.getCidr() != null)) { hasFreeIps = _ipAddressDao.countFreeIPsInNetwork(network.getId()) > 0; } if (!hasFreeIps) { @@ -3097,7 +3103,8 @@ public boolean checkSecurityGroupSupportForNetwork(Account account, DataCenter z if (network == null) { throw new InvalidParameterValueException("Unable to find network by id " + networkId); } - if (network.getGuestType() == Network.GuestType.Shared && isSecurityGroupSupportedInNetwork(network)) { + if ((network.getGuestType() == Network.GuestType.Shared || network.getGuestType() == Network.GuestType.L3) + && isSecurityGroupSupportedInNetwork(network)) { return true; } } diff --git a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java index 2853fa96330d..1251658d7070 100644 --- a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java @@ -868,6 +868,132 @@ public boolean stop() { protected NetworkServiceImpl() { } + /** + * An L3 (Direct Routed) network has no DHCP and no password/metadata service, so nothing in + * it depends on IPv4: each address family is optional, making IPv6-only networks possible. + * Gateways play no part at all — the Instance's gateway is always the shared link-local + * address, so none needs to be declared (or burnt in the subnet). What remains mandatory is + * that a given family is complete — IPv4 is a subnet (cidr, or netmask and startip; endip + * defaults to startip), IPv6 is ip6cidr alone (no range: addresses derive from the subnet + * and the NIC MAC with EUI-64) — and that at least one family is present at all. A cidr + * parameter has already been expanded into netmask and startip when this runs. + */ + protected void validateL3AddressFamilies(String netmask, String startIP, String endIP, + String ip6Cidr, String startIPv6, String endIPv6) { + boolean anyIpv4 = !StringUtils.isAllBlank(netmask, startIP, endIP); + boolean completeIpv4 = StringUtils.isNoneBlank(netmask, startIP); + boolean anyIpv6 = !StringUtils.isAllBlank(ip6Cidr, startIPv6, endIPv6); + boolean completeIpv6 = StringUtils.isNotBlank(ip6Cidr); + if (anyIpv4 && !completeIpv4) { + throw new InvalidParameterValueException(String.format( + "IPv4 is optional for %s networks, but when any IPv4 detail is given, either cidr, or netmask and startip, is required", GuestType.L3)); + } + if (anyIpv6 && !completeIpv6) { + throw new InvalidParameterValueException(String.format( + "IPv6 is optional for %s networks, but when any IPv6 detail is given, ip6cidr is required", GuestType.L3)); + } + if (!anyIpv4 && !anyIpv6) { + throw new InvalidParameterValueException(String.format( + "A %s network needs at least one address family: IPv4 (cidr, or netmask and startip) or IPv6 (ip6cidr)", GuestType.L3)); + } + } + + /** + * Expands the cidr parameter of an L3 network — the IPv4 counterpart of ip6cidr — into the + * netmask/startip/endip triple the rest of the creation flow works with. Without startip the + * IP range defaults to the subnet's usable addresses (network and broadcast excluded; give + * startip and endip explicitly to include them). An explicit startip/endip narrows the range + * and must lie inside the subnet, which is what keeps the network's stored subnet — derived + * from startip and netmask further down the flow — equal to the requested one. cidr and + * netmask are two ways of defining the same subnet, so they are mutually exclusive. + * + * @return {netmask, startIP, endIP} + */ + protected String[] expandL3Ipv4Cidr(String requestedCidr, String netmask, String startIP, String endIP) { + if (StringUtils.isNotBlank(netmask)) { + throw new InvalidParameterValueException("Specify either cidr or netmask, not both: they both define the IPv4 subnet"); + } + if (!NetUtils.isValidIp4Cidr(requestedCidr)) { + throw new InvalidParameterValueException(String.format("Invalid cidr %s", requestedCidr)); + } + if (StringUtils.isBlank(startIP) && StringUtils.isNotBlank(endIP)) { + throw new InvalidParameterValueException("endip requires startip"); + } + final Pair parsedCidr = NetUtils.getCidr(requestedCidr); + final String derivedNetmask = NetUtils.getCidrNetmask(parsedCidr.second()); + if (StringUtils.isBlank(startIP)) { + if (parsedCidr.second() > 30) { + throw new InvalidParameterValueException(String.format( + "A /%d subnet has no derivable IP range; give startip and endip explicitly", parsedCidr.second())); + } + final String[] range = NetUtils.getIpRangeFromCidr(parsedCidr.first(), parsedCidr.second()); + startIP = range[0]; + endIP = range[1]; + } else { + for (String ip : new String[] {startIP, endIP}) { + if (StringUtils.isBlank(ip)) { + continue; + } + if (!NetUtils.isValidIp4(ip) || !NetUtils.isIpWithInCidrRange(ip, requestedCidr)) { + throw new InvalidParameterValueException(String.format("IP address %s is not within the subnet %s", ip, requestedCidr)); + } + } + } + return new String[] {derivedNetmask, startIP, endIP}; + } + + /** + * Canonicalises the routed id given for an L3 network through the vlan parameter. The id + * names the network's bridge on every hypervisor and derives that bridge's MAC address, so it + * is a positive integer without leading zeros; the routed:// scheme may be given or omitted. + * + * @return the bare id, which the orchestrator encodes into the routed://<id> broadcast URI + */ + protected String canonicalizeRoutedId(String vlanId) { + String routedId = BroadcastDomainType.getRoutedId(vlanId); + if (routedId == null) { + throw new InvalidParameterValueException(String.format( + "The vlan parameter of a %s network is its routed id, a positive integer of at most %d digits (optionally prefixed with routed://), got %s", + GuestType.L3, BroadcastDomainType.ROUTED_ID_MAX_DIGITS, vlanId)); + } + return routedId; + } + + /** + * The L3 counterpart of {@code NetworkModel.checkIp6Parameters()}: no gateway is involved, + * and the range is optional — when given it must lie inside the subnet. + */ + protected void checkL3Ip6Parameters(String startIPv6, String endIPv6, String ip6Cidr) { + if (!NetUtils.isValidIp6Cidr(ip6Cidr)) { + throw new InvalidParameterValueException("Invalid ip6cidr"); + } + for (String ip : new String[] {startIPv6, endIPv6}) { + if (StringUtils.isBlank(ip)) { + continue; + } + if (!NetUtils.isValidIp6(ip)) { + throw new InvalidParameterValueException(String.format("Invalid IPv6 address %s", ip)); + } + if (!NetUtils.isIp6InNetwork(ip, ip6Cidr)) { + throw new InvalidParameterValueException(String.format("IPv6 address %s is not within the subnet %s", ip, ip6Cidr)); + } + } + } + + /** + * Whether the NIC sits on a Direct Routed (L3) network. Such a NIC needs the agent told about + * secondary IPs regardless of the zone's security group setting: the host route and neighbour + * entry the agent installs are what make the address reachable. + */ + protected boolean isDirectRoutedNic(long nicId) { + NicVO nic = _nicDao.findById(nicId); + if (nic == null) { + return false; + } + Network network = _networksDao.findById(nic.getNetworkId()); + return network != null && GuestType.L3.equals(network.getGuestType()); + } + @Override @ActionEvent(eventType = EventTypes.EVENT_NIC_SECONDARY_IP_CONFIGURE, eventDescription = "Configuring secondary IP " + "rules", async = true) public boolean configureNicSecondaryIp(NicSecondaryIp secIp, boolean isZoneSgEnabled) { @@ -877,9 +1003,9 @@ public boolean configureNicSecondaryIp(NicSecondaryIp secIp, boolean isZoneSgEna secondaryIp = secIp.getIp6Address(); } - if (isZoneSgEnabled) { + if (isZoneSgEnabled || isDirectRoutedNic(secIp.getNicId())) { success = _securityGroupService.securityGroupRulesForVmSecIp(secIp.getNicId(), secondaryIp, true); - logger.info("Associated IP address to NIC : " + secIp.getIp4Address()); + logger.info("Associated IP address to NIC : " + secondaryIp); } else { success = true; } @@ -941,7 +1067,7 @@ public NicSecondaryIp allocateSecondaryGuestIP(final long nicId, IpAddresses req if (StringUtils.isNotBlank(ipv6Address)) { ip6addr = ipv6AddrMgr.allocateGuestIpv6(network, ipv6Address); } - } else if (network.getGuestType() == Network.GuestType.Shared) { + } else if (network.getGuestType() == Network.GuestType.Shared || network.getGuestType() == Network.GuestType.L3) { //for basic zone, need to provide the podId to ensure proper IP allocation Long podId = null; DataCenter dc = _dcDao.findById(network.getDataCenterId()); @@ -1504,6 +1630,7 @@ public Network createGuestNetwork(CreateNetworkCmd cmd) throws InsufficientCapac String startIP = cmd.getStartIp(); String endIP = cmd.getEndIp(); String netmask = cmd.getNetmask(); + final String requestedCidr = cmd.getCidr(); String networkDomain = cmd.getNetworkDomain(); boolean adminCalledUs = cmd instanceof CreateNetworkCmdByAdmin; @@ -1558,6 +1685,13 @@ public Network createGuestNetwork(CreateNetworkCmd cmd) throws InsufficientCapac ACLType aclType = getAclType(caller, cmd.getAclType(), ntwkOff); + if (ntwkOff.getGuestType() == GuestType.L3 + && (pNtwk.getIsolationMethods() == null || !pNtwk.getIsolationMethods().contains("ROUTED"))) { + throw new InvalidParameterValueException(String.format( + "Networks of guest type %s can only be created on a physical network with isolation method ROUTED; physical network %s carries %s", + GuestType.L3, pNtwk.getName(), pNtwk.getIsolationMethods())); + } + if (ntwkOff.getGuestType() != GuestType.Shared && (!StringUtils.isAllBlank(routerIPv4, routerIPv6))) { throw new InvalidParameterValueException("Router IP can be specified only for Shared networks"); } @@ -1581,12 +1715,37 @@ public Network createGuestNetwork(CreateNetworkCmd cmd) throws InsufficientCapac } } - boolean ipv4 = false, ipv6 = false; - if (org.apache.commons.lang3.StringUtils.isNoneBlank(gateway, netmask)) { - ipv4 = true; + final boolean isL3 = ntwkOff.getGuestType() == GuestType.L3; + if (!isL3 && StringUtils.isNotBlank(requestedCidr)) { + throw new InvalidParameterValueException(String.format( + "The cidr parameter is supported for %s networks only; use gateway and netmask instead", GuestType.L3)); + } + if (isL3) { + if (StringUtils.isNotBlank(requestedCidr)) { + String[] expanded = expandL3Ipv4Cidr(requestedCidr, netmask, startIP, endIP); + netmask = expanded[0]; + startIP = expanded[1]; + endIP = expanded[2]; + } + validateL3AddressFamilies(netmask, startIP, endIP, ip6Cidr, startIPv6, endIPv6); + if (!StringUtils.isAllBlank(gateway, ip6Gateway)) { + logger.debug("Ignoring the gateway(s) given for {} network {}: instances always use the shared link-local gateway", GuestType.L3, name); + gateway = null; + ip6Gateway = null; + } } - if (StringUtils.isNoneBlank(ip6Cidr, ip6Gateway)) { - ipv6 = true; + + boolean ipv4 = false, ipv6 = false; + if (isL3) { + ipv4 = StringUtils.isNoneBlank(netmask, startIP); + ipv6 = StringUtils.isNotBlank(ip6Cidr); + } else { + if (org.apache.commons.lang3.StringUtils.isNoneBlank(gateway, netmask)) { + ipv4 = true; + } + if (StringUtils.isNoneBlank(ip6Cidr, ip6Gateway)) { + ipv6 = true; + } } if (gateway != null) { @@ -1626,11 +1785,16 @@ public Network createGuestNetwork(CreateNetworkCmd cmd) throws InsufficientCapac } else if (!NetUtils.isValidIp4(endIP)) { throw new InvalidParameterValueException("Invalid format for the endIp parameter"); } - if (!(gateway != null && netmask != null)) { + if (!(netmask != null && (gateway != null || isL3))) { throw new InvalidParameterValueException("gateway and netmask should be defined when startIP/endIP are passed in"); } } - if (gateway != null && netmask != null) { + if (isL3 && netmask != null && startIP != null) { + if (!NetUtils.isValidIp4Netmask(netmask)) { + throw new InvalidParameterValueException("Invalid netmask"); + } + cidr = NetUtils.getCidrFromGatewayAndNetmask(startIP, netmask); + } else if (gateway != null && netmask != null) { if (NetUtils.isNetworkorBroadcastIP(gateway, netmask)) { if (logger.isDebugEnabled()) { logger.debug("The gateway IP provided is " + gateway + " and netmask is " + netmask + ". The IP is either broadcast or network IP."); @@ -1654,16 +1818,24 @@ public Network createGuestNetwork(CreateNetworkCmd cmd) throws InsufficientCapac if (endIPv6 == null) { endIPv6 = startIPv6; } - _networkModel.checkIp6Parameters(startIPv6, endIPv6, ip6Gateway, ip6Cidr); - if (!GuestType.Shared.equals(ntwkOff.getGuestType())) { + if (isL3) { + checkL3Ip6Parameters(startIPv6, endIPv6, ip6Cidr); + } else { + _networkModel.checkIp6Parameters(startIPv6, endIPv6, ip6Gateway, ip6Cidr); + } + if (!GuestType.Shared.equals(ntwkOff.getGuestType()) && !GuestType.L3.equals(ntwkOff.getGuestType())) { _networkModel.checkIp6CidrSizeEqualTo64(ip6Cidr); } + if (GuestType.L3.equals(ntwkOff.getGuestType()) && NetUtils.getIp6CidrSize(ip6Cidr) > 64) { + throw new InvalidParameterValueException(String.format( + "The IPv6 subnet of a %s network must be /64 or larger: addresses are derived with EUI-64 from the subnet and the NIC MAC", GuestType.L3)); + } - if (zone.getNetworkType() != NetworkType.Advanced || ntwkOff.getGuestType() != Network.GuestType.Shared) { - throw new InvalidParameterValueException("Can only support create IPv6 network with advance shared network!"); + if (zone.getNetworkType() != NetworkType.Advanced || (ntwkOff.getGuestType() != Network.GuestType.Shared && ntwkOff.getGuestType() != Network.GuestType.L3)) { + throw new InvalidParameterValueException(String.format("Can only support create IPv6 network with advanced %s or %s network!", GuestType.Shared, GuestType.L3)); } - if(StringUtils.isAllBlank(ip6Dns1, ip6Dns2, zone.getIp6Dns1(), zone.getIp6Dns2())) { + if (!isL3 && StringUtils.isAllBlank(ip6Dns1, ip6Dns2, zone.getIp6Dns1(), zone.getIp6Dns2())) { throw new InvalidParameterValueException("Can only create IPv6 network if the zone has IPv6 DNS! Please configure the zone IPv6 DNS1 and/or IPv6 DNS2."); } @@ -1699,7 +1871,7 @@ public Network createGuestNetwork(CreateNetworkCmd cmd) throws InsufficientCapac if (!_accountMgr.isRootAdmin(caller.getId())) { throw new InvalidParameterValueException("Only ROOT admin is allowed to create Private VLAN network"); } - if (zone.getNetworkType() != NetworkType.Advanced || ntwkOff.getGuestType() == GuestType.Isolated) { + if (zone.getNetworkType() != NetworkType.Advanced || ntwkOff.getGuestType() == GuestType.Isolated || ntwkOff.getGuestType() == GuestType.L3) { throw new InvalidParameterValueException("Can only support create Private VLAN network with advanced shared or L2 network!"); } if (ipv6) { @@ -1722,9 +1894,12 @@ public Network createGuestNetwork(CreateNetworkCmd cmd) throws InsufficientCapac } // Ignore vlanId if it is passed but specifyvlan=false in network offering - if (ntwkOff.getGuestType() == GuestType.Shared && ! ntwkOff.isSpecifyVlan() && vlanId != null) { + if ((ntwkOff.getGuestType() == GuestType.Shared || ntwkOff.getGuestType() == GuestType.L3) && ! ntwkOff.isSpecifyVlan() && vlanId != null) { throw new InvalidParameterValueException("Cannot specify vlanId when create a network from network offering with specifyvlan=false"); } + if (isL3 && vlanId != null) { + vlanId = canonicalizeRoutedId(vlanId); + } // Don't allow to specify vlan if the caller is not ROOT admin if (!_accountMgr.isRootAdmin(caller.getId()) && (ntwkOff.isSpecifyVlan() || vlanId != null || bypassVlanOverlapCheck)) { @@ -1778,9 +1953,8 @@ public Network createGuestNetwork(CreateNetworkCmd cmd) throws InsufficientCapac } } - // Vlan is created in 1 cases - works in Advance zone only: - // 1) GuestType is Shared boolean createVlan = (startIP != null && endIP != null && zone.getNetworkType() == NetworkType.Advanced && ((ntwkOff.getGuestType() == Network.GuestType.Shared) + || (ntwkOff.getGuestType() == Network.GuestType.L3) || (ntwkOff.getGuestType() == GuestType.Isolated && !areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat)))); if (!createVlan) { @@ -1797,9 +1971,9 @@ public Network createGuestNetwork(CreateNetworkCmd cmd) throws InsufficientCapac - if (GuestType.Shared == ntwkOff.getGuestType()) { + if (GuestType.Shared == ntwkOff.getGuestType() || GuestType.L3 == ntwkOff.getGuestType()) { if (!ntwkOff.isSpecifyIpRanges()) { - throw new CloudRuntimeException("The 'specifyipranges' parameter should be true for Shared Networks"); + throw new CloudRuntimeException(String.format("The 'specifyipranges' parameter should be true for %s Networks", ntwkOff.getGuestType())); } if (ipv4 && Objects.isNull(startIP)) { throw new CloudRuntimeException("IPv4 address range needs to be provided"); @@ -2018,7 +2192,7 @@ private static ACLType getAclType(String aclTypeStr, NetworkOffering ntwkOff) { } private ACLType getAclType(Account caller, NetworkOffering ntwkOff, ACLType aclType) { - if (ntwkOff.getGuestType() == GuestType.Isolated || ntwkOff.getGuestType() == GuestType.L2) { + if (ntwkOff.getGuestType() == GuestType.Isolated || ntwkOff.getGuestType() == GuestType.L2 || ntwkOff.getGuestType() == GuestType.L3) { aclType = ACLType.Account; } else if (ntwkOff.getGuestType() == GuestType.Shared) { if (_accountMgr.isRootAdmin(caller.getId())) { @@ -2326,15 +2500,15 @@ public Network doInTransaction(TransactionStatus status) throws InsufficientCapa } String vlanId = vlanIdFinal; - if (createVlan && vlanId == null && ntwkOff.getGuestType() == Network.GuestType.Shared && ! ntwkOff.isSpecifyVlan()) { + if (createVlan && vlanId == null && (ntwkOff.getGuestType() == Network.GuestType.Shared || ntwkOff.getGuestType() == Network.GuestType.L3) + && ! ntwkOff.isSpecifyVlan()) { if (associatedNetwork != null) { // Get vlanId from associated network vlanId = associatedNetwork.getBroadcastUri().toString(); } else { - // Allocate a vnet to shared network with specifyvlan=false vlanId = _dcDao.allocateVnet(zoneId, physicalNetworkId, owner.getAccountId(), null, GuestNetworkGuru.UseSystemGuestVlans.valueIn(owner.getAccountId())); if (vlanId == null) { - throw new InvalidParameterValueException("Cannot allocate a vnet for this Shared network"); + throw new InvalidParameterValueException(String.format("Cannot allocate a vnet for this %s network", ntwkOff.getGuestType())); } } } diff --git a/server/src/main/java/com/cloud/network/guru/DirectRoutedNetworkGuru.java b/server/src/main/java/com/cloud/network/guru/DirectRoutedNetworkGuru.java new file mode 100644 index 000000000000..d6bf8f8da209 --- /dev/null +++ b/server/src/main/java/com/cloud/network/guru/DirectRoutedNetworkGuru.java @@ -0,0 +1,192 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.guru; + +import org.apache.commons.lang3.StringUtils; + +import com.cloud.dc.DataCenter; +import com.cloud.dc.DataCenter.NetworkType; +import com.cloud.deploy.DeployDestination; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientAddressCapacityException; +import com.cloud.exception.InsufficientVirtualNetworkCapacityException; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.network.Network; +import com.cloud.network.Network.GuestType; +import com.cloud.network.Network.State; +import com.cloud.network.Networks.BroadcastDomainType; +import com.cloud.network.Networks.Mode; +import com.cloud.network.PhysicalNetwork; +import com.cloud.network.PhysicalNetwork.IsolationMethod; +import com.cloud.network.dao.NetworkVO; +import com.cloud.network.dao.PhysicalNetworkVO; +import com.cloud.offering.NetworkOffering; +import com.cloud.user.Account; +import com.cloud.utils.net.NetUtils; +import com.cloud.vm.NicProfile; +import com.cloud.vm.ReservationContext; +import com.cloud.vm.VirtualMachineProfile; + +/** + * Network guru for L3 (Direct Routed) guest networks: the hypervisor routes a public IPv4 + * and/or IPv6 address directly to the Instance. There is no Virtual Router, no NAT and no + * DHCP; the Instance learns its addressing exclusively from ConfigDrive. + * + * Address allocation is inherited from {@link DirectNetworkGuru}: the operator supplies a + * subnet, CloudStack assigns individual addresses out of it. What differs is the form those + * addresses take on the NIC — an IPv4 address is a /32 and an IPv6 address a /128, with the + * shared, host-independent on-link gateway (169.254.0.1 / fe80::1) that every hypervisor + * carries on the network's bridge. + * + * These networks live on a dedicated physical network whose isolation method is ROUTED — the + * network operator's explicit, zone-level opt-in. Each network carries a broadcast domain of + * type routed://<id>; the id is a label naming the per-network, uplink-less bridge on every + * host (brdr-<id>), never an encapsulation. It is chosen by the operator at network creation + * (specifyVlan offerings, via the vlan parameter) or allocated from the physical network's vnet + * range (otherwise), and is stable for the network's life. + */ +public class DirectRoutedNetworkGuru extends DirectNetworkGuru { + + public DirectRoutedNetworkGuru() { + super(); + _isolationMethods = new IsolationMethod[] {new IsolationMethod("ROUTED")}; + } + + @Override + protected boolean canHandle(NetworkOffering offering, DataCenter dc, PhysicalNetwork physnet) { + if (dc.getNetworkType() == NetworkType.Advanced && isMyTrafficType(offering.getTrafficType()) && offering.getGuestType() == GuestType.L3 + && isMyIsolationMethod(physnet)) { + return true; + } + logger.trace("We only take care of {} guest networks in zones of type {} on physical networks with isolation method ROUTED", GuestType.L3, NetworkType.Advanced); + return false; + } + + /** + * Designs the network with {@code Mode.Static} (Instances are configured via ConfigDrive, + * there is no DHCP) and a {@code BroadcastDomainType.Routed} broadcast domain: routed://<id> + * names the per-network bridge (brdr-<id>), consuming no VLAN/VXLAN and nothing on the wire. + * The routed id — operator-specified or allocated from the physical network's vnet range — + * arrives as the broadcast URI, stamped by the orchestrator before design, and never changes: + * the bridge name derives from it. The IPv4/IPv6 subnets are allocation pools routed to the + * hypervisors, not broadcast domains, and carry no gateways of their own: the Instance's + * gateway is always the shared link-local address, so a subnet gateway would only burn an + * address. + */ + @Override + public Network design(NetworkOffering offering, DeploymentPlan plan, Network userSpecified, String name, Long vpcId, Account owner) { + DataCenter dc = _dcDao.findById(plan.getDataCenterId()); + PhysicalNetworkVO physnet = _physicalNetworkDao.findById(plan.getPhysicalNetworkId()); + + if (!canHandle(offering, dc, physnet)) { + return null; + } + + if (vpcId != null) { + throw new InvalidParameterValueException(String.format("%s networks are never part of a VPC", GuestType.L3)); + } + + NetworkVO config = new NetworkVO(offering.getTrafficType(), Mode.Static, BroadcastDomainType.Routed, offering.getId(), State.Allocated, + plan.getDataCenterId(), plan.getPhysicalNetworkId(), false); + + if (userSpecified != null) { + if (userSpecified.getBroadcastUri() != null) { + if (BroadcastDomainType.getSchemeValue(userSpecified.getBroadcastUri()) != BroadcastDomainType.Routed) { + throw new InvalidParameterValueException(String.format("A %s network requires a routed:// broadcast domain, got %s", + GuestType.L3, userSpecified.getBroadcastUri())); + } + config.setBroadcastUri(userSpecified.getBroadcastUri()); + config.setState(State.Setup); + } + + if (userSpecified.getCidr() != null) { + config.setCidr(userSpecified.getCidr()); + config.setGateway(userSpecified.getGateway()); + } + + if (userSpecified.getIp6Cidr() != null) { + config.setIp6Cidr(userSpecified.getIp6Cidr()); + config.setIp6Gateway(userSpecified.getIp6Gateway()); + } + + if (userSpecified.getPublicMtu() != null) { + config.setPublicMtu(userSpecified.getPublicMtu()); + } + if (userSpecified.getPrivateMtu() != null) { + config.setPrivateMtu(userSpecified.getPrivateMtu()); + } + + if (StringUtils.isNotBlank(userSpecified.getDns1())) { + config.setDns1(userSpecified.getDns1()); + } + if (StringUtils.isNotBlank(userSpecified.getDns2())) { + config.setDns2(userSpecified.getDns2()); + } + if (StringUtils.isNotBlank(userSpecified.getIp6Dns1())) { + config.setIp6Dns1(userSpecified.getIp6Dns1()); + } + if (StringUtils.isNotBlank(userSpecified.getIp6Dns2())) { + config.setIp6Dns2(userSpecified.getIp6Dns2()); + } + } + + return config; + } + + @Override + public NicProfile allocate(Network network, NicProfile nic, VirtualMachineProfile vm) throws InsufficientVirtualNetworkCapacityException, + InsufficientAddressCapacityException, ConcurrentOperationException { + NicProfile profile = super.allocate(network, nic, vm); + applyDirectRoutedAddressing(profile); + return profile; + } + + @Override + public void reserve(NicProfile nic, Network network, VirtualMachineProfile vm, DeployDestination dest, ReservationContext context) + throws InsufficientVirtualNetworkCapacityException, InsufficientAddressCapacityException, ConcurrentOperationException { + super.reserve(nic, network, vm, dest, context); + applyDirectRoutedAddressing(nic); + } + + /** + * The inherited allocation ({@code IpAddressManagerImpl.allocateDirectIp()}) sets the NIC's + * gateway and netmask from the vlan row, as a Shared network needs. Here the address is a + * host route, not a subnet membership: force the /32 (or /128) form and the shared on-link + * gateway over whatever the vlan row provided. This is also the signature by which the KVM + * agent recognises a direct routed NIC. The inherited allocation likewise labels the NIC's + * isolation URI vlan://<tag> from the vlan row; there is no VLAN here — the only + * isolation-shaped fact about this NIC is its routed://<id> broadcast domain, so that + * replaces the isolation URI as well. + */ + protected void applyDirectRoutedAddressing(NicProfile nic) { + if (nic == null) { + return; + } + if (nic.getBroadCastUri() != null && BroadcastDomainType.getSchemeValue(nic.getBroadCastUri()) == BroadcastDomainType.Routed) { + nic.setIsolationUri(nic.getBroadCastUri()); + } + if (nic.getIPv4Address() != null) { + nic.setIPv4Netmask(NetUtils.IPV4_HOST_NETMASK); + nic.setIPv4Gateway(NetUtils.getLinkLocalGateway()); + } + if (nic.getIPv6Address() != null) { + nic.setIPv6Cidr(nic.getIPv6Address() + "/" + NetUtils.IPV6_HOST_PREFIX_LENGTH); + nic.setIPv6Gateway(NetUtils.getIpv6LinkLocalGateway()); + } + } +} diff --git a/server/src/main/java/com/cloud/network/guru/PublicNetworkGuru.java b/server/src/main/java/com/cloud/network/guru/PublicNetworkGuru.java index b7e4f334622c..9d7caef43484 100644 --- a/server/src/main/java/com/cloud/network/guru/PublicNetworkGuru.java +++ b/server/src/main/java/com/cloud/network/guru/PublicNetworkGuru.java @@ -18,6 +18,8 @@ import javax.inject.Inject; +import org.apache.commons.lang3.StringUtils; + import com.cloud.dc.dao.VlanDetailsDao; import com.cloud.network.vpc.dao.VpcDao; import com.cloud.network.vpc.dao.VpcOfferingDao; @@ -57,11 +59,13 @@ import com.cloud.utils.db.TransactionCallbackNoReturn; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.NetUtils; import com.cloud.vm.Nic.ReservationStrategy; import com.cloud.vm.NicProfile; import com.cloud.vm.ReservationContext; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; +import com.googlecode.ipv6.IPv6Address; public class PublicNetworkGuru extends AdapterBase implements NetworkGuru { @@ -134,6 +138,14 @@ protected PublicNetworkGuru() { super(); } + /** + * On a direct routed public range (SystemVMs on a ROUTED physical network) the address is a + * host route, not a subnet membership, and the NIC takes the same form as a guest NIC on a + * direct routed network: /32 with the shared on-link gateway, and the routed://<id> + * broadcast domain that names the bridge on the host. The MAC and format are set before the + * branch on the range type, because setRoutedRangeIpv6() derives the EUI-64 address from the + * MAC and upgrades the format to DualStack. + */ protected void getIp(NicProfile nic, DataCenter dc, VirtualMachineProfile vm, Network network) throws InsufficientVirtualNetworkCapacityException, InsufficientAddressCapacityException, ConcurrentOperationException { if (nic.getIPv4Address() == null) { @@ -145,7 +157,17 @@ protected void getIp(NicProfile nic, DataCenter dc, VirtualMachineProfile vm, Ne nic.setIPv4Address(ip.getAddress().toString()); nic.setIPv4Gateway(ip.getGateway()); nic.setIPv4Netmask(ip.getNetmask()); - if (network.getBroadcastDomainType() == BroadcastDomainType.Vxlan) { + nic.setFormat(AddressFormat.Ip4); + nic.setReservationId(String.valueOf(ip.getVlanTag())); + nic.setMacAddress(ip.getMacAddress()); + if (isRoutedRange(ip)) { + nic.setIPv4Netmask(NetUtils.IPV4_HOST_NETMASK); + nic.setIPv4Gateway(NetUtils.getLinkLocalGateway()); + nic.setIsolationUri(BroadcastDomainType.Routed.toUri(ip.getVlanTag())); + nic.setBroadcastUri(BroadcastDomainType.Routed.toUri(ip.getVlanTag())); + nic.setBroadcastType(BroadcastDomainType.Routed); + setRoutedRangeIpv6(nic, dc, ip, network); + } else if (network.getBroadcastDomainType() == BroadcastDomainType.Vxlan) { nic.setIsolationUri(BroadcastDomainType.Vxlan.toUri(ip.getVlanTag())); nic.setBroadcastUri(BroadcastDomainType.Vxlan.toUri(ip.getVlanTag())); nic.setBroadcastType(BroadcastDomainType.Vxlan); @@ -154,9 +176,6 @@ protected void getIp(NicProfile nic, DataCenter dc, VirtualMachineProfile vm, Ne nic.setBroadcastUri(BroadcastDomainType.Vlan.toUri(ip.getVlanTag())); nic.setBroadcastType(BroadcastDomainType.Vlan); } - nic.setFormat(AddressFormat.Ip4); - nic.setReservationId(String.valueOf(ip.getVlanTag())); - nic.setMacAddress(ip.getMacAddress()); } Pair dns = networkModel.getNetworkIp4Dns(network, dc); @@ -166,6 +185,41 @@ protected void getIp(NicProfile nic, DataCenter dc, VirtualMachineProfile vm, Ne ipv6Service.updateNicIpv6(nic, dc, network); } + /** + * A public IP range on a ROUTED physical network is created with routed:// as its "vlan"; + * the tag flowing through the vlan row is what marks the addresses it holds as direct routed. + */ + private boolean isRoutedRange(PublicIp ip) { + String vlanTag = ip.getVlanTag(); + return vlanTag != null && vlanTag.startsWith(BroadcastDomainType.Routed.scheme() + "://"); + } + + /** + * IPv6 on a direct routed public range is computed with EUI-64 from the range's subnet and + * the NIC's MAC — the same stateless model guest NICs use (no pool, no reservation, only the + * result on the NIC) — and takes host-route form: a /128 with the shared link-local gateway. + * + * Deliberately not {@code ipv6Service.updateNicIpv6()}: that path is gated on the public + * network offering's internet protocol, and its per-network placeholder reservation would + * hand every SystemVM on the shared Public network the same address. Neither the gate nor + * the reservation has anything to decide here — the MAC makes the address unique. + */ + private void setRoutedRangeIpv6(NicProfile nic, DataCenter dc, PublicIp ip, Network network) { + String ip6Cidr = ip.vlan() != null ? ip.vlan().getIp6Cidr() : null; + if (StringUtils.isBlank(ip6Cidr) || nic.getIPv6Address() != null) { + return; + } + IPv6Address ipv6Address = NetUtils.EUI64Address(ip6Cidr, nic.getMacAddress()); + logger.info("Calculated IPv6 address {} using EUI-64 for direct routed public NIC {}", ipv6Address, nic); + nic.setIPv6Address(ipv6Address.toString()); + nic.setIPv6Cidr(ipv6Address + "/" + NetUtils.IPV6_HOST_PREFIX_LENGTH); + nic.setIPv6Gateway(NetUtils.getIpv6LinkLocalGateway()); + nic.setFormat(AddressFormat.DualStack); + Pair ip6Dns = networkModel.getNetworkIp6Dns(network, dc); + nic.setIPv6Dns1(ip6Dns.first()); + nic.setIPv6Dns2(ip6Dns.second()); + } + @Override public void updateNicProfile(NicProfile profile, Network network) { DataCenter dc = _dcDao.findById(network.getDataCenterId()); diff --git a/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java b/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java index 585b65aa4d94..fd8651ae90d8 100644 --- a/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java +++ b/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java @@ -1451,18 +1451,26 @@ public boolean securityGroupRulesForVmSecIp(long nicId, String secondaryIp, bool // Verify permissions _accountMgr.checkAccess(caller, null, false, vm); + Network network = _networkModel.getNetwork(nic.getNetworkId()); + if (network == null) { + throw new InvalidParameterValueException(String.format("Unable to find the network of NIC %s", nic)); + } + + boolean directRouted = Network.GuestType.L3.equals(network.getGuestType()); + // Validate parameters List vmSgGrps = getSecurityGroupsForVm(vmId); + boolean applySecurityGroupRules = true; if (vmSgGrps.isEmpty()) { logger.debug("Vm is not in any Security group "); - return true; - } - - //If network does not support SG service, no need add SG rules for secondary ip - Network network = _networkModel.getNetwork(nic.getNetworkId()); - if (!_networkModel.isSecurityGroupSupportedInNetwork(network)) { + applySecurityGroupRules = false; + } else if (!_networkModel.isSecurityGroupSupportedInNetwork(network)) { logger.debug("Network " + network + " is not enabled with security group service, "+ "so not applying SG rules for secondary ip"); + applySecurityGroupRules = false; + } + + if (!applySecurityGroupRules && !directRouted) { return true; } @@ -1473,7 +1481,8 @@ public boolean securityGroupRulesForVmSecIp(long nicId, String secondaryIp, bool } //create command for the to add ip in ipset and arptables rules - NetworkRulesVmSecondaryIpCommand cmd = new NetworkRulesVmSecondaryIpCommand(vmName, vmMac, secondaryIp, ruleAction); + NetworkRulesVmSecondaryIpCommand cmd = new NetworkRulesVmSecondaryIpCommand(vmName, vmMac, secondaryIp, ruleAction, + directRouted, applySecurityGroupRules); logger.debug("Asking agent to configure rules for vm secondary ip"); Commands cmds = null; diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 60fdee8feaa7..535c2c91fb4f 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -2003,7 +2003,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { return null; } } else { - throw new InvalidParameterValueException("UpdateVmNicIpCmd is not supported in L2 network"); + throw new InvalidParameterValueException(String.format("UpdateVmNicIpCmd is not supported in %s networks", network.getGuestType())); } logger.debug("Updating IPv4 address of NIC " + nicVO + " to " + ipaddr + "/" + nicVO.getIPv4Netmask() + " with gateway " + nicVO.getIPv4Gateway()); diff --git a/server/src/main/resources/META-INF/cloudstack/server-network/spring-server-network-context.xml b/server/src/main/resources/META-INF/cloudstack/server-network/spring-server-network-context.xml index 97ed36e515c7..f3d518cd185c 100644 --- a/server/src/main/resources/META-INF/cloudstack/server-network/spring-server-network-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/server-network/spring-server-network-context.xml @@ -48,6 +48,9 @@ + + + diff --git a/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java b/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java index 9a0b150780e4..0cc6b063cbec 100644 --- a/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java +++ b/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java @@ -35,6 +35,10 @@ import com.cloud.network.Networks; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.NetrisProviderDao; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.network.dao.PhysicalNetworkVO; +import com.cloud.dc.DataCenterVnetVO; import com.cloud.network.dao.NsxProviderDao; import com.cloud.network.dao.PhysicalNetworkDao; import com.cloud.network.element.NsxProviderVO; @@ -165,6 +169,8 @@ public class ConfigurationManagerImplTest { @Mock PhysicalNetworkDao physicalNetworkDao; @Mock + NetworkDao networkDao; + @Mock ImageStoreDao imageStoreDao; @Mock VlanDao vlanDao; @@ -1410,4 +1416,120 @@ public void testGetExternalNetworkProviderReturnsNullWhenNoExternalProviders() { mapWithEmptySet.put(Network.Service.Firewall, Collections.emptySet()); Assert.assertNull(ConfigurationManagerImpl.getExternalNetworkProvider(null, mapWithEmptySet)); } + + private Map> validL3ServiceProviderMap() { + Map> map = new HashMap<>(); + map.put(Network.Service.UserData, Collections.singleton(Network.Provider.ConfigDrive)); + map.put(Network.Service.Dns, Collections.singleton(Network.Provider.ConfigDrive)); + map.put(Network.Service.SecurityGroup, Collections.singleton(Network.Provider.SecurityGroupProvider)); + return map; + } + + @Test + public void validateL3NetworkOfferingAcceptsUserDataDnsAndSecurityGroup() { + configurationManagerImplSpy.validateL3NetworkOffering(validL3ServiceProviderMap(), null, false, true, false); + } + + @Test + public void validateL3NetworkOfferingAcceptsOfferingWithoutDns() { + Map> map = validL3ServiceProviderMap(); + map.remove(Network.Service.Dns); + configurationManagerImplSpy.validateL3NetworkOffering(map, null, false, true, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3NetworkOfferingRejectsDhcp() { + Map> map = validL3ServiceProviderMap(); + map.put(Network.Service.Dhcp, Collections.singleton(Network.Provider.ConfigDrive)); + configurationManagerImplSpy.validateL3NetworkOffering(map, null, false, true, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3NetworkOfferingRejectsMissingUserData() { + Map> map = validL3ServiceProviderMap(); + map.remove(Network.Service.UserData); + configurationManagerImplSpy.validateL3NetworkOffering(map, null, false, true, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3NetworkOfferingRejectsUserDataWithoutProvider() { + Map> map = validL3ServiceProviderMap(); + map.put(Network.Service.UserData, Collections.emptySet()); + configurationManagerImplSpy.validateL3NetworkOffering(map, null, false, true, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3NetworkOfferingRejectsNonConfigDriveDns() { + Map> map = validL3ServiceProviderMap(); + map.put(Network.Service.Dns, Collections.singleton(Network.Provider.VirtualRouter)); + configurationManagerImplSpy.validateL3NetworkOffering(map, null, false, true, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3NetworkOfferingRejectsUnsupportedService() { + Map> map = validL3ServiceProviderMap(); + map.put(Network.Service.SourceNat, Collections.singleton(Network.Provider.VirtualRouter)); + configurationManagerImplSpy.validateL3NetworkOffering(map, null, false, true, false); + } + + private PhysicalNetworkVO routedPhysicalNetwork(long id) { + PhysicalNetworkVO physicalNetwork = Mockito.mock(PhysicalNetworkVO.class); + Mockito.when(physicalNetwork.getId()).thenReturn(id); + Mockito.when(physicalNetwork.getName()).thenReturn("routed-physnet"); + Mockito.when(physicalNetwork.getIsolationMethods()).thenReturn(List.of("ROUTED")); + return physicalNetwork; + } + + @Test + public void canonicalizeRoutedRangeIdAcceptsFreeId() { + PhysicalNetworkVO routedPhysicalNetwork = routedPhysicalNetwork(7L); + Mockito.when(networkDao.listByZoneAndUriAndGuestType(1L, "routed://5828", null)).thenReturn(Collections.emptyList()); + Mockito.when(physicalNetworkDao.listByZone(1L)).thenReturn(List.of(routedPhysicalNetwork)); + Mockito.when(zoneDao.findVnet(1L, 7L, "5828")).thenReturn(Collections.emptyList()); + + Assert.assertEquals("routed://5828", configurationManagerImplSpy.canonicalizeRoutedRangeId(1L, "routed://5828")); + } + + @Test(expected = InvalidParameterValueException.class) + public void canonicalizeRoutedRangeIdRejectsMalformedId() { + configurationManagerImplSpy.canonicalizeRoutedRangeId(1L, "routed://abc"); + } + + @Test(expected = InvalidParameterValueException.class) + public void canonicalizeRoutedRangeIdRejectsIdOfGuestNetwork() { + NetworkVO guestNetwork = Mockito.mock(NetworkVO.class); + Mockito.when(networkDao.listByZoneAndUriAndGuestType(1L, "routed://5828", null)).thenReturn(List.of(guestNetwork)); + + configurationManagerImplSpy.canonicalizeRoutedRangeId(1L, "routed://5828"); + } + + @Test(expected = InvalidParameterValueException.class) + public void canonicalizeRoutedRangeIdRejectsIdInsideRoutedPhysicalNetworkRange() { + PhysicalNetworkVO routedPhysicalNetwork = routedPhysicalNetwork(7L); + Mockito.when(networkDao.listByZoneAndUriAndGuestType(1L, "routed://5828", null)).thenReturn(Collections.emptyList()); + Mockito.when(physicalNetworkDao.listByZone(1L)).thenReturn(List.of(routedPhysicalNetwork)); + Mockito.when(zoneDao.findVnet(1L, 7L, "5828")).thenReturn(List.of(new DataCenterVnetVO("5828", 1L, 7L))); + + configurationManagerImplSpy.canonicalizeRoutedRangeId(1L, "routed://5828"); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3NetworkOfferingRejectsNetworkMode() { + configurationManagerImplSpy.validateL3NetworkOffering(validL3ServiceProviderMap(), NetworkOffering.NetworkMode.ROUTED, false, true, false); + } + + @Test + public void validateL3NetworkOfferingAcceptsSpecifyVlan() { + configurationManagerImplSpy.validateL3NetworkOffering(validL3ServiceProviderMap(), null, true, true, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3NetworkOfferingRejectsWithoutSpecifyIpRanges() { + configurationManagerImplSpy.validateL3NetworkOffering(validL3ServiceProviderMap(), null, false, false, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3NetworkOfferingRejectsVpc() { + configurationManagerImplSpy.validateL3NetworkOffering(validL3ServiceProviderMap(), null, false, true, true); + } } diff --git a/server/src/test/java/com/cloud/network/NetworkModelImplTest.java b/server/src/test/java/com/cloud/network/NetworkModelImplTest.java index 7810662c617b..4d7a3ac32674 100644 --- a/server/src/test/java/com/cloud/network/NetworkModelImplTest.java +++ b/server/src/test/java/com/cloud/network/NetworkModelImplTest.java @@ -86,6 +86,8 @@ public class NetworkModelImplTest { @Mock private VpcDao vpcDao; @Mock + private com.cloud.network.dao.IPAddressDao _ipAddressDao; + @Mock private NetworkDao _networksDao; @Inject private NetworkOfferingServiceMapDao networkOfferingServiceMapDao; @@ -452,4 +454,64 @@ public void listSupportedNetworkServiceProvidersExcludesExtensionBackedProviders Mockito.verify(physicalNetworkServiceProviderDao, Mockito.times(1)).listAll(); Mockito.verify(physicalNetworkServiceProviderDao, Mockito.never()).listBy(Mockito.anyLong()); } + + private boolean checkSecurityGroupSupportFor(Network.GuestType guestType, boolean sgSupportedInNetwork) { + DataCenter zone = mock(DataCenter.class); + when(zone.isSecurityGroupEnabled()).thenReturn(false); + NetworkVO network = mock(NetworkVO.class); + when(network.getGuestType()).thenReturn(guestType); + when(_networksDao.findById(42L)).thenReturn(network); + doReturn(sgSupportedInNetwork).when(networkModel).isSecurityGroupSupportedInNetwork(network); + return networkModel.checkSecurityGroupSupportForNetwork(mock(com.cloud.user.Account.class), zone, List.of(42L), null); + } + + private NetworkVO l3NetworkForDeploy(String cidr) { + NetworkVO network = mock(NetworkVO.class); + when(network.getTrafficType()).thenReturn(com.cloud.network.Networks.TrafficType.Guest); + when(network.getGuestType()).thenReturn(Network.GuestType.L3); + Mockito.lenient().when(network.getGateway()).thenReturn(null); + Mockito.lenient().when(network.getCidr()).thenReturn(cidr); + doReturn(List.of(Network.Service.UserData)).when(networkModel).listNetworkOfferingServices(Mockito.anyLong()); + return network; + } + + /** + * Regression: an IPv6-only L3 network has no IPv4 cidr, which the Isolated-style branch of + * canUseForDeploy() treated as unusable - hiding the network from the deploy wizard. IPv6 + * needs no free-IP check at all: addresses are EUI-64 computed and never run out. + */ + @Test + public void canUseForDeployAcceptsIpv6OnlyL3Network() { + assertTrue(networkModel.canUseForDeploy(l3NetworkForDeploy(null))); + } + + @Test + public void canUseForDeployChecksIpv4PoolOfGatewaylessL3Network() { + NetworkVO network = l3NetworkForDeploy("203.0.113.0/24"); + when(_ipAddressDao.countFreeIPsInNetwork(Mockito.anyLong())).thenReturn(0L); + assertFalse(networkModel.canUseForDeploy(network)); + when(_ipAddressDao.countFreeIPsInNetwork(Mockito.anyLong())).thenReturn(5L); + assertTrue(networkModel.canUseForDeploy(network)); + } + + @Test + public void checkSecurityGroupSupportForNetworkAcceptsSharedNetworkWithSecurityGroupService() { + assertTrue(checkSecurityGroupSupportFor(Network.GuestType.Shared, true)); + } + + /** + * Regression: deploying with securitygroupids into an L3 (Direct Routed) network failed with + * "security group feature is not enabled per zone" because the guest-type check accepted + * only Shared, while L3 offerings carry the SecurityGroup service per network. + */ + @Test + public void checkSecurityGroupSupportForNetworkAcceptsL3NetworkWithSecurityGroupService() { + assertTrue(checkSecurityGroupSupportFor(Network.GuestType.L3, true)); + } + + @Test + public void checkSecurityGroupSupportForNetworkRejectsNetworkWithoutSecurityGroupService() { + assertFalse(checkSecurityGroupSupportFor(Network.GuestType.L3, false)); + assertFalse(checkSecurityGroupSupportFor(Network.GuestType.Isolated, true)); + } } diff --git a/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java b/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java index cd7d40d68951..6ba6a1a803e0 100644 --- a/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java +++ b/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java @@ -1378,4 +1378,133 @@ public void getAndValidateSupportForKeepMacAddressOnPublicNicParameterTestReturn Assert.assertFalse(service.getAndValidateSupportForKeepMacAddressOnPublicNicParameter(false, networkOfferingVO)); } + + @Test + public void validateL3AddressFamiliesAcceptsDualStackWithoutGateways() { + service.validateL3AddressFamilies("255.255.255.0", "10.1.1.10", "10.1.1.20", "fd00::/64", null, null); + } + + @Test + public void validateL3AddressFamiliesAcceptsIpv4Only() { + service.validateL3AddressFamilies("255.255.255.0", "10.1.1.10", null, null, null, null); + } + + @Test + public void validateL3AddressFamiliesAcceptsIpv6CidrAlone() { + service.validateL3AddressFamilies(null, null, null, "fd00::/64", null, null); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3AddressFamiliesRejectsNetmaskWithoutStartIp() { + service.validateL3AddressFamilies("255.255.255.0", null, null, "fd00::/64", null, null); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3AddressFamiliesRejectsStartIpWithoutNetmask() { + service.validateL3AddressFamilies(null, "10.1.1.10", null, "fd00::/64", null, null); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3AddressFamiliesRejectsIpv6RangeWithoutCidr() { + service.validateL3AddressFamilies(null, null, null, null, "fd00::100", "fd00::200"); + } + + @Test + public void expandL3Ipv4CidrDerivesRangeFromSubnet() { + String[] expanded = service.expandL3Ipv4Cidr("192.0.2.0/24", null, null, null); + Assert.assertArrayEquals(new String[] {"255.255.255.0", "192.0.2.1", "192.0.2.254"}, expanded); + } + + @Test + public void expandL3Ipv4CidrKeepsExplicitRangeInsideSubnet() { + String[] expanded = service.expandL3Ipv4Cidr("192.0.2.0/24", null, "192.0.2.0", "192.0.2.255"); + Assert.assertArrayEquals(new String[] {"255.255.255.0", "192.0.2.0", "192.0.2.255"}, expanded); + } + + @Test(expected = InvalidParameterValueException.class) + public void expandL3Ipv4CidrRejectsStartIpOutsideSubnet() { + service.expandL3Ipv4Cidr("192.0.2.0/24", null, "198.51.100.7", null); + } + + @Test(expected = InvalidParameterValueException.class) + public void expandL3Ipv4CidrRejectsEndIpOutsideSubnet() { + service.expandL3Ipv4Cidr("192.0.2.0/24", null, "192.0.2.10", "192.0.3.10"); + } + + @Test(expected = InvalidParameterValueException.class) + public void expandL3Ipv4CidrRejectsEndIpWithoutStartIp() { + service.expandL3Ipv4Cidr("192.0.2.0/24", null, null, "192.0.2.10"); + } + + @Test(expected = InvalidParameterValueException.class) + public void expandL3Ipv4CidrRejectsCidrTogetherWithNetmask() { + service.expandL3Ipv4Cidr("192.0.2.0/24", "255.255.255.0", null, null); + } + + @Test + public void canonicalizeRoutedIdStripsScheme() { + Assert.assertEquals("5828", service.canonicalizeRoutedId("routed://5828")); + Assert.assertEquals("5828", service.canonicalizeRoutedId("5828")); + } + + @Test(expected = InvalidParameterValueException.class) + public void canonicalizeRoutedIdRejectsNonNumericId() { + service.canonicalizeRoutedId("routed://abc"); + } + + @Test + public void expandL3Ipv4CidrDerivesNetmaskAndUsableRange() { + String[] expanded = service.expandL3Ipv4Cidr("2.57.59.0/24", null, null, null); + Assert.assertEquals("255.255.255.0", expanded[0]); + Assert.assertEquals("2.57.59.1", expanded[1]); + Assert.assertEquals("2.57.59.254", expanded[2]); + } + + @Test + public void expandL3Ipv4CidrKeepsExplicitRange() { + String[] expanded = service.expandL3Ipv4Cidr("2.57.59.0/24", null, "2.57.59.10", "2.57.59.20"); + Assert.assertEquals("255.255.255.0", expanded[0]); + Assert.assertEquals("2.57.59.10", expanded[1]); + Assert.assertEquals("2.57.59.20", expanded[2]); + } + + @Test(expected = InvalidParameterValueException.class) + public void expandL3Ipv4CidrRejectsNetmaskAlongside() { + service.expandL3Ipv4Cidr("2.57.59.0/24", "255.255.255.0", null, null); + } + + @Test(expected = InvalidParameterValueException.class) + public void expandL3Ipv4CidrRejectsInvalidCidr() { + service.expandL3Ipv4Cidr("2.57.59.0/33", null, null, null); + } + + @Test(expected = InvalidParameterValueException.class) + public void expandL3Ipv4CidrRejectsTinySubnetWithoutExplicitRange() { + service.expandL3Ipv4Cidr("2.57.59.0/31", null, null, null); + } + + @Test + public void checkL3Ip6ParametersAcceptsCidrAlone() { + service.checkL3Ip6Parameters(null, null, "fd00::/64"); + } + + @Test + public void checkL3Ip6ParametersAcceptsRangeInsideCidr() { + service.checkL3Ip6Parameters("fd00::100", "fd00::200", "fd00::/64"); + } + + @Test(expected = InvalidParameterValueException.class) + public void checkL3Ip6ParametersRejectsRangeOutsideCidr() { + service.checkL3Ip6Parameters("fd00:1::100", null, "fd00::/64"); + } + + @Test(expected = InvalidParameterValueException.class) + public void checkL3Ip6ParametersRejectsInvalidCidr() { + service.checkL3Ip6Parameters(null, null, "not-a-cidr"); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateL3AddressFamiliesRejectsNoFamilyAtAll() { + service.validateL3AddressFamilies(null, null, null, null, null, null); + } } diff --git a/server/src/test/java/com/cloud/network/guru/DirectRoutedNetworkGuruTest.java b/server/src/test/java/com/cloud/network/guru/DirectRoutedNetworkGuruTest.java new file mode 100644 index 000000000000..07ab289110c2 --- /dev/null +++ b/server/src/test/java/com/cloud/network/guru/DirectRoutedNetworkGuruTest.java @@ -0,0 +1,197 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.guru; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.dc.DataCenter.NetworkType; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.network.Network; +import com.cloud.network.Network.GuestType; +import com.cloud.network.Networks.BroadcastDomainType; +import com.cloud.network.Networks.Mode; +import com.cloud.network.Networks.TrafficType; +import com.cloud.network.dao.NetworkVO; +import com.cloud.network.dao.PhysicalNetworkDao; +import com.cloud.network.dao.PhysicalNetworkVO; +import com.cloud.offering.NetworkOffering; +import com.cloud.user.Account; +import com.cloud.vm.NicProfile; + +@RunWith(MockitoJUnitRunner.Silent.class) +public class DirectRoutedNetworkGuruTest { + + @InjectMocks + protected DirectRoutedNetworkGuru guru = new DirectRoutedNetworkGuru(); + + @Mock + DataCenterDao dcDao; + @Mock + PhysicalNetworkDao physicalNetworkDao; + + @Mock + NetworkOffering offering; + @Mock + DataCenterVO dc; + @Mock + PhysicalNetworkVO physicalNetwork; + @Mock + DeploymentPlan plan; + @Mock + Account owner; + + @Before + public void setUp() { + lenient().when(dc.getNetworkType()).thenReturn(NetworkType.Advanced); + lenient().when(offering.getTrafficType()).thenReturn(TrafficType.Guest); + lenient().when(offering.getGuestType()).thenReturn(GuestType.L3); + lenient().when(physicalNetwork.getIsolationMethods()).thenReturn(Arrays.asList("ROUTED")); + lenient().when(plan.getDataCenterId()).thenReturn(1L); + lenient().when(plan.getPhysicalNetworkId()).thenReturn(1L); + lenient().when(dcDao.findById(1L)).thenReturn(dc); + lenient().when(physicalNetworkDao.findById(1L)).thenReturn(physicalNetwork); + } + + @Test + public void canHandleAcceptsL3OnRoutedPhysicalNetwork() { + assertTrue(guru.canHandle(offering, dc, physicalNetwork)); + } + + @Test + public void canHandleRejectsBasicZone() { + when(dc.getNetworkType()).thenReturn(NetworkType.Basic); + assertFalse(guru.canHandle(offering, dc, physicalNetwork)); + } + + @Test + public void canHandleRejectsOtherGuestTypes() { + for (GuestType type : new GuestType[] {GuestType.Shared, GuestType.Isolated, GuestType.L2}) { + when(offering.getGuestType()).thenReturn(type); + assertFalse("guru must not claim guest type " + type, guru.canHandle(offering, dc, physicalNetwork)); + } + } + + @Test + public void canHandleRejectsPhysicalNetworkWithoutRoutedIsolation() { + for (List methods : Arrays.asList(Arrays.asList("VLAN"), Arrays.asList("VXLAN"), Collections.emptyList())) { + when(physicalNetwork.getIsolationMethods()).thenReturn(methods); + assertFalse("guru must not claim a physical network with isolation methods " + methods, guru.canHandle(offering, dc, physicalNetwork)); + } + } + + @Test + public void designProducesRoutedStaticNetwork() { + Network network = guru.design(offering, plan, null, "test", null, owner); + assertNotNull(network); + NetworkVO config = (NetworkVO)network; + assertEquals(BroadcastDomainType.Routed, config.getBroadcastDomainType()); + assertEquals(Mode.Static, config.getMode()); + assertNull(config.getBroadcastUri()); + } + + @Test + public void designCarriesRoutedBroadcastUri() throws URISyntaxException { + Network userSpecified = mock(Network.class); + when(userSpecified.getBroadcastUri()).thenReturn(new URI("routed://5828")); + + Network network = guru.design(offering, plan, userSpecified, "test", null, owner); + assertNotNull(network); + NetworkVO config = (NetworkVO)network; + assertEquals(BroadcastDomainType.Routed, config.getBroadcastDomainType()); + assertEquals("routed://5828", config.getBroadcastUri().toString()); + assertEquals(Network.State.Setup, config.getState()); + } + + @Test(expected = InvalidParameterValueException.class) + public void designRejectsNonRoutedBroadcastUri() throws URISyntaxException { + Network userSpecified = mock(Network.class); + when(userSpecified.getBroadcastUri()).thenReturn(new URI("vlan://5828")); + guru.design(offering, plan, userSpecified, "test", null, owner); + } + + @Test(expected = InvalidParameterValueException.class) + public void designRejectsVpc() { + guru.design(offering, plan, null, "test", 42L, owner); + } + + @Test + public void designReturnsNullForOtherGuestTypes() { + when(offering.getGuestType()).thenReturn(GuestType.Shared); + assertNull(guru.design(offering, plan, null, "test", null, owner)); + } + + @Test + public void applyDirectRoutedAddressingForcesHostRouteForm() { + NicProfile nic = new NicProfile(); + nic.setIPv4Address("203.0.113.55"); + nic.setIPv4Netmask("255.255.255.0"); + nic.setIPv4Gateway("203.0.113.1"); + nic.setIPv6Address("2001:db8:1::55"); + nic.setIPv6Cidr("2001:db8:1::/64"); + nic.setIPv6Gateway("2001:db8:1::1"); + + guru.applyDirectRoutedAddressing(nic); + + assertEquals("255.255.255.255", nic.getIPv4Netmask()); + assertEquals("169.254.0.1", nic.getIPv4Gateway()); + assertEquals("2001:db8:1::55/128", nic.getIPv6Cidr()); + assertEquals("fe80::1", nic.getIPv6Gateway()); + } + + @Test + public void applyDirectRoutedAddressingLeavesAbsentFamiliesAlone() { + NicProfile nic = new NicProfile(); + nic.setIPv4Address("203.0.113.55"); + nic.setIPv4Netmask("255.255.255.0"); + nic.setIPv4Gateway("203.0.113.1"); + + guru.applyDirectRoutedAddressing(nic); + + assertEquals("255.255.255.255", nic.getIPv4Netmask()); + assertEquals("169.254.0.1", nic.getIPv4Gateway()); + assertNull(nic.getIPv6Cidr()); + assertNull(nic.getIPv6Gateway()); + } + + @Test + public void applyDirectRoutedAddressingIsNullSafe() { + guru.applyDirectRoutedAddressing(null); + } +} diff --git a/server/src/test/java/com/cloud/network/guru/PublicNetworkGuruTest.java b/server/src/test/java/com/cloud/network/guru/PublicNetworkGuruTest.java new file mode 100644 index 000000000000..e71169faa691 --- /dev/null +++ b/server/src/test/java/com/cloud/network/guru/PublicNetworkGuruTest.java @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.guru; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.when; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.dc.DataCenter; +import com.cloud.dc.Vlan.VlanType; +import com.cloud.dc.VlanVO; +import com.cloud.network.IpAddressManager; +import com.cloud.network.Ipv6Service; +import com.cloud.network.Network; +import com.cloud.network.NetworkModel; +import com.cloud.network.Networks.AddressFormat; +import com.cloud.network.Networks.BroadcastDomainType; +import com.cloud.network.addr.PublicIp; +import com.cloud.network.dao.IPAddressVO; +import com.cloud.user.Account; +import com.cloud.utils.Pair; +import com.cloud.utils.net.Ip; +import com.cloud.utils.net.NetUtils; +import com.cloud.vm.Nic.ReservationStrategy; +import com.cloud.vm.NicProfile; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachineProfile; + +@RunWith(MockitoJUnitRunner.Silent.class) +public class PublicNetworkGuruTest { + + @InjectMocks + protected PublicNetworkGuru guru = new PublicNetworkGuru(); + + @Mock + IpAddressManager ipAddrMgr; + @Mock + Ipv6Service ipv6Service; + @Mock + NetworkModel networkModel; + + @Mock + DataCenter dc; + @Mock + Network network; + @Mock + VirtualMachineProfile vm; + @Mock + Account owner; + + private static final long MAC = 0x1e003c000102L; + + @Before + public void setUp() { + lenient().when(vm.getType()).thenReturn(VirtualMachine.Type.ConsoleProxy); + lenient().when(vm.getOwner()).thenReturn(owner); + lenient().when(networkModel.getNetworkIp4Dns(network, dc)).thenReturn(new Pair<>(null, null)); + lenient().when(networkModel.getNetworkIp6Dns(network, dc)).thenReturn(new Pair<>(null, null)); + } + + private PublicIp publicIp(String vlanTag, String ip6Cidr) { + IPAddressVO addr = new IPAddressVO(new Ip("2.57.59.68"), 1L, 5L, 1L, false); + VlanVO vlan = new VlanVO(VlanType.VirtualNetwork, vlanTag, "2.57.59.65", "255.255.255.192", 1L, + "2.57.59.66-2.57.59.94", 200L, 200L, ip6Cidr == null ? null : "2a00:f10:402:2::1", ip6Cidr, null); + return new PublicIp(addr, vlan, MAC); + } + + private NicProfile allocateOn(PublicIp ip) throws Exception { + when(ipAddrMgr.assignPublicIpAddress(anyLong(), any(), any(), any(), any(), any(), anyBoolean(), anyBoolean())).thenReturn(ip); + NicProfile nic = new NicProfile(ReservationStrategy.Create, null, null, null, null); + guru.getIp(nic, dc, vm, network); + return nic; + } + + /** + * Regression: setRoutedRangeIpv6() runs inside getIp()'s routed branch and needs the NIC's + * MAC (EUI-64) — the MAC must be on the profile before that branch, not after it (NPE), and + * the DualStack format it sets must not be clobbered back to Ip4 afterwards. + */ + @Test + public void getIpOnRoutedRangeComputesEui64Ipv6FromNicMac() throws Exception { + PublicIp ip = publicIp("routed://534", "2a00:f10:402:2::/64"); + NicProfile nic = allocateOn(ip); + + assertEquals(ip.getMacAddress(), nic.getMacAddress()); + assertEquals("2.57.59.68", nic.getIPv4Address()); + assertEquals(NetUtils.IPV4_HOST_NETMASK, nic.getIPv4Netmask()); + assertEquals(NetUtils.getLinkLocalGateway(), nic.getIPv4Gateway()); + assertEquals("routed://534", nic.getBroadCastUri().toString()); + assertEquals(BroadcastDomainType.Routed, nic.getBroadcastType()); + assertEquals(NetUtils.EUI64Address("2a00:f10:402:2::/64", ip.getMacAddress()).toString(), nic.getIPv6Address()); + assertEquals(NetUtils.getIpv6LinkLocalGateway(), nic.getIPv6Gateway()); + assertEquals(AddressFormat.DualStack, nic.getFormat()); + } + + @Test + public void getIpOnVlanRangeKeepsClassicShape() throws Exception { + when(network.getBroadcastDomainType()).thenReturn(BroadcastDomainType.Vlan); + PublicIp ip = publicIp("50", null); + NicProfile nic = allocateOn(ip); + + assertEquals(ip.getMacAddress(), nic.getMacAddress()); + assertEquals("2.57.59.68", nic.getIPv4Address()); + assertEquals("255.255.255.192", nic.getIPv4Netmask()); + assertEquals("2.57.59.65", nic.getIPv4Gateway()); + assertEquals("vlan://50", nic.getBroadCastUri().toString()); + assertNull(nic.getIPv6Address()); + assertEquals(AddressFormat.Ip4, nic.getFormat()); + } +} diff --git a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java index 1239dd23e7ec..97710768a7dc 100644 --- a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java +++ b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java @@ -1049,6 +1049,11 @@ public boolean isSharedNetworkWithoutSpecifyVlan(NetworkOffering offering) { return false; } + @Override + public boolean isL3NetworkWithoutSpecifyVlan(NetworkOffering offering) { + return false; + } + @Override public IpAddress updateIP(Long id, String customId, Boolean displayIp) { // TODO Auto-generated method stub diff --git a/utils/src/main/java/com/cloud/utils/net/NetUtils.java b/utils/src/main/java/com/cloud/utils/net/NetUtils.java index d89d9fa2d93c..5e8b1607793e 100644 --- a/utils/src/main/java/com/cloud/utils/net/NetUtils.java +++ b/utils/src/main/java/com/cloud/utils/net/NetUtils.java @@ -92,6 +92,20 @@ public class NetUtils { public final static int PORT_RANGE_MIN = 0; public final static int PORT_RANGE_MAX = 65535; + /** + * The IPv4 link-local block and its first address, which KVM hosts carry on cloud0 for the + * control network and which Direct Routed (L3) guest networks use as the shared, + * host-independent gateway on every network's bridge. Defined here and nowhere else; use the + * getLinkLocal*() / getIpv6LinkLocalGateway() getters rather than these constants. The guru + * stamps the gateways on the NIC at allocation, and the agent passes the NIC's values down to + * modifybrdr.sh. + */ + private final static String IPV4_LINK_LOCAL_CIDR = "169.254.0.0/16"; + private final static String IPV4_LINK_LOCAL_GATEWAY = "169.254.0.1"; + private final static String IPV6_LINK_LOCAL_GATEWAY = "fe80::1"; + public final static String IPV4_HOST_NETMASK = "255.255.255.255"; + public final static int IPV6_HOST_PREFIX_LENGTH = 128; + public final static int DEFAULT_AUTOSCALE_EXPUNGE_VM_GRACE_PERIOD = 2 * 60; // Grace period before Vm is expunged public final static int DEFAULT_AUTOSCALE_POLICY_INTERVAL_TIME = 30; public final static int DEFAULT_AUTOSCALE_POLICY_QUIET_TIME = 5 * 60; @@ -1032,11 +1046,15 @@ public static String getLinkLocalGateway(String cidr) { } public static String getLinkLocalGateway() { - return getLinkLocalGateway(getLinkLocalCIDR()); + return IPV4_LINK_LOCAL_GATEWAY; + } + + public static String getIpv6LinkLocalGateway() { + return IPV6_LINK_LOCAL_GATEWAY; } public static String getLinkLocalCIDR() { - return "169.254.0.0/16"; + return IPV4_LINK_LOCAL_CIDR; } public static String getLinkLocalFirstAddressFromCIDR(final String cidr) { From aa564feec15aa9f7574d8a45f03879aadb94351b Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Tue, 15 Sep 2026 08:33:56 +0000 Subject: [PATCH 2/8] SystemVMs: boot on routed public ranges Console proxies and secondary storage VMs can take their public address from a routed public range (createVlanIpRange with vlan=routed://). The management server then hands the address to the SystemVM in host-route form, a /32 or /128 with the shared link-local gateway on-link, and the SystemVM boot scripts configure it that way for IPv4 and IPv6 alike. --- .../consoleproxy/ConsoleProxyManagerImpl.java | 8 +++++++ .../SecondaryStorageManagerImpl.java | 8 +++++++ systemvm/debian/opt/cloud/bin/setup/common.sh | 21 ++++++++++++++++++- 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index 4401dc2d1529..29aad1d92ba2 100644 --- a/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -1239,8 +1239,16 @@ public boolean finalizeVirtualMachineProfile(VirtualMachineProfile profile, Depl buf.append(" eth").append(deviceId).append("mask=").append(nic.getIPv4Netmask()); } + if (nic.getIPv6Address() != null) { + buf.append(" eth").append(deviceId).append("ip6=").append(nic.getIPv6Address()); + buf.append(" eth").append(deviceId).append("ip6prelen=").append(NetUtils.getIp6CidrSize(nic.getIPv6Cidr())); + } + if (nic.isDefaultNic()) { buf.append(" gateway=").append(nic.getIPv4Gateway()); + if (nic.getIPv6Gateway() != null) { + buf.append(" ip6gateway=").append(nic.getIPv6Gateway()); + } } if (nic.getTrafficType() == TrafficType.Management) { diff --git a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java index 9d4c73111595..084166dce083 100644 --- a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java +++ b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java @@ -1194,8 +1194,16 @@ public boolean finalizeVirtualMachineProfile(VirtualMachineProfile profile, Depl buf.append(" eth").append(deviceId).append("mask=").append(nic.getIPv4Netmask()); } + if (nic.getIPv6Address() != null) { + buf.append(" eth").append(deviceId).append("ip6=").append(nic.getIPv6Address()); + buf.append(" eth").append(deviceId).append("ip6prelen=").append(NetUtils.getIp6CidrSize(nic.getIPv6Cidr())); + } + if (nic.isDefaultNic()) { buf.append(" gateway=").append(nic.getIPv4Gateway()); + if (nic.getIPv6Gateway() != null) { + buf.append(" ip6gateway=").append(nic.getIPv6Gateway()); + } } if (nic.getTrafficType() == TrafficType.Management) { String mgmt_cidr = _configDao.getValue(Config.ManagementNetwork.key()); diff --git a/systemvm/debian/opt/cloud/bin/setup/common.sh b/systemvm/debian/opt/cloud/bin/setup/common.sh index ef1576ab588c..2cf91201f45e 100755 --- a/systemvm/debian/opt/cloud/bin/setup/common.sh +++ b/systemvm/debian/opt/cloud/bin/setup/common.sh @@ -396,7 +396,26 @@ setup_common() { gwdev="eth0" fi - ip route add default via $GW dev $gwdev + onlink="" + case "$GW" in + 169.254.*) + # Direct routed public interface: the address is a /32, so the shared + # link-local gateway lies outside it and the kernel rejects the route + # unless it is marked on-link. Same rule cloud-init applies for guest + # Instances; systemvms have no cloud-init, so it is applied here. + onlink="onlink" + ;; + esac + ip route add default via $GW dev $gwdev $onlink + + # The IPv6 default route is installed statically when a gateway was passed + # down: on a direct routed bridge no router advertisement ever arrives, so + # relying on accept_ra would leave the systemvm without v6 connectivity. + # A link-local gateway (fe80::1) needs the dev and is on-link by definition. + if [ -n "$IP6GW" ] + then + ip -6 route replace default via $IP6GW dev $gwdev + fi fi # Workaround to activate vSwitch under VMware From 004af9dda2d48beb31d7fdc0ad5f203bd7e76a03 Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Tue, 15 Sep 2026 08:33:13 +0000 Subject: [PATCH 3/8] KVM: per-network bridges and host routes for Direct Routed Instances The agent recognises a Direct Routed NIC by its routed:// broadcast domain and asks the new modifybrdr.sh for the network's bridge. The script creates one uplink-less bridge per network, named brdr-, with the gateway addresses passed in by the agent, forwarding sysctls, strict rp_filter, arp_ignore/arp_announce for the shared gateway address, and a MAC derived from the routed id so it is identical on every hypervisor and live migration never invalidates the guest's neighbour cache. The existing modifymacip.sh installs the per-address host route and static neighbour entry when a NIC is plugged, and removes them at unplug before the script may delete an empty bridge. How these bridges are named is known only to the script; the agent asks it (query) rather than parsing names itself. Every script operation prints exactly one token on stdout with diagnostics on stderr, inputs are validated, and the agent reads the last line and checks it is an interface name, so a stray warning can never end up as the bridge name in the domain XML. A missing modifymacip.sh is fatal for a Direct Routed NIC. Because the host routes for its Instances, their packets enter the host's own IP stack. Per bridge, the script inserts an INPUT rule that drops everything from the bridge that is not ICMP(v6), which gateway resolution and reachability checks need, and an ip6tables raw rpfilter rule as the IPv6 counterpart of rp_filter=1. Both are removed with the bridge. A failed VM start or migration prepare now unplugs the NICs it plugged. The security group wrapper passes --directrouted to security_group.py for these NICs, and a failed ipset update for secondary IPs is reported instead of returning success. --- .../kvm/resource/BridgeVifDriver.java | 213 ++++++++++++++++- .../resource/LibvirtComputingResource.java | 72 +++++- ...tworkRulesVmSecondaryIpCommandWrapper.java | 2 +- ...virtPrepareForMigrationCommandWrapper.java | 7 + ...bvirtSecurityGroupRulesCommandWrapper.java | 9 +- .../wrapper/LibvirtStartCommandWrapper.java | 6 + .../kvm/resource/BridgeVifDriverTest.java | 223 ++++++++++++++++++ .../LibvirtComputingResourceTest.java | 8 +- ...PrepareForMigrationCommandWrapperTest.java | 50 ++++ scripts/vm/network/vnet/modifybrdr.sh | 162 +++++++++++++ scripts/vm/network/vnet/modifymacip.sh | 29 ++- 11 files changed, 764 insertions(+), 17 deletions(-) create mode 100755 scripts/vm/network/vnet/modifybrdr.sh diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java index eefe491e8b27..e438910a4bb8 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java @@ -49,9 +49,41 @@ public class BridgeVifDriver extends VifDriverBase { private String _modifyVlanPath; private String _modifyVxlanPath; private String _macIpScriptPath; + private boolean _macIpStaticEnabled; + private String _modifyBrdrPath; private String _controlCidr = NetUtils.getLinkLocalCIDR(); private Long libvirtVersion; + /** + * What a Linux interface name may look like; modifybrdr.sh prints exactly one such name on + * a successful add, and anything else in its output is a diagnostic. + */ + private static final Pattern BRIDGE_NAME_PATTERN = Pattern.compile("^[A-Za-z0-9_.-]{1,15}$"); + private static final String BRDR_MINE = "mine"; + private static final String BRDR_NOTMINE = "notmine"; + private static final String BRDR_KEPT = "kept"; + private static final String BRDR_DELETED = "deleted"; + + /** + * A NIC on a Direct Routed (L3) network is recognised by its broadcast domain: routed://, + * stamped by the management server. The id is a label naming the per-network bridge + * (brdr-), never an encapsulation. An earlier revision inferred this from the address + * form (/32 + link-local gateway); the explicit broadcast type supersedes that implicit + * contract, and works for guest and (systemvm) public NICs alike. + * + * The URI is checked as well as the type enum: a SystemVM's public NIC profile is rebuilt at + * start from the shared Public network, whose broadcast domain type is Vlan — only the NIC's + * own broadcast URI carries the routed:// stamp then (NicProfile copies broadcastType from + * the network, not the nic row). + */ + public static boolean isDirectRoutedNic(NicTO nic) { + if (nic == null) { + return false; + } + return nic.getBroadcastType() == Networks.BroadcastDomainType.Routed + || (nic.getBroadcastUri() != null && Networks.BroadcastDomainType.Routed.scheme().equals(nic.getBroadcastUri().getScheme())); + } + private static boolean isVxlanOrNetris(String protocol) { return protocol.equals(Networks.BroadcastDomainType.Vxlan.scheme()) || protocol.equals(Networks.BroadcastDomainType.Netris.scheme()); } @@ -84,14 +116,17 @@ public void configure(Map params) throws ConfigurationException throw new ConfigurationException("Unable to find " + vxlanScript); } - if (Boolean.TRUE.equals(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VM_NETWORK_MACIP_STATIC))) { - _macIpScriptPath = Script.findScript(networkScriptsDir, "modifymacip.sh"); + _macIpScriptPath = Script.findScript(networkScriptsDir, "modifymacip.sh"); + _macIpStaticEnabled = Boolean.TRUE.equals(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VM_NETWORK_MACIP_STATIC)); + if (_macIpStaticEnabled) { if (_macIpScriptPath == null) { throw new ConfigurationException("Unable to find modifymacip.sh"); } logger.info("VM network MAC/IP static script configured: {}", _macIpScriptPath); } + _modifyBrdrPath = Script.findScript(networkScriptsDir, "modifybrdr.sh"); + libvirtVersion = (Long) params.get("libvirtVersion"); if (libvirtVersion == null) { libvirtVersion = 0L; @@ -221,6 +256,15 @@ protected String createStorageVnetBridgeIfNeeded(NicTO nic, String trafficLabel, return createVnetBr(vNetId, storageBrName, protocol); } + /** + * Direct Routed (L3) NICs — guest Instances and the public NICs of SystemVMs alike — go on + * the network's own uplink-less bridge, never on a shared guest/public bridge, and always get + * the MAC/IP script run: the host route and static neighbour entry it installs are what + * deliver their traffic. For every other NIC the MAC/IP script runs only when the host-wide + * vm.network.macip.static property (the EVPN use case) enables it — which is why the script + * path is resolved unconditionally in configure() but only required there when the property + * is set. + */ @Override public LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map extraConfig) throws InternalErrorException, LibvirtException { @@ -247,7 +291,10 @@ public LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicA networkRateKBps = getNetworkRateKbps(nic); } - if (nic.getType() == Networks.TrafficType.Guest) { + if ((nic.getType() == Networks.TrafficType.Guest || nic.getType() == Networks.TrafficType.Public) && isDirectRoutedNic(nic)) { + String brName = createDirectRoutedBridge(nic); + intf.defBridgeNet(brName, null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), networkRateKBps); + } else if (nic.getType() == Networks.TrafficType.Guest) { if (isBroadcastTypeVlanOrVxlan(nic) && isValidProtocolAndVnetId(vNetId, protocol)) { if (trafficLabel != null && !trafficLabel.isEmpty()) { logger.debug("creating a vNet dev and bridge for guest traffic per traffic label " + trafficLabel); @@ -295,17 +342,173 @@ public LibvirtVMDef.InterfaceDef plug(NicTO nic, String guestOsType, String nicA } intf.setLinkStateUp(nic.isEnabled()); - executeMacIpScript(intf.getBrName(), nic.getMac(), nic.getIp(), nic.getIp6Address(), nic.getNicSecIps()); + if (_macIpStaticEnabled || isDirectRoutedNic(nic)) { + executeMacIpScript(intf.getBrName(), nic.getMac(), nic.getIp(), nic.getIp6Address(), nic.getNicSecIps()); + } return intf; } + /** + * How the bridges of Direct Routed (L3) networks are named is known only to modifybrdr.sh; + * the interface is classified at unplug by asking the script ("query"), which answers + * "notmine" when it is not such a bridge and the regular unplug handling applies. For a + * Direct Routed bridge the host route and neighbour entry of the interface are removed + * first, while the bridge they reference still exists, and only then is the script asked to + * delete the bridge should nothing else be attached to it. + */ @Override public void unplug(LibvirtVMDef.InterfaceDef iface, boolean deleteBr) { - executeMacIpScript(iface.getBrName(), iface.getMacAddress()); + boolean directRouted = isDirectRoutedBridge(iface.getBrName()); + if (_macIpStaticEnabled || directRouted) { + executeMacIpScript(iface.getBrName(), iface.getMacAddress()); + } + if (directRouted) { + deleteDirectRoutedBridge(iface.getBrName()); + return; + } deleteVnetBr(iface.getBrName(), deleteBr); } + /** + * Ensures the per-network bridge for a Direct Routed NIC exists, with the shared gateway + * addresses and sysctls applied. Idempotent and flock'd in the script itself. A failure here + * is fatal to the NIC plug: without the bridge the domain XML would reference a nonexistent + * device and the Instance would fail to start with a far less useful error. A missing + * modifymacip.sh is fatal for the same reason: without the host route and neighbour entry it + * installs the Instance would start with no connectivity at all. + * + * The routed id — the value of the network's routed://<id> broadcast domain, operator + * controlled and stable for the network's life — names the bridge, but how these bridges are + * named is known only to modifybrdr.sh: the script prints the bridge name it created, never + * the agent. The script's stderr is merged into its stdout by Script, so the last non-blank + * line is taken and must look like an interface name; anything else means the script did not + * produce a bridge. The gateway addresses passed along come from the NIC, whose values the + * management server stamped at allocation (NetUtils.getLinkLocalGateway() / + * getIpv6LinkLocalGateway()); the script has no defaults of its own, so the addresses are + * defined in exactly one place. + */ + protected String createDirectRoutedBridge(NicTO nic) throws InternalErrorException { + if (_modifyBrdrPath == null) { + throw new InternalErrorException("Unable to find modifybrdr.sh: this host cannot run Instances on Direct Routed (L3) networks"); + } + if (_macIpScriptPath == null) { + throw new InternalErrorException("Unable to find modifymacip.sh: this host cannot run Instances on Direct Routed (L3) networks"); + } + String routedId = nic.getBroadcastUri() != null ? Networks.BroadcastDomainType.getValue(nic.getBroadcastUri()) : null; + if (StringUtils.isBlank(routedId)) { + throw new InternalErrorException("Direct Routed NIC " + nic.getMac() + " carries no routed:// broadcast URI; cannot derive its bridge"); + } + Script command = new Script(_modifyBrdrPath, _timeout, logger); + command.add("-o", "add"); + command.add("-n", routedId); + if (StringUtils.isNotBlank(nic.getGateway())) { + command.add("-4", nic.getGateway()); + } + if (StringUtils.isNotBlank(nic.getIp6Gateway())) { + command.add("-6", nic.getIp6Gateway()); + } + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + String result = command.execute(parser); + if (result != null) { + throw new InternalErrorException("Failed to create bridge for routed id " + routedId + ": " + result); + } + String brName = lastNonBlankLine(parser.getLines()); + if (brName == null || !BRIDGE_NAME_PATTERN.matcher(brName).matches()) { + throw new InternalErrorException("modifybrdr.sh did not return a usable bridge name for routed id " + routedId + ": " + parser.getLines()); + } + return brName; + } + + /** + * Asks modifybrdr.sh whether the bridge is one of its own without changing anything. Only an + * exact "mine" counts; "notmine", a script failure or an unexpected answer all send the + * caller down the regular unplug handling. + */ + protected boolean isDirectRoutedBridge(String brName) { + if (_modifyBrdrPath == null || brName == null) { + return false; + } + String verdict = runModifyBrdrVerdict("query", brName); + if (verdict == null) { + return false; + } + if (BRDR_MINE.equals(verdict)) { + return true; + } + if (!BRDR_NOTMINE.equals(verdict)) { + logger.warn("Unexpected answer from modifybrdr.sh query for bridge {}: {}", brName, verdict); + } + return false; + } + + /** + * Asks modifybrdr.sh to remove the bridge if it is one of its own and nothing is attached to + * it any more. Returns whether the script gave one of its three answers (notmine, kept, + * deleted); anything else is a failure and is logged. Best-effort beyond that: the script + * keeps the bridge while other Instances of the network still use it, and a leftover empty + * bridge is harmless and re-used on the next plug. + */ + protected boolean deleteDirectRoutedBridge(String brName) { + if (_modifyBrdrPath == null || brName == null) { + return false; + } + String verdict = runModifyBrdrVerdict("delete", brName); + if (verdict == null) { + return false; + } + if (BRDR_NOTMINE.equals(verdict) || BRDR_KEPT.equals(verdict) || BRDR_DELETED.equals(verdict)) { + logger.debug("modifybrdr.sh delete on bridge {}: {}", brName, verdict); + return true; + } + logger.warn("Unexpected answer from modifybrdr.sh delete for bridge {}: {}", brName, verdict); + return false; + } + + /** + * Runs a modifybrdr.sh operation that takes a bridge name and answers with a single token, + * returning the last non-blank line of its output, or null when the script failed. + */ + private String runModifyBrdrVerdict(String operation, String brName) { + try { + Script command = new Script(_modifyBrdrPath, _timeout, logger); + command.add("-o", operation); + command.add("-b", brName); + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + String result = command.execute(parser); + if (result != null) { + logger.warn("modifybrdr.sh {} failed for bridge {}: {}", operation, brName, result); + return null; + } + String verdict = lastNonBlankLine(parser.getLines()); + if (verdict == null) { + logger.warn("modifybrdr.sh {} gave no answer for bridge {}", operation, brName); + } + return verdict; + } catch (Exception e) { + logger.warn("Failed to run modifybrdr.sh {} for bridge {}", operation, brName, e); + return null; + } + } + + /** + * Returns the last non-blank line of a script's merged stdout/stderr output, trimmed, or + * null when there is none. The result token of modifybrdr.sh is always printed last. + */ + protected static String lastNonBlankLine(String output) { + if (output == null) { + return null; + } + String[] lines = output.split("\n"); + for (int i = lines.length - 1; i >= 0; i--) { + String line = lines[i].trim(); + if (!line.isEmpty()) { + return line; + } + } + return null; + } + @Override public void attach(LibvirtVMDef.InterfaceDef iface) { Script.runSimpleBashScript("ip link set " + iface.getDevName() + " master " + iface.getBrName()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index a5df0b3347f8..6565e08b5ec7 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -430,6 +430,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv private boolean imageServerTlsEnabled = false; private String imageServerListenAddress; private String securityGroupPath; + private String macIpPath; private String ovsPvlanDhcpHostPath; private String ovsPvlanVmPath; private String routerProxyPath; @@ -1198,6 +1199,7 @@ public boolean configure(final String name, final Map params) th } securityGroupPath = Script.findScript(networkScriptsDir, "security_group.py"); + macIpPath = Script.findScript(networkScriptsDir, "modifymacip.sh"); if (securityGroupPath == null) { throw new ConfigurationException("Unable to find the security_group.py"); } @@ -5037,7 +5039,7 @@ private DiskDef.DiskBus getGuestDiskModel(final String platformEmulator, boolean } } - private void cleanupVMNetworks(final Connect conn, final List nics) { + public void cleanupVMNetworks(final Connect conn, final List nics) { if (nics != null) { for (final InterfaceDef nic : nics) { for (final VifDriver vifDriver : getAllVifDrivers()) { @@ -5741,6 +5743,9 @@ public boolean defaultNetworkRules(final Connect conn, final String vmName, fina if (checkBeforeApply) { cmd.add("--check"); } + if (BridgeVifDriver.isDirectRoutedNic(nic)) { + cmd.add("--directrouted"); + } final String result = cmd.execute(); if (result != null) { return false; @@ -5869,7 +5874,7 @@ private Answer listLvmVolumes(String localPath, int startIndex, int pageSize) { } public boolean addNetworkRules(final String vmName, final String vmId, final String guestIP, final String guestIP6, final String sig, final String seq, final String mac, final String rules, final String vif, final String brname, - final String secIps) { + final String secIps, final boolean directRouted) { if (!canBridgeFirewall) { return false; } @@ -5879,7 +5884,9 @@ public boolean addNetworkRules(final String vmName, final String vmId, final Str cmd.add("add_network_rules"); cmd.add("--vmname", vmName); cmd.add("--vmid", vmId); - cmd.add("--vmip", guestIP); + if (StringUtils.isNotBlank(guestIP)) { + cmd.add("--vmip", guestIP); + } if (StringUtils.isNotBlank(guestIP6)) { cmd.add("--vmip6", guestIP6); } @@ -5892,6 +5899,9 @@ public boolean addNetworkRules(final String vmName, final String vmId, final Str if (newRules != null && !newRules.isEmpty()) { cmd.add("--rules", newRules); } + if (directRouted) { + cmd.add("--directrouted"); + } final String result = cmd.execute(); if (result != null) { return false; @@ -5900,8 +5910,27 @@ public boolean addNetworkRules(final String vmName, final String vmId, final Str } public boolean configureNetworkRulesVMSecondaryIP(final Connect conn, final String vmName, final String vmMac, final String secIp, final String action) { + return configureNetworkRulesVMSecondaryIP(conn, vmName, vmMac, secIp, action, false, true); + } + + /** + * On a Direct Routed network the secondary address only reaches the Instance once the host + * has a route and a neighbour entry for it. Security groups may be disabled there, so that + * is done regardless of canBridgeFirewall, and before any firewall rules. + */ + public boolean configureNetworkRulesVMSecondaryIP(final Connect conn, final String vmName, final String vmMac, final String secIp, final String action, + final boolean directRouted, final boolean applySecurityGroupRules) { + + if (directRouted && !configureDirectRoutedSecondaryIp(conn, vmName, vmMac, secIp, action)) { + return false; + } + + if (!applySecurityGroupRules) { + return true; + } if (!canBridgeFirewall) { + LOGGER.warn("Security group rules were requested for secondary IP {} of {} but this host cannot bridge firewall; the ipset was not updated", secIp, vmName); return false; } @@ -5919,6 +5948,43 @@ public boolean configureNetworkRulesVMSecondaryIP(final Connect conn, final Stri return true; } + /** + * Adds or removes the host route and static neighbour entry for a secondary IP of an + * Instance on a Direct Routed network, so the address is reachable without restarting it. + */ + private boolean configureDirectRoutedSecondaryIp(final Connect conn, final String vmName, final String vmMac, final String secIp, final String action) { + if (macIpPath == null) { + LOGGER.warn("Unable to find modifymacip.sh, cannot configure secondary IP {} for {}", secIp, vmName); + return false; + } + if (StringUtils.isBlank(vmMac) || StringUtils.isBlank(secIp)) { + LOGGER.warn("Cannot configure secondary IP for {}: MAC '{}' or address '{}' is missing", vmName, vmMac, secIp); + return false; + } + String brName = null; + for (final InterfaceDef intf : getInterfaces(conn, vmName)) { + if (vmMac.equalsIgnoreCase(intf.getMacAddress())) { + brName = intf.getBrName(); + break; + } + } + if (brName == null) { + LOGGER.warn("Unable to find the interface of {} with MAC {}", vmName, vmMac); + return false; + } + final Script cmd = new Script(macIpPath, timeout, LOGGER); + cmd.add("-o", "-A".equals(action) ? "add" : "delete"); + cmd.add("-b", brName); + cmd.add("-m", vmMac); + cmd.add(NetUtils.isValidIp6(secIp) ? "-6" : "-4", secIp); + final String result = cmd.execute(); + if (result != null) { + LOGGER.warn("Failed to configure secondary IP {} for {}: {}", secIp, vmName, result); + return false; + } + return true; + } + public boolean setupTungstenVRouter(final String oper, final String inf, final String subnet, final String route, final String vrf) { final Script cmd = new Script(setupTungstenVrouterPath, timeout, LOGGER); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtNetworkRulesVmSecondaryIpCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtNetworkRulesVmSecondaryIpCommandWrapper.java index 890558ca3651..018d3b929284 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtNetworkRulesVmSecondaryIpCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtNetworkRulesVmSecondaryIpCommandWrapper.java @@ -39,7 +39,7 @@ public Answer execute(final NetworkRulesVmSecondaryIpCommand command, final Libv final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper(); final Connect conn = libvirtUtilitiesHelper.getConnectionByVmName(command.getVmName()); - result = libvirtComputingResource.configureNetworkRulesVMSecondaryIP(conn, command.getVmName(), command.getVmMac(), command.getVmSecIp(), command.getAction()); + result = libvirtComputingResource.configureNetworkRulesVMSecondaryIP(conn, command.getVmName(), command.getVmMac(), command.getVmSecIp(), command.getAction(), command.isDirectRouted(), command.isApplySecurityGroupRules()); } catch (final LibvirtException e) { logger.debug("Could not configure VM secondary IP! => " + e.getLocalizedMessage()); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapper.java index f7ca79127dad..f399c26d264f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapper.java @@ -20,6 +20,7 @@ package com.cloud.hypervisor.kvm.resource.wrapper; import java.net.URISyntaxException; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -71,6 +72,7 @@ public Answer execute(final PrepareForMigrationCommand command, final LibvirtCom boolean skipDisconnect = false; + final List pluggedNics = new ArrayList<>(); final KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr(); try { final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper(); @@ -79,6 +81,9 @@ public Answer execute(final PrepareForMigrationCommand command, final LibvirtCom for (final NicTO nic : nics) { LibvirtVMDef.InterfaceDef interfaceDef = libvirtComputingResource.getVifDriver(nic.getType(), nic.getName()).plug(nic, null, "", vm.getExtraConfig()); + if (interfaceDef != null) { + pluggedNics.add(interfaceDef); + } if (vm.getDetails() != null) { libvirtComputingResource.setInterfaceDefQueueSettings(vm.getDetails(), vm.getCpus(), interfaceDef); } @@ -122,6 +127,7 @@ public Answer execute(final PrepareForMigrationCommand command, final LibvirtCom skipDisconnect = true; if (!storagePoolMgr.connectPhysicalDisksViaVmSpec(vm, true)) { + libvirtComputingResource.cleanupVMNetworks(conn, pluggedNics); return new PrepareForMigrationAnswer(command, "failed to connect physical disks to host"); } @@ -146,6 +152,7 @@ public Answer execute(final PrepareForMigrationCommand command, final LibvirtCom removeDpdkPort(to.getPort()); } } + libvirtComputingResource.cleanupVMNetworks(null, pluggedNics); return new PrepareForMigrationAnswer(command, e.toString()); } finally { if (!skipDisconnect) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtSecurityGroupRulesCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtSecurityGroupRulesCommandWrapper.java index 33164264ae80..aa6a13b416f5 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtSecurityGroupRulesCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtSecurityGroupRulesCommandWrapper.java @@ -28,6 +28,7 @@ import com.cloud.agent.api.SecurityGroupRuleAnswer; import com.cloud.agent.api.SecurityGroupRulesCmd; import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.hypervisor.kvm.resource.BridgeVifDriver; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef; import com.cloud.resource.CommandWrapper; @@ -59,8 +60,14 @@ public Answer execute(final SecurityGroupRulesCmd command, final LibvirtComputin return new SecurityGroupRuleAnswer(command, false, e.toString()); } + boolean directRouted = false; + final VirtualMachineTO vmTO = command.getVmTO(); + if (vmTO != null && vmTO.getNics() != null && vmTO.getNics().length > 0) { + directRouted = BridgeVifDriver.isDirectRoutedNic(vmTO.getNics()[0]); + } + final boolean result = libvirtComputingResource.addNetworkRules(command.getVmName(), Long.toString(command.getVmId()), command.getGuestIp(), command.getGuestIp6(), command.getSignature(), - Long.toString(command.getSeqNum()), command.getGuestMac(), command.stringifyRules(), vif, brname, command.getSecIpsString()); + Long.toString(command.getSeqNum()), command.getGuestMac(), command.stringifyRules(), vif, brname, command.getSecIpsString(), directRouted); if (!result) { logger.warn("Failed to program network rules for vm " + command.getVmName()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java index 486989661909..2ce64a80a4de 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java @@ -169,6 +169,12 @@ public Answer execute(final StartCommand command, final LibvirtComputingResource libvirtComputingResource.handleVmStartFailure(conn, vmName, vm); } return new StartAnswer(command, e.getMessage()); + } catch (final RuntimeException e) { + logger.warn("RuntimeException while starting VM {}", vmName, e); + if (conn != null) { + libvirtComputingResource.handleVmStartFailure(conn, vmName, vm); + } + return new StartAnswer(command, e.toString()); } finally { if (state != DomainState.VIR_DOMAIN_RUNNING) { storagePoolMgr.disconnectPhysicalDisksViaVmSpec(vmSpec); diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriverTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriverTest.java index 00364948f828..d86d31859197 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriverTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriverTest.java @@ -16,30 +16,83 @@ // under the License. package com.cloud.hypervisor.kvm.resource; +import java.io.BufferedReader; +import java.io.StringReader; +import java.lang.reflect.Field; import java.net.URI; import java.net.URISyntaxException; +import java.util.List; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; +import org.mockito.MockedConstruction; import org.mockito.Mockito; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; import com.cloud.agent.api.to.NicTO; import com.cloud.exception.InternalErrorException; import com.cloud.network.Networks; +import com.cloud.utils.script.OutputInterpreter; +import com.cloud.utils.script.Script; @RunWith(MockitoJUnitRunner.class) public class BridgeVifDriverTest { private static final String BRIDGE_NAME = "cloudbr1"; + private static final String MODIFY_BRDR = "/scripts/modifybrdr.sh"; + private static final String MODIFY_MACIP = "/scripts/modifymacip.sh"; @Spy @InjectMocks private BridgeVifDriver driver = new BridgeVifDriver(); + private static void setField(Object target, String name, Object value) throws Exception { + Field field = BridgeVifDriver.class.getDeclaredField(name); + field.setAccessible(true); + field.set(target, value); + } + + /** + * Feeds the given text to the OutputInterpreter a Script.execute(parser) call passes, the + * way Script does with the merged stdout/stderr of the real process, and reports success. + */ + private static Answer scriptOutput(String output) { + return invocation -> { + OutputInterpreter parser = invocation.getArgument(0); + parser.interpret(new BufferedReader(new StringReader(output))); + return null; + }; + } + + private static MockedConstruction + + diff --git a/ui/src/views/network/CreateNetwork.vue b/ui/src/views/network/CreateNetwork.vue index 46ed9d2e039f..385bb1bba98e 100644 --- a/ui/src/views/network/CreateNetwork.vue +++ b/ui/src/views/network/CreateNetwork.vue @@ -34,6 +34,13 @@ @refresh-data="refreshParent" @refresh="handleRefresh"/> + + + {{ $t('label.l2') }} + + {{ $t('label.l3') }} + {{ $t('label.shared') }} @@ -97,13 +100,15 @@ - + @@ -206,6 +211,20 @@ + + + + +
+ + + + + + + @@ -292,7 +311,7 @@ - + @@ -802,7 +821,7 @@ export default { this.guestType = val this.networkmode = '' this.form.networkmode = '' - if (val === 'l2') { + if (val === 'l2' || val === 'l3') { this.form.forvpc = false this.form.lbtype = 'publicLb' this.isVirtualRouterForAtLeastOneService = false @@ -821,6 +840,10 @@ export default { this.firewallServiceProvider = '' this.selectedServiceProviderMap = {} } + if (val === 'l3') { + this.form.dnsl3 = true + this.form.securitygroupl3 = false + } this.fetchSupportedServiceData() }, fetchSupportedServiceData () { @@ -1137,7 +1160,7 @@ export default { var keys = Object.keys(values) const detailsKey = ['promiscuousmode', 'macaddresschanges', 'forgedtransmits', 'maclearning'] - const ignoredKeys = [...detailsKey, 'state', 'status', 'allocationstate', 'forvpc', 'lbType', 'specifyvlan', 'ispublic', 'domainid', 'zoneid', 'egressdefaultpolicy', 'isolation', 'supportspublicaccess'] + const ignoredKeys = [...detailsKey, 'state', 'status', 'allocationstate', 'forvpc', 'lbType', 'specifyvlan', 'ispublic', 'domainid', 'zoneid', 'egressdefaultpolicy', 'isolation', 'supportspublicaccess', 'dnsl3', 'securitygroupl3'] keys.forEach(function (key, keyIndex) { if (!ignoredKeys.includes(key) && values[key] != null && values[key] !== undefined && @@ -1161,6 +1184,29 @@ export default { } else { // Isolated Network with Non-persistent network delete params.ispersistent } + } else if (values.guestiptype === 'l3') { + if (values.specifyvlan === true) { + params.specifyvlan = true + } + params.specifyipranges = true + delete params.ispersistent + delete params.conservemode + const l3Services = ['UserData'] + params['serviceProviderList[0].service'] = 'UserData' + params['serviceProviderList[0].provider'] = 'ConfigDrive' + var l3ServiceIndex = 1 + if (values.dnsl3 === true) { + params['serviceProviderList[' + l3ServiceIndex + '].service'] = 'Dns' + params['serviceProviderList[' + l3ServiceIndex + '].provider'] = 'ConfigDrive' + l3Services.push('Dns') + l3ServiceIndex++ + } + if (values.securitygroupl3 === true) { + params['serviceProviderList[' + l3ServiceIndex + '].service'] = 'SecurityGroup' + params['serviceProviderList[' + l3ServiceIndex + '].provider'] = 'SecurityGroupProvider' + l3Services.push('SecurityGroup') + } + params.supportedservices = l3Services.join(',') } else if (values.guestiptype === 'l2') { if (values.specifyvlan === true) { params.specifyvlan = true From 2a2f1ef97ee1126c8a39cd1c21be9d44c33edbe6 Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Tue, 15 Sep 2026 08:33:13 +0000 Subject: [PATCH 7/8] Marvin: smoke tests for Direct Routed networks test_l3_networks covers offering and network creation, IPv4-only, IPv6-only and dual-stack networks, zone-wide overlap detection, a requested IPv4 address, the subnet's .0 address and an operator-chosen routed id, and deploys an Instance to assert both address families in host-route form. Marvin gains the cidr and routed id parameters and the test data for the offering and networks. --- test/integration/smoke/test_l3_networks.py | 402 +++++++++++++++++++++ tools/marvin/marvin/config/test_data.py | 37 ++ tools/marvin/marvin/lib/base.py | 2 + 3 files changed, 441 insertions(+) create mode 100644 test/integration/smoke/test_l3_networks.py diff --git a/test/integration/smoke/test_l3_networks.py b/test/integration/smoke/test_l3_networks.py new file mode 100644 index 000000000000..25811581eb6d --- /dev/null +++ b/test/integration/smoke/test_l3_networks.py @@ -0,0 +1,402 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" Tests for L3 (Direct Routed) guest networks: the hypervisor routes a public IPv4/IPv6 + address directly to the Instance - no Virtual Router, no NAT, no DHCP. Instances receive + a /32 (or /128) with the shared link-local gateway, delivered via ConfigDrive. +""" + +import random + +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.lib.base import (Account, + Network, + NetworkOffering, + PhysicalNetwork, + ServiceOffering, + VirtualMachine, + Zone) +from marvin.lib.common import (get_domain, + get_template, + get_zone) +from marvin.lib.utils import cleanup_resources +from nose.plugins.attrib import attr + + +class TestL3Networks(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + testClient = super(TestL3Networks, cls).getClsTestClient() + cls.apiclient = testClient.getApiClient() + cls.services = testClient.getParsedTestDataConfig() + + cls.domain = get_domain(cls.apiclient) + cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests()) + cls.template = get_template(cls.apiclient, cls.zone.id, cls.services["ostype"]) + + cls._cleanup = [] + cls.skip = False + + zone = Zone(cls.zone.__dict__) + if zone.networktype != 'Advanced': + cls.skip = True + return + + cls.services["virtual_machine"]["zoneid"] = cls.zone.id + cls.services["virtual_machine"]["template"] = cls.template.id + + # The network operator's opt-in: L3 networks live on a dedicated physical network + # with isolation method ROUTED. Its "vlan" range is the routed-id pool for networks + # whose offering does not carry specifyVlan; its tag steers L3 offerings to it. + cls.physical_network = PhysicalNetwork.create( + cls.apiclient, + {"name": "l3-direct-routed"}, + zoneid=cls.zone.id, + isolationmethods="ROUTED" + ) + cls._cleanup.append(cls.physical_network) + cls.physical_network.addTrafficType(cls.apiclient, "Guest") + cls.physical_network.update( + cls.apiclient, + vlan="5800-5899", + tags="l3routed", + state="Enabled" + ) + + cls.service_offering = ServiceOffering.create( + cls.apiclient, + cls.services["service_offering"] + ) + cls._cleanup.append(cls.service_offering) + + cls.network_offering = NetworkOffering.create( + cls.apiclient, + cls.services["l3_network_offering"] + ) + cls._cleanup.append(cls.network_offering) + cls.network_offering.update(cls.apiclient, state='Enabled') + + cls.network_offering_specifyid = NetworkOffering.create( + cls.apiclient, + cls.services["l3_network_offering_specifyid"] + ) + cls._cleanup.append(cls.network_offering_specifyid) + cls.network_offering_specifyid.update(cls.apiclient, state='Enabled') + + cls.account = Account.create( + cls.apiclient, + cls.services["account"], + admin=True, + domainid=cls.domain.id + ) + cls._cleanup.append(cls.account) + + @classmethod + def tearDownClass(cls): + try: + cleanup_resources(cls.apiclient, reversed(cls._cleanup)) + except Exception as e: + raise Exception("Warning: Exception during class cleanup : %s" % e) + + def setUp(self): + if self.skip: + self.skipTest("L3 networks require an Advanced zone, skipping") + self.cleanup = [] + + def tearDown(self): + try: + cleanup_resources(self.apiclient, reversed(self.cleanup)) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + def create_l3_network(self, startip="203.0.113.10", endip="203.0.113.50"): + services = dict(self.services["l3_network"]) + services["startip"] = startip + services["endip"] = endip + return Network.create( + self.apiclient, + services, + zoneid=self.zone.id, + networkofferingid=self.network_offering.id, + accountid=self.account.name, + domainid=self.account.domainid + ) + + def deploy_vm(self, network, **kwargs): + virtual_machine = VirtualMachine.create( + self.apiclient, + self.services["virtual_machine"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.service_offering.id, + networkids=[network.id], + **kwargs + ) + self.cleanup.append(virtual_machine) + self.assertEqual(virtual_machine.state, "Running") + return virtual_machine + + @attr(tags=["advanced", "smoke"], required_hardware="false") + def test_01_create_l3_network(self): + """ An L3 network is created like a Shared network: with a subnet. The subnet is an + allocation pool routed to the hypervisors, not a broadcast domain. The network + carries a routed:// broadcast domain - here allocated from the ROUTED + physical network's range - whose id names the bridge (brdr-) on the hosts. """ + network = self.create_l3_network() + self.cleanup.append(network) + + self.assertEqual(network.type, "L3", "network type should be L3") + self.assertEqual(network.broadcastdomaintype, "Routed", "an L3 network has a routed broadcast domain") + self.assertTrue(network.broadcasturi.startswith("routed://"), + "the broadcast URI must be routed://, got %s" % network.broadcasturi) + allocated_id = int(network.broadcasturi.replace("routed://", "")) + self.assertTrue(5800 <= allocated_id <= 5899, + "the routed id must come from the physical network's range, got %d" % allocated_id) + self.assertIn(network.state, ["Setup", "Allocated"], "unexpected network state") + self.assertFalse(getattr(network, "gateway", None), + "an L3 network stores no IPv4 gateway: instances use the shared link-local gateway") + + @attr(tags=["advanced", "smoke"], required_hardware="false") + def test_01b_create_l3_network_with_operator_specified_id(self): + """ With a specifyVlan offering the operator picks the routed id at creation via the + vlan parameter, so bridge names (brdr-) are plannable before the network + exists. The id must lie outside the physical network's dynamic range. """ + services = dict(self.services["l3_network"]) + network = None + # Outside the physical network's dynamic range (5800-5899); retried once in case the + # id is already taken by a pre-existing network or public range in the zone. + for attempt in range(2): + routed_id = random.randint(6000, 9999) + try: + network = Network.create( + self.apiclient, + services, + zoneid=self.zone.id, + networkofferingid=self.network_offering_specifyid.id, + accountid=self.account.name, + domainid=self.account.domainid, + vlan=str(routed_id) + ) + break + except Exception as e: + if attempt == 0 and "already" in str(e): + continue + raise + self.cleanup.append(network) + self.assertEqual(network.broadcasturi, "routed://%d" % routed_id, + "the operator-specified routed id must be carried verbatim, got %s" % network.broadcasturi) + + @attr(tags=["advanced", "smoke"], required_hardware="false") + def test_02_deploy_vm_in_l3_network(self): + """ The Instance's NIC carries its address as a host route: a /32 with the shared, + host-independent link-local gateway. The subnet's own gateway is never used. """ + network = self.create_l3_network() + self.cleanup.append(network) + + virtual_machine = self.deploy_vm(network) + nic = virtual_machine.nic[0] + self.assertEqual(nic.netmask, "255.255.255.255", "an L3 NIC address is a host route (/32)") + self.assertEqual(nic.gateway, "169.254.0.1", "an L3 NIC uses the shared link-local gateway") + self.assertTrue(nic.ipaddress.startswith("203.0.113."), "the address must come from the network's subnet") + + @attr(tags=["advanced", "smoke"], required_hardware="false") + def test_02b_deploy_vm_with_requested_ip_in_l3_network(self): + """ As on a Shared network, a user may ask for a specific address from the network's + range at deploy time and the Instance receives exactly that address. """ + network = self.create_l3_network() + self.cleanup.append(network) + + virtual_machine = self.deploy_vm(network, ipaddress="203.0.113.25") + nic = virtual_machine.nic[0] + self.assertEqual(nic.ipaddress, "203.0.113.25", "the Instance must receive the requested address") + self.assertEqual(nic.netmask, "255.255.255.255", "an L3 NIC address is a host route (/32)") + self.assertEqual(nic.gateway, "169.254.0.1", "an L3 NIC uses the shared link-local gateway") + + @attr(tags=["advanced", "smoke"], required_hardware="false") + def test_02c_network_address_is_assignable(self): + """ There is no broadcast domain, so the subnet's first (.0) and last (.255) addresses + are ordinary routable addresses: every Instance is a /32 behind the host's routing + table and nothing broadcasts to them. A range consisting of only the network + address must therefore be accepted, and an Instance must receive it. """ + network = self.create_l3_network(startip="203.0.115.0", endip="203.0.115.0") + self.cleanup.append(network) + self.assertEqual(network.cidr, "203.0.115.0/24", "the subnet must derive from the range and netmask") + + virtual_machine = self.deploy_vm(network) + nic = virtual_machine.nic[0] + self.assertEqual(nic.ipaddress, "203.0.115.0", "the network address must be assignable to an Instance") + self.assertEqual(nic.netmask, "255.255.255.255", "an L3 NIC address is a host route (/32)") + self.assertEqual(nic.gateway, "169.254.0.1", "an L3 NIC uses the shared link-local gateway") + + @attr(tags=["advanced", "smoke"], required_hardware="false") + def test_03_l3_offering_rejects_dhcp(self): + """ DHCP is not supported and not needed on L3 networks: ConfigDrive carries the + address, netmask, gateway and routes, so DHCP would have nothing left to hand out. """ + services = dict(self.services["l3_network_offering"]) + services["name"] = "Test L3 offering with Dhcp - must fail" + services["supportedservices"] = "UserData,Dns,Dhcp" + services["serviceProviderList"] = { + "UserData": "ConfigDrive", + "Dns": "ConfigDrive", + "Dhcp": "ConfigDrive" + } + with self.assertRaises(Exception): + NetworkOffering.create(self.apiclient, services) + + @attr(tags=["advanced", "smoke"], required_hardware="false") + def test_04_l3_offering_requires_userdata(self): + """ UserData via ConfigDrive is mandatory: it is the only channel that carries the + Instance's network configuration. """ + services = dict(self.services["l3_network_offering"]) + services["name"] = "Test L3 offering without UserData - must fail" + services["supportedservices"] = "Dns" + services["serviceProviderList"] = {"Dns": "ConfigDrive"} + with self.assertRaises(Exception): + NetworkOffering.create(self.apiclient, services) + + @attr(tags=["advanced", "smoke"], required_hardware="false") + def test_05_l3_subnets_may_not_overlap_zone_wide(self): + """ All L3 subnets share one host routing table and one routing fabric, so an overlap + is an address conflict, not a policy preference. The check is zone wide. """ + network = self.create_l3_network(startip="203.0.113.10", endip="203.0.113.30") + self.cleanup.append(network) + + with self.assertRaises(Exception): + overlapping = self.create_l3_network(startip="203.0.113.20", endip="203.0.113.40") + self.cleanup.append(overlapping) + + def create_ipv6_only_l3_network(self, ip6cidr="2001:db8:113::/64"): + services = { + "name": "Test IPv6-only L3 Network", + "displaytext": "Test IPv6-only L3 Network", + "ip6cidr": ip6cidr + } + return Network.create( + self.apiclient, + services, + zoneid=self.zone.id, + networkofferingid=self.network_offering.id, + accountid=self.account.name, + domainid=self.account.domainid + ) + + @attr(tags=["advanced", "smoke"], required_hardware="false") + def test_06_create_ipv6_only_l3_network(self): + """ Nothing on an L3 network depends on IPv4 - no DHCP, no password or metadata + service - so IPv4 is optional and an IPv6-only network is valid. Addresses + derive from the subnet and the NIC MAC (EUI-64), so no IPv6 range is needed + either: ip6cidr alone defines the network, and a given ip6gateway is ignored. """ + network = self.create_ipv6_only_l3_network() + self.cleanup.append(network) + + self.assertEqual(network.type, "L3", "network type should be L3") + self.assertEqual(network.ip6cidr, "2001:db8:113::/64", "the network must carry its IPv6 CIDR") + self.assertFalse(getattr(network, "cidr", None), "an IPv6-only network must carry no IPv4 CIDR") + self.assertFalse(getattr(network, "ip6gateway", None), "an L3 network stores no IPv6 gateway") + + @attr(tags=["advanced", "smoke"], required_hardware="false") + def test_07_deploy_vm_in_ipv6_only_l3_network(self): + """ An Instance in an IPv6-only L3 network gets a /128 with the shared link-local + gateway and no IPv4 address at all. """ + network = self.create_ipv6_only_l3_network() + self.cleanup.append(network) + + virtual_machine = self.deploy_vm(network) + nic = virtual_machine.nic[0] + self.assertTrue(getattr(nic, "ip6address", None), "the NIC must carry an IPv6 address") + self.assertEqual(nic.ip6gateway, "fe80::1", "an L3 NIC uses the shared link-local IPv6 gateway") + self.assertFalse(getattr(nic, "ipaddress", None), "the NIC of an IPv6-only network must carry no IPv4 address") + + @attr(tags=["advanced", "smoke"], required_hardware="false") + def test_08_l3_network_rejects_incomplete_address_family(self): + """ Each address family is optional, but a given family must be complete, and at + least one family must be present. """ + incomplete_ipv4 = { + "name": "Test L3 incomplete IPv4 - must fail", + "displaytext": "Test L3 incomplete IPv4 - must fail", + "gateway": "203.0.113.1", + "netmask": "255.255.255.0" + } + no_family = { + "name": "Test L3 without addresses - must fail", + "displaytext": "Test L3 without addresses - must fail" + } + for services in [incomplete_ipv4, no_family]: + with self.assertRaises(Exception): + network = Network.create( + self.apiclient, + services, + zoneid=self.zone.id, + networkofferingid=self.network_offering.id, + accountid=self.account.name, + domainid=self.account.domainid + ) + self.cleanup.append(network) + + @attr(tags=["advanced", "smoke"], required_hardware="false") + def test_09_l3_network_ignores_gateways(self): + """ Gateways play no part on an L3 network - the Instance's gateway is always the + shared link-local address - so given gateways are accepted but ignored, and no + address in the subnet is burnt for one. An Instance on this dual-stack network + gets a /32 and a /128, each with its shared link-local gateway. """ + services = dict(self.services["l3_network"]) + services["gateway"] = "203.0.113.1" + services["ip6gateway"] = "2001:db8:113::1" + services["ip6cidr"] = "2001:db8:113::/64" + network = Network.create( + self.apiclient, + services, + zoneid=self.zone.id, + networkofferingid=self.network_offering.id, + accountid=self.account.name, + domainid=self.account.domainid + ) + self.cleanup.append(network) + self.assertFalse(getattr(network, "gateway", None), "the given IPv4 gateway must be ignored") + self.assertFalse(getattr(network, "ip6gateway", None), "the given IPv6 gateway must be ignored") + + virtual_machine = self.deploy_vm(network) + nic = virtual_machine.nic[0] + self.assertTrue(getattr(nic, "ipaddress", None), "the NIC must carry an IPv4 address") + self.assertEqual(nic.netmask, "255.255.255.255", "an L3 NIC address is a host route (/32)") + self.assertEqual(nic.gateway, "169.254.0.1", "an L3 NIC uses the shared link-local gateway") + self.assertTrue(getattr(nic, "ip6address", None), "the NIC must carry an IPv6 address") + self.assertTrue(getattr(nic, "ip6cidr", "").endswith("/128"), + "an L3 NIC IPv6 address is a host route (/128), got %s" % getattr(nic, "ip6cidr", None)) + self.assertEqual(nic.ip6gateway, "fe80::1", "an L3 NIC uses the shared link-local IPv6 gateway") + + @attr(tags=["advanced", "smoke"], required_hardware="false") + def test_10_create_l3_network_by_cidr(self): + """ The IPv4 subnet can be given as a CIDR, like IPv6: CloudStack derives the netmask + and defaults the IP range to the subnet's usable addresses. """ + services = { + "name": "Test L3 Network by CIDR", + "displaytext": "Test L3 Network by CIDR", + "cidr": "203.0.114.0/24" + } + network = Network.create( + self.apiclient, + services, + zoneid=self.zone.id, + networkofferingid=self.network_offering.id, + accountid=self.account.name, + domainid=self.account.domainid + ) + self.cleanup.append(network) + self.assertEqual(network.cidr, "203.0.114.0/24", "the network must carry the given IPv4 subnet") + self.assertEqual(network.netmask, "255.255.255.0", "the netmask must derive from the CIDR") + diff --git a/tools/marvin/marvin/config/test_data.py b/tools/marvin/marvin/config/test_data.py index e3d4022cf0f9..e6d3df6bd12d 100644 --- a/tools/marvin/marvin/config/test_data.py +++ b/tools/marvin/marvin/config/test_data.py @@ -221,6 +221,43 @@ "endip": "172.16.15.41", "acltype": "Account" }, + "l3_network_offering": { + "name": "Test L3 Direct Routed - Network offering", + "displaytext": "Test L3 Direct Routed - Network offering", + "guestiptype": "L3", + "supportedservices": "UserData,Dns", + "specifyIpRanges": "True", + "specifyVlan": "False", + "traffictype": "GUEST", + "availability": "Optional", + "tags": "l3routed", + "serviceProviderList": { + "UserData": "ConfigDrive", + "Dns": "ConfigDrive" + } + }, + "l3_network_offering_specifyid": { + "name": "Test L3 Direct Routed - Network offering, operator-specified id", + "displaytext": "Test L3 Direct Routed - Network offering, operator-specified id", + "guestiptype": "L3", + "supportedservices": "UserData,Dns", + "specifyIpRanges": "True", + "specifyVlan": "True", + "traffictype": "GUEST", + "availability": "Optional", + "tags": "l3routed", + "serviceProviderList": { + "UserData": "ConfigDrive", + "Dns": "ConfigDrive" + } + }, + "l3_network": { + "name": "Test L3 Direct Routed Network", + "displaytext": "Test L3 Direct Routed Network", + "netmask": "255.255.255.0", + "startip": "203.0.113.10", + "endip": "203.0.113.50" + }, "l2-network_offering": { "name": "Test L2 - Network offering", "displaytext": "Test L2 - Network offering", diff --git a/tools/marvin/marvin/lib/base.py b/tools/marvin/marvin/lib/base.py index e7fa2f763db5..d0a03ff8b0be 100755 --- a/tools/marvin/marvin/lib/base.py +++ b/tools/marvin/marvin/lib/base.py @@ -3706,6 +3706,8 @@ def create(cls, apiclient, services, accountid=None, domainid=None, cmd.cidrsize = cidrsize elif "cidrsize" in services: cmd.cidrsize = services["cidrsize"] + if "cidr" in services: + cmd.cidr = services["cidr"] if "startip" in services: cmd.startip = services["startip"] if "endip" in services: From d7f67e8520c3a32472cf207002fbd3d048b0eb81 Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Tue, 15 Sep 2026 08:33:13 +0000 Subject: [PATCH 8/8] Design document for Direct Routed networks Records the network model, the decisions taken and their alternatives, the hypervisor and security group design, the review findings that were deferred, and the implementation status. Excluded from the RAT check. --- docs/design/direct-routed-networks.md | 1672 +++++++++++++++++++++++++ pom.xml | 1 + 2 files changed, 1673 insertions(+) create mode 100644 docs/design/direct-routed-networks.md diff --git a/docs/design/direct-routed-networks.md b/docs/design/direct-routed-networks.md new file mode 100644 index 000000000000..1222373a2ae9 --- /dev/null +++ b/docs/design/direct-routed-networks.md @@ -0,0 +1,1672 @@ +# Direct Routed Networks + +**Status:** design agreed — open items are implementation and verification only +**Branch:** `direct-routed-network` +**Author:** Wido den Hollander +**Last updated:** 2026-09-04 + +> **Revision 2026-09-04.** The isolation model changed: direct routed networks now live on a +> **dedicated physical network with isolation method `ROUTED`**, and every network carries a +> broadcast domain of the new type **`routed://`**, whose id names the network's bridge +> (`brdr-`). This reverses two earlier decisions — "no isolation method" and "no broadcast +> domain" — recorded with rationale in §6.7 and §9.2.1 and in the decision log (§16). + +--- + +## 1. Summary + +A new guest network type in which the **hypervisor performs L3 routing for the guest**. There is +no Virtual Router, no NAT, and **no DHCP** — the defining characteristic of this network type. + +The operator creates a network and adds a subnet to it, as for a Shared network. CloudStack then +allocates **individual addresses** out of that subnet to guest NICs, and hands each NIC: + +* its IPv4 address as a **/32** and its IPv6 address as a **/128** +* a **shared, host-independent gateway**: `169.254.0.1` (IPv4) and `fe80::1` (IPv6), configured on + the network's bridge on every hypervisor +* the whole configuration via **ConfigDrive / cloud-init** — mandatory, since without DHCP or RA + there is no other way for the guest to learn its address + +The hypervisor's CloudStack agent installs, per guest address, a host route and a static neighbour +entry on the guest bridge, binding the address to that guest's MAC. This reuses the existing +`modifymacip.sh` hook unchanged (§9.1). A routing daemon on the host advertises those addresses to +the fabric. **The routing daemon is out of scope for this feature** — see §10. + +Direct routed networks are the network operator's choice, expressed in the zone design: they live +on a **dedicated physical network whose isolation method is `ROUTED`**. Each network carries a +broadcast domain of the new type **`routed://`** — for example `routed://5828` — and that id +names the network's bridge on every hypervisor: `brdr-5828`. The id is not an encapsulation and +never appears on the wire; it is the stable, operator-controlled handle that ties a network to +host-side routing policy. The operator either picks it at network creation or lets CloudStack +allocate one from the range configured on the physical network (§6.7, §9.2). + +## 2. Motivation + +Compared with what CloudStack offers today: + +| | Shared | Isolated (NATTED) | Isolated (ROUTED, 4.20+) | **Direct Routed** | +|---|---|---|---|---| +| VR in data path | for DHCP/DNS | yes | yes | **no** | +| DHCP | yes | yes | yes | **no** | +| VM gets routable IP | yes | no (NAT) | yes | yes | +| Guest netmask | subnet mask | subnet mask | subnet mask | **/32, /128** | +| Shared broadcast domain | yes | yes | yes | **no** | +| Guest-to-guest | switched | switched | switched | **routed by host** | +| Isolation ID | VLAN/VXLAN | VLAN/VXLAN | VLAN/VXLAN | **routed id — a label, no encapsulation** | + +What this closes: + +* **No VR in the data path.** `ROUTED` networks (4.20) removed NAT but kept a VR in the path and an + L2 segment per network. Throughput, failover, and per-network VR footprint remain concerns. +* **No shared broadcast domain between networks.** Each network gets its own uplink-less bridge + (§9.2), so there is no L2 path of any kind between networks — no ARP spoofing, no rogue DHCP + server, no rogue RA reaching another tenant. Isolation is topological, so it does not depend on a + rule set being correct, and an administrator can turn security groups off for a network without + weakening it. Within a single network guests still share a bridge; that residual exposure is + intra-tenant and is documented in §12.3. +* **No encapsulation at all.** No VLAN on the wire, no VXLAN, no tunnel. The routed id in + `routed://` is a **label, not a fabric resource**: it exists only to name the per-network + bridge (`brdr-`), consumes nothing on the switches, and is either chosen by the operator or + allocated from a range the operator configures on the `ROUTED` physical network (§6.7, §9.2.1). + Guest network count is bounded only by that range, and each network still gets a real L2 boundary. +* **Per-network routing policy on the hypervisor.** Each network is a distinct, named L3 interface + (`brdr-5828`), so the operator can apply different route-maps, redistribution filters, policy + routing or QoS per network in the host's own configuration. CloudStack does not need to know or + care; it is entirely a local network design decision (§9.2). Because the operator can pick the + routed id at network creation, the interface name is **plannable in advance** — host policy can + exist before the network does (§6.7.2). +* **IP mobility.** Because the gateway is identical on every hypervisor, the guest's network + configuration is entirely host-independent. A VM can start, stop, and migrate anywhere in the + routing domain with no reconfiguration — its address follows it as an advertised route. +* **Fits L3-to-the-host fabrics.** Operators already running BGP or OSPF on the hypervisor get an + addressing model that matches their fabric instead of fighting it with stretched VLANs. +* **Denser subnet use.** No broadcast domain means no network or broadcast address to reserve — + every address in the subnet is usable (§6.3.1). + +## 3. Non-goals + +* Not a replacement for `NetworkMode.ROUTED` + BGP-per-network (4.20). That stays. +* No NAT, source NAT, static NAT, port forwarding, LB, or VPN in this network type. +* No VR at all — not even for DHCP/DNS/UserData. +* No DHCP, DHCPv6, or SLAAC/RA for guest addressing. +* No support for guests that cannot consume ConfigDrive (§6.5). +* **No routing-daemon management.** CloudStack does not install, configure, or monitor FRR/BIRD + (§10). +* No per-tenant VRFs, and therefore no overlapping subnets between networks (§6.3.2). +* **Never part of a VPC** — VPC already covers BGP-routed subnets with a VR; this is the different + case of a public address on the Instance itself (§6.6). + +## 4. Terminology + +| Term | Meaning | +|---|---| +| Direct routed network | The new guest network type described here | +| `ROUTED` | The new isolation method; a physical network carrying it is where direct routed networks live (§6.7) | +| Routed id | The number in the network's `routed://` broadcast URI — operator-chosen at creation, or allocated from the `ROUTED` physical network's id range (§6.7.2) | +| `brdr-` | Bridge-DirectRouted: the uplink-less per-network bridge, named from the network's routed id; created by `modifybrdr.sh` | +| Shared gateway | `169.254.0.1` / `fe80::1`, present on **every** `brdr-*` bridge on **every** host | +| Host route | `ip route replace /32 dev `, installed by `modifymacip.sh` | +| Static neighbour | `ip neigh replace lladdr dev nud permanent` | +| Routing daemon | FRR/BIRD/other, run and configured by the operator — not by CloudStack | + +## 5. Network model + +### 5.1 Addressing + +* Guest NIC: `203.0.113.55/32`, `2001:db8:1::55/128` +* Guest default route: `default via 169.254.0.1 dev eth0 onlink` and `default via fe80::1 dev eth0` +* Host guest bridge: `169.254.0.1/32` and `fe80::1/64` — identical on every host +* Host, per guest address: a /32 (or /128) route and a permanent neighbour entry, both on the + **bridge** (§9.1.2) + +### 5.2 Packet walk — guest to elsewhere + +1. The guest has no on-link neighbours (it is a /32) → everything goes to the default route. +2. The guest ARPs for `169.254.0.1` — permitted because the route is `onlink` — or ND-solicits + `fe80::1`. The host bridge answers. +3. The host routes the packet per its own routing table, out to the fabric. + +### 5.3 Packet walk — fabric to guest + +1. The fabric has learned `203.0.113.55/32` from this host via BGP/OSPF. +2. The packet arrives; the host matches `203.0.113.55/32 dev `. +3. The host does **not** need to ARP — `modifymacip.sh` installed a permanent neighbour entry + mapping the address to the guest's MAC. The frame is handed to the bridge, which forwards it to + the port where that MAC was learned. +4. Delivered. + +Static neighbour entries rather than ARP are deliberate: they remove a resolution round-trip from +VM start, make host→guest delivery independent of whether the guest answers ARP, and close off +ARP-based address takeover between guests on the same host. + +Note the address is pinned to a **MAC**, not to a port — the MAC-to-port mapping comes from ordinary +bridge FDB learning. See §12.1 for why that is still sound, and what it depends on. + +### 5.4 Packet walk — guest to guest, same host + +Between guests of **different** networks, the two addresses are on different bridges (§9.2), so the +host routes between them and there is no L2 path at all. + +Between guests of the **same** network, both host routes are local to one bridge and the host +hairpins via that bridge. Either way the traffic passes through the host's forwarding table, and +therefore through security group filtering where those are in use (§12.2). + +Remaining asymmetry: same-host traffic never reaches the fabric, so fabric-level policy does not see +it. Since security groups are optional here, same-network guest-to-guest traffic on one host may be +unfiltered — accepted for v1, see §12.3. + +### 5.5 What the network object looks like + +Like a **Shared** network, not like L2: the network has a subnet. The operator supplies the IPv4 +subnet (`cidr`, or `netmask` plus `startip`/`endip`) and/or an IPv6 prefix (`ip6cidr`) at +`createNetwork`; they are stored via the usual IP-range mechanism (`vlan` rows — `vlan_netmask`, +`ip4_range`, `ip6_cidr` in `engine/schema/src/main/java/com/cloud/dc/VlanVO.java`). + +The difference is what CloudStack does with it: the subnet is an **allocation pool that is routed to +the hypervisors**, not a broadcast domain. Guests never see the subnet mask or a subnet gateway. + +**No subnet gateway exists. DECIDED — revised 2026-09-09 (supersedes "required, and ignored").** + +`vlan_gateway` / `ip6_gateway` are meaningless for this network type: the guest's gateway is always +the shared link-local address and the whole subnet is routed to the hosts. Requiring one only burnt +an address, so guest L3 networks store **NULL** for both and `gateway`/`ip6gateway` given to +`createNetwork` are accepted for API compatibility and ignored (`NetworkServiceImpl.createGuestNetwork()`, +`ConfigurationManagerImpl.createVlanAndPublicIpRange()` — the gateway-less L3 branch). Every code +path that keyed on the gateway's presence keys on the cidr instead (§6.3). + +The one place a gateway is still typed is the **routed public IP range** for SystemVMs (§8.5): +`createVlanIpRange` shares its validation with every other public range and still requires an +IPv4 gateway outside the range (and an IPv6 gateway when `ip6cidr` is given). The guru replaces +both with the link-local gateway on the NIC, so the typed addresses are never configured anywhere. +Relaxing that stays available later; it is a validation change only. + +## 6. Design decisions + +### 6.1 How is the network type modelled? **DECIDED — `GuestType.L3`** + +`Network.GuestType` becomes `Shared, Isolated, L2, L3` +(`api/src/main/java/com/cloud/network/Network.java:45`). + +Chosen because it is self-describing and symmetrical with the existing `L2`, and because every +existing `GuestType.L2` / `Shared` branch then becomes an obvious place to decide what `L3` should +do. The cost is accepted: a new enum value touches API responses, UI and upgrade, and roughly a +dozen files special-case `L2` with a similar number special-casing `Shared`. + +Rejected alternatives, for the record: + +* **`GuestType.Shared` plus a flag or `NetworkMode` on the offering** — no new enum, and it would + inherit Shared's subnet and allocation handling for free, but it overloads `NetworkMode.ROUTED` + which already means "VR routes, no NAT". Every Shared code path would have to ask "but is it the + routed kind?", which is worse than a new value. +* **`GuestType.DirectRouted`** — explicit, but mixes a topology concept (`L2`) with a routing one in + the same enum. + +Files to review — those branching on `GuestType.L2`: + +* `server/src/main/java/com/cloud/network/NetworkServiceImpl.java` +* `server/src/main/java/com/cloud/network/NetworkModelImpl.java` +* `server/src/main/java/com/cloud/network/guru/GuestNetworkGuru.java` +* `engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java` +* `engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java` +* `server/src/main/java/com/cloud/vm/UserVmManagerImpl.java` +* `api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java` +* `engine/schema/src/main/java/com/cloud/offerings/dao/NetworkOfferingDaoImpl.java` +* `plugins/network-elements/vxlan/.../VxlanGuestNetworkGuru.java` +* `server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java` + +…plus the `GuestType.Shared` branches, which is where subnet and IP-range handling lives. + +### 6.2 Which gateway address? **DECIDED — static `169.254.0.1` / `fe80::1`** + +Fixed, not configurable. Requirements met: identical on every host, never globally routable, cannot +collide with guest address space. + +`fe80::1` is unambiguously correct — link-local next-hops are the norm in IPv6 and need no special +handling. + +`169.254.0.1` requires `onlink` on the guest route because it falls outside the guest's /32 (§8.1), +and that is fine: **the address is a /32, so the guest has to treat its gateway as on-link whatever +that gateway is.** There is no configuration of the gateway address that would avoid needing +`onlink`, so making it configurable would buy nothing — any guest that can work at all here can use +an on-link next-hop. + +An earlier draft left open whether to add a global setting for operators whose images might refuse a +link-local next-hop. Closed: no setting. Keeping it fixed preserves the "guest configuration is +completely host-independent" property by construction, and removes a knob that could only ever be set +wrong. + +Consequences: + +* §8.1's rule — emit `on-link: true` when the gateway is in `169.254.0.0/16` — is now permanently + sufficient. There is no need to generalise it to "gateway outside the address's own prefix", + because the gateway will never be anything else. +* `modifybrdr.sh` keeps its `-4` / `-6` options with these values as defaults, **deliberately**. + Nothing calls them with anything other than the defaults today, and they are not a CloudStack + setting — but they cost nothing, make the script testable in isolation, and mean a future change of + heart is a caller-side change rather than a script rewrite. They should not be removed as dead + options. + +### 6.3 Where do the addresses come from? **DECIDED** + +Users add a subnet to the network; CloudStack assigns **individual** IPv4 and IPv6 addresses out of +it — but the two families work fundamentally differently, and both mechanisms are reused unchanged: + +* IPv4 → drawn from a **pre-populated pool**: `createVlanIpRange` writes every address of the range + into `user_ip_address` (`engine/schema/src/main/java/com/cloud/network/dao/IPAddressVO.java`) and + allocation marks rows. +* IPv6 → **computed, never pooled**: the address is calculated from the subnet and the NIC's MAC + (EUI-64) at allocation time and stored only on the NIC — see §6.3.4. +* Subnet definition → `vlan` rows, as for Shared networks + +This is the mechanism Shared networks already use: `DirectNetworkGuru.allocateDirectIp()` +(`server/src/main/java/com/cloud/network/guru/DirectNetworkGuru.java:316`) → +`IpAddressManagerImpl.allocateDirectIp()` +(`server/src/main/java/com/cloud/network/IpAddressManagerImpl.java:2434`). + +**Each family is optional (added 2026-09-09): IPv6-only networks are supported.** Nothing on an +L3 network depends on IPv4 — no DHCP, no password or metadata service, and ConfigDrive carries +whatever families exist — so network creation requires only that at least one family is given and +that a given family is complete: IPv4 is a subnet with a pool (v4 addresses are drawn from one), +IPv6 is ip6cidr alone (no range — §6.3.4). Enforced by +`NetworkServiceImpl.validateL3AddressFamilies()`; the allocation chain gates on the cidrs. +IPv4-only networks work symmetrically. + +**The IPv4 subnet is given as `cidr`, or as `netmask` + `startip` (+ `endip`).** `createNetwork` +gained an optional `cidr` parameter for L3 networks (the IPv4 counterpart of `ip6cidr`; rejected +for other guest types). `NetworkServiceImpl.expandL3Ipv4Cidr()` expands it into the +netmask/startip/endip triple the rest of the flow uses: without `startip` the range defaults to +the subnet's usable addresses (network and broadcast excluded — give `startip`/`endip` explicitly +to include them, §6.3.1); an explicit `startip`/`endip` must lie inside the given cidr, and +`endip` without `startip` is rejected. `cidr` and `netmask` are mutually exclusive. With +`netmask` + `startip`, the subnet derives from the two. + +**Gateways play no part at all (revised 2026-09-09).** The Instance's gateway is always the +shared link-local address (§6.2), so declaring a subnet gateway only burnt an address the +operator then had to keep free. `gateway` and `ip6gateway` are accepted for API compatibility +but ignored and stored as NULL; start/end IPs are validated against the subnet itself. The two +allocation gates that historically keyed on the gateway (`allocateDirectIp()` for IPv4, +`setNicIp6Address()` for IPv6) gate on the L3 network's cidr instead. + +Consequences, all good: + +* No new tables, no new allocation logic, no new capacity accounting. +* Existing IP reservation, `listPublicIpAddresses`, and quota/usage machinery apply. +* Requested-IP (`ipaddress=` on `deployVirtualMachine`) works for free. + +The one thing the new guru **must** override: `allocateDirectIp()` sets the NIC's gateway and +netmask from the vlan row (lines 2459–2461). For this network type they must instead be forced to +`255.255.255.255` / `169.254.0.1` and `/128` / `fe80::1`. + +#### 6.3.1 Network and broadcast addresses are usable **DECIDED — must work** + +There is no broadcast domain, so the first and last address of a subnet are ordinary, routable, +assignable addresses. `203.0.113.0` and `203.0.113.255` in a /24 are just addresses; nothing +broadcasts to them, and every guest is a /32 behind the host's routing table. + +**They must be assignable.** The gain is proportionally largest exactly where address space is +tightest, which is the common case for this feature: + +| Subnet | Addresses | Usable today (minus network, broadcast, gateway) | Usable here | Gain | +|---|---|---|---|---| +| /29 | 8 | 5 | 8 | +60% | +| /28 | 16 | 13 | 16 | +23% | +| /27 | 32 | 29 | 32 | +10% | +| /24 | 256 | 253 | 256 | +1.2% | + +(No gateway is deducted either, since §5.5 stores none.) + +**Verified during implementation: no code change was needed.** The earlier draft assumed the +exclusion lived in the `NetUtils` CIDR helpers (`getIpRangeFromCidr()` and friends, whose +`start = (ip & netmask) + 1` / `end - 2` arithmetic does skip the two addresses) and would need +inclusive variants. Tracing the actual path an operator-supplied range takes showed those helpers +are only used where CloudStack *derives* a range from a CIDR — Isolated networks. The explicit +start/end path this network type uses (Shared-style `createVlanIpRange`) validates with +`sameSubnet()` — a plain bitmask comparison that `.0` and `.255` pass — plus start≤end and +gateway-not-in-range, and `savePublicIPRange()` then iterates the range without exclusions into +`user_ip_address`. Allocation is row-based from that table and never re-derives from the CIDR. + +**`.0` and `.255` are therefore already assignable end-to-end on this path** — when the operator +gives `startip`/`endip` explicitly. The `cidr`-only form defaults the range to the usable +addresses and so excludes them (§6.3, deliberately conservative). `test_l3_networks.py` asserts +an Instance actually receives `.0`, since this rests on tracing rather than a guarantee anyone +maintains. + +#### 6.3.2 Subnets must be unique across the routing domain **DECIDED — constraint** + +Because every direct routed network on a host shares one routing table, and all subnets are +advertised into one fabric, **subnets cannot overlap between networks**. Two tenants cannot both +use `10.0.0.0/24`. + +This follows from §9.2: networks separate tenants in the API and UI, not in address space. There is +no VRF, no per-tenant routing table, and no NAT to hide behind. Addresses must be unique and +routable within the routing domain. + +Implications: + +* Tenants cannot bring their own overlapping RFC1918 space. Operators assign from a pool they + control — public space, or private space that is unique zone-wide. +* **Validation must reject a new subnet that overlaps an existing direct routed subnet anywhere in + the zone. Required — an overlap is an address conflict, not a policy preference.** Two networks + sharing a subnet would produce duplicate /32s in one host routing table and duplicate + advertisements into the fabric, with traffic delivered to whichever Instance the host resolved + last. +* **Implemented:** the IPv6 vlan overlap check was already zone-wide (`_vlanDao.listByZone()`). + IPv4 was not — the `user_ip_address` unique key is `(public_ip_address, source_network_id)`, + i.e. per network — so the range-creation path L3 networks take + (`ConfigurationManagerImpl.createVlanAndPublicIpRange()`, the long-argument form called from + `NetworkServiceImpl.commitNetwork()`) calls the existing zone-wide `checkOverlapPublicIpRange()` + for L3 networks: an L3 range may not contain any address already present in the zone, whether + it belongs to a public range, a Shared network or another L3 network. The reverse direction is + covered by `checkZoneVlanIpOverlap()`, which now also considers gateway-less (L3) vlan rows and + treats a subnet overlap with an L3 network like one with a public range, i.e. rejects it. Shared + networks among themselves keep their historical behaviour, where the same IPv4 range in two + VLANs is legitimate. (A first version placed the L3 call in the `createVlanIpRange` API path, + which L3 networks never take; the smoke test that should have caught it swallowed its own + assertion. Both are fixed.) +* This is the main user-visible limitation of "networks separate tenants administratively only" and + must be explicit in the documentation. +* Per-tenant VRFs would lift the restriction but mean per-VRF routing tables on the host and + per-VRF sessions in the routing daemon — well beyond v1 (§15). + +#### 6.3.3 Route scale **DECIDED — out of scope** + +Each guest address is advertised as an individual /32 or /128, so a zone with 50k Instances puts 50k +routes into the fabric. This is inherent to the design: any aggregation scheme pins addresses to +hosts and breaks migration (§11), so per-address advertisement is deliberate. + +**How that scales is the operator's concern, not CloudStack's.** Fabric capacity, aggregation and +route policy are local network design — the same boundary already drawn around the routing daemon in +§10. CloudStack states no supported ceiling, because it has no way to know one: the answer depends +entirely on the hardware and topology in front of it. + +For context rather than as a commitment: route counts in the order of 100k are not usually a problem +for modern equipment. Operators running L3-to-the-host fabrics are typically already carrying host +routes at that scale. + +#### 6.3.4 IPv6 is calculated from subnet + MAC, never drawn from a stored pool **DECIDED** + +**The IPv6 address of a NIC is computed with EUI-64 from the network's IPv6 subnet and the NIC's +MAC address, at allocation time. CloudStack never stores IPv6 addresses that are *to be* allocated; +it stores only what *was* allocated, on the NIC itself (`nics.ip6_address`).** This applies to +regular Instances and to SystemVMs alike. + +Verified in the code — this is already how CloudStack behaves, on both paths this feature uses: + +* **Guest NICs** (the `DirectNetworkGuru` lineage): `IpAddressManagerImpl.allocateDirectIp()` + (`:2479`) calls `Ipv6AddressManagerImpl.setNicIp6Address()` + (`server/src/main/java/com/cloud/network/Ipv6AddressManagerImpl.java:206`), which computes + `NetUtils.EUI64Address(network.getIp6Cidr(), nic.getMacAddress())` and sets the result on the + `NicProfile`. `DirectRoutedNetworkGuru.applyDirectRoutedAddressing()` then reshapes it to `/128` + + `fe80::1`. +* **Public NICs** (SystemVMs, §8.5): `PublicNetworkGuru.getIp()` calls + `Ipv6ServiceImpl.updateNicIpv6()` (`Ipv6ServiceImpl.java:440`), which selects the range's + `ip6_cidr`, narrows it to a /64 if it is larger, and computes + `NetUtils.EUI64Address(ipv6Network, nicMacAddress)` (`:236`); the reservation is a placeholder + NIC, i.e. again a NIC row, not a pool entry. +* **`user_ipv6_address` is not an allocation pool.** No production code path inserts rows into it + (verified by search: only reads — taken-checks and counts — remain). §6.3's earlier draft listed + it as the IPv6 counterpart of `user_ip_address`; that was wrong and is corrected above. The table + stays untouched by this feature. + +Why this is the right model, stated so it survives review: + +* **Recomputable = host- and database-independent.** Subnet + MAC always yields the same address: + what SLAAC would have produced, what the guest can verify, and what migration preserves for free. +* **Uniqueness needs no coordination.** MACs are unique per network, subnets are unique zone-wide + (§6.3.2), so EUI-64 addresses collide nowhere — with no counter, no lock and no pool to maintain. +* **Nothing to pre-populate or garbage-collect.** A /64 has 2⁶⁴ addresses; a pool table for that is + a non-starter anyway. The IPv4 pool model stays where it belongs, on IPv4. +* A user-requested IPv6 (`ipaddress6=` on deploy) is rejected when it is EUI-64-shaped + (`Ipv6AddressManagerImpl.acquireGuestIpv6Address()`, `:112`), so a manual address can never + collide with a computed one. + +Consequences: + +* **The IPv6 subnet must be /64 or larger.** `NetUtils.EUI64Address()` throws for prefixes longer + than /64 (`NetUtils.java:1731`) — the interface identifier needs 64 bits. Today an L3 network's + IPv6 CIDR skips the Shared-network "/64 exactly" check, which leaves a longer prefix (e.g. /80) + to blow up at Instance deploy instead of at network creation. **Validation to add:** reject an + IPv6 CIDR with a prefix longer than /64 at `createNetwork`/`createVlanIpRange` time for L3 + networks. +* The IPv6 usable-address accounting of §6.3.1 is moot for v6 — there is no pool whose ends could + be excluded; every EUI-64 result inside the subnet is valid. + +### 6.4 Service/provider matrix for the offering **DECIDED** + +`ConfigDriveNetworkElement` advertises `UserData`, `Dhcp`, `Dns` +(`server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java:203`). This type uses +two of the three. + +| Service | Provider | Note | +|---|---|---| +| `UserData` | `ConfigDrive` | **mandatory** | +| `Dns` | `ConfigDrive` | optional but **strongly recommended** — the only way a guest learns its resolvers unless its template already has them (§6.5) | +| `SecurityGroup` | `SecurityGroupProvider` | optional (§12.2) | +| `Dhcp` | — | **not supported, and not needed** | +| `SourceNat`, `StaticNat`, `PortForwarding`, `Lb`, `Firewall`, `Vpn`, `NetworkACL`, `Gateway` | — | not supported | + +* `specifyIpRanges` → `true` (the operator supplies the subnet) +* `specifyVlan` → the operator's choice, exactly as for Shared offerings: `true` means the routed + id is given at network creation (via the existing `vlan` parameter), `false` means CloudStack + allocates one from the `ROUTED` physical network's id range (§6.7.2) +* `NetworkMode` → not applicable; reject `NATTED`/`ROUTED` for this guest type. (The *isolation + method* `ROUTED` (§6.7) is a different axis: `NetworkMode.ROUTED` says how a VR routes, the + isolation method says which physical network these VR-less networks live on.) + +**Why DNS is recommended rather than mandatory.** There is no VR and no resolver on the host, so +`network_data.json` `services` is the only channel by which CloudStack can tell a guest its DNS +servers (§8.3). The `Dns` service on the offering is what makes the offering *declare* that it +delivers resolvers; a template that already carries its own is unusual but legitimate and the +operator's call (§6.5). Note, as verified in review, that `ConfigDriveBuilder` writes the +`services` entries whenever the NIC profile carries DNS servers, which the allocation path sets +from the network or the zone regardless of the `Dns` service (`getServicesJsonArrayForNic()` never +consults the service list). In practice an L3 Instance therefore receives resolvers whenever the +network or zone has any, with or without `Dns` on the offering. That is pre-existing ConfigDrive +behaviour shared with every other network type and is left alone here. + +**Why DHCP is not merely unsupported but unnecessary.** ConfigDrive delivers the address, netmask, +gateway and routes directly (§8.1). DHCP would have nothing left to hand out, and offering it would +reintroduce exactly the L2 broadcast dependency this network type removes. + +Validation to add: + +* reject the offering unless `UserData` is provided by `ConfigDrive`; `Dns` is permitted but not + required (§6.5) +* reject `Dhcp` outright for this guest type +* `NetworkServiceImpl.java:657` currently rejects DNS on L2 networks. This type *requires* DNS, so + that branch must distinguish L2 from L3 rather than treating them alike. + +### 6.5 ConfigDrive is mandatory, DNS is not **DECIDED** + +**`UserData` via ConfigDrive is mandatory. `Dns` is optional but strongly recommended.** + +ConfigDrive itself cannot be optional: with no DHCP and no RA, it is the only channel that carries +the address, netmask, gateway and routes. A guest that ignores it comes up with no addresses at all, +and nothing in CloudStack reports an error. + +DNS is a different matter. A template may already have resolvers baked in, or be configured by +whatever provisions it afterwards. That is unusual, but it is the operator's call, not something the +network type needs to enforce — so an offering may omit `Dns`. The documentation should recommend +`UserData` + `Dns` together, since omitting DNS leaves an Instance with connectivity but no name +resolution unless its template handles it. + +**This had a hard prerequisite — see §8.2, implemented.** `ConfigDriveBuilder.needForGeneratingNetworkData()` +used to write network data only when the network supports `Dhcp` **or** `Dns`. Since this type +never has `Dhcp`, an offering without `Dns` would have produced an **empty `network_data.json` and +an Instance with no addressing at all** — a far worse outcome than missing resolvers. Network data +is now always generated when a direct routed NIC is present. + +The IPv6 zone-DNS requirement that `createNetwork` applies to IPv6 Shared networks ("the zone has +no IPv6 DNS") is not applied to L3 networks, for the same reason: DNS is optional here. + +**No warning is raised when a template ignores ConfigDrive. DECIDED.** Whether cloud-init is present +and configured inside the guest is the operator's responsibility, not something CloudStack should +police. There is no reliable way to detect it from outside the Instance in any case, so any check +would be a guess presented as a fact. Documented as a requirement of the network type; not enforced, +not warned about. + +### 6.6 VPC support **DECIDED — never** + +**Not out of scope for v1; out of scope permanently.** Standalone networks only. + +The purpose of this network type is to route a public IPv4 and IPv6 address directly to an Instance. +A VPC is the opposite proposition: a private, self-contained address space with a VR, tiers holding +their own CIDRs, and ACLs between them. Every one of those is something this type deliberately +removes, so "direct routed inside a VPC" would not be a reduced VPC — it would be a contradiction. + +The case that *sounds* like it overlaps is already covered: **VPC supports BGP routing with subnets +today** (`NetworkMode.ROUTED`, 4.20). An operator who wants tenant subnets advertised into the fabric +with a VR in the path should use that. This feature addresses the different case where the Instance +itself holds the public address and there is no VR at all. + +Keeping the two apart is deliberate. They are not two settings of one feature, and treating them as +such would make both harder to reason about. + +### 6.7 What makes a network direct routed **DECIDED — revised 2026-09-04** + +**The offering's guest type (`L3`) on a physical network whose isolation method is `ROUTED`.** +The network operator is in the lead: direct routed is a property of the zone's fabric — hosts that +route, a routing daemon, an id plan for the bridges — so it is declared where fabric properties are +declared, on a **dedicated physical network**, not inferred from an offering that any admin can +create. Every direct routed network then carries a broadcast domain of the new type +**`routed://`** (`BroadcastDomainType.Routed`, scheme `routed`), and that id names the bridge: +`routed://5828` → `brdr-5828` (§9.2). + +* **Guru selection follows the standard contract.** `canHandle()` tests the zone type, + `isMyTrafficType()`, `offering.getGuestType() == GuestType.L3` **and** `isMyIsolationMethod()`, + exactly like its siblings (`DirectNetworkGuru.java:147`, `VxlanGuestNetworkGuru.java:56`). The + guru registers `new IsolationMethod("ROUTED")`. +* **A dedicated physical network.** The operator creates a guest physical network with isolation + method `ROUTED` (zone wizard or `createPhysicalNetwork isolationmethods=ROUTED`), gives it an id + range (the physical network's existing VLAN/VNI range field — here it is the routed-id pool), and + steers L3 offerings to it with tags, as for any other physical network. The traffic label is + irrelevant for these networks — the bridges have no uplink (§9.3) — but the physical network is + where the id range, the tags and the operational statement "this zone routes to the host" live. +* **The isolation method is named `ROUTED`, deliberately.** It shares the word with + `NetworkMode.ROUTED` (4.20) but sits on a different axis: the network mode describes how a + Virtual Router routes an Isolated/VPC network; the isolation method marks the physical network + that carries these VR-less, hypervisor-routed networks. The documentation must state the + distinction once, plainly. + +**This reverses the earlier decision** ("the offering, and nothing else" — no isolation method, no +broadcast domain, bridge named from `networks.id`). What changed the call: + +* **Operator opt-in became explicit.** Previously any admin creating an L3 offering implicitly + asserted that every host in the zone runs a routing daemon. A physical network with `ROUTED` makes + that a deliberate, zone-level act by the network operator, and an L3 network simply cannot be + designed in a zone whose operator has not made it. +* **The id became operator property.** `networks.id` was unique but uncontrollable: bridge names + could only be discovered after creation, never planned. With `routed://` the id is chosen by + the operator or drawn from a range the operator sized — host routing policy per `brdr-` can be + written before the network exists (§6.7.2). +* **The agent gets an explicit signal.** `BroadcastDomainType.Routed` on the `NicTO` replaces + inferring "this is direct routed" from the /32-plus-link-local-gateway address form (§9.1.3). +* **No new plumbing after all.** The earlier draft rejected this model as "an allocation mechanism, + a range to configure, and a choice to make". Working through the code showed all three already + exist for Shared networks — the `vlan` parameter, the physical network's vnet range and + `_dcDao.allocateVnet()`, and `specifyVlan` — and are reused verbatim (§6.7.2). What remains new is + one enum value and one registered isolation method. + +Offering creation rejects `Dhcp` for this guest type (§6.4), so a direct routed offering cannot be +built with DHCP in the first place. + +#### 6.7.1 Changing a network's offering afterwards **ACCEPTED** + +`NetworkServiceImpl.canUpgrade()` (`server/src/main/java/com/cloud/network/NetworkServiceImpl.java:4193`) +gates `updateNetwork`'s offering change on SecurityGroup parity, tag equality, `specifyVlan` +equality, `NetworkMode` equality, and `canMoveToPhysicalNetwork()`. It does **not** compare guest +type, so an administrator can in principle move a network onto an offering of a different type — for +example one without ConfigDrive, leaving Instances with no way to get their addressing. + +**Accepted, not guarded.** This is an administrative action with an obvious cause and effect; adding +a special case to `canUpgrade()` for this type is not worth the complexity. Worth a line in the +documentation, nothing more. + +One incidental note from reading that method: the **SecurityGroup parity check** means security +groups cannot be toggled on a live network by swapping offerings — enabling them is a choice made +when the network is created. + +#### 6.7.2 Where the routed id comes from **DECIDED — Shared-network mechanics, reused** + +The id lifecycle is exactly the Shared network's VLAN lifecycle, with a different URI scheme. No +new allocation mechanism is introduced: + +* **`specifyVlan=true` on the offering** — the operator passes the id at network creation through + the existing `vlan` parameter: `createNetwork ... vlan=5828` → `broadcast_uri=routed://5828` → + `brdr-5828`. This is the flagship path: the operator plans the id space and can pre-provision + per-bridge routing policy on the hosts. +* **`specifyVlan=false`** — CloudStack allocates a free id from the `ROUTED` physical network's + range at creation (`_dcDao.allocateVnet()`, the same call Shared networks without `specifyVlan` + use — `NetworkServiceImpl.commitNetwork()`), and releases it when the network is deleted (the + existing release path in `NetworkOrchestrator`). + +The value must be numeric — `routed://` carries a number, nothing else — and is enforced: +`Networks.BroadcastDomainType.getRoutedId()` accepts a positive integer of at most ten digits +without leading zeros (so `brdr-` fits the 15-character interface-name limit and the bridge +MAC derives from five bytes, §9.2), given bare or as `routed://`, and canonicalises it. +`NetworkServiceImpl.canonicalizeRoutedId()` applies it to the `vlan` parameter of an L3 network, +`ConfigurationManagerImpl.canonicalizeRoutedRangeId()` to a routed public range (§8.5). Without +this, the generic URI fallback accepted anything parseable (`routed://abc`, `routed://0534`), +which bypassed the string-compare uniqueness checks. The existing Shared-network checks then +apply unchanged: an operator-specified id must not fall inside the dynamic range +(`_dcDao.findVnet()`), and must not collide with another network's broadcast URI. + +**Uniqueness must be zone-wide, not per physical network.** Bridge names are global on a host, so +two networks with routed id 5828 anywhere in the zone would share `brdr-5828` and merge their L2 +domains. The existing zone-wide URI overlap check +(`_networksDao.listByZoneAndUriAndGuestType()`) covers guest-vs-guest; with a single `ROUTED` +physical network per zone — the expected deployment — it is equivalent to the per-physnet check +anyway. **Guest networks and public ranges (§8.5) share the same id space** and are guarded in +both directions: creating a guest network whose routed id a public range already carries is +rejected (`NetworkOrchestrator`, `_vlanDao.findByZoneAndVlanId()`), and so is creating a public +range whose id a guest network holds — or that lies inside the routed-id range of any `ROUTED` +physical network in the zone, since auto-allocated networks draw from that range and a later +random draw would otherwise collide with the public range +(`ConfigurationManagerImpl.canonicalizeRoutedRangeId()`). + +Either way the URI is set at creation and **stable for the network's life**: the bridge name never +changes, which is what makes it something host policy can reference. + +### 6.8 Hypervisor support + +**KVM only for v1.** The host must program routes and neighbour entries and run a routing daemon; +that is only realistic where the host OS is ours to configure. Other hypervisors: reject at network +creation with a clear error. + +## 7. API and data model changes + +### 7.1 API + +* `createPhysicalNetwork` / `updatePhysicalNetwork` — accept `ROUTED` as an isolation method; the + physical network's VLAN/VNI range field doubles as the routed-id pool (§6.7.2). UI: add `ROUTED` + to the isolation method choices (zone wizard and physical network form). +* `createNetworkOffering` — accept the new `guestiptype`; validate the service matrix (§6.4); + `specifyVlan` is a free choice (§6.7.2). +* `createNetwork` — accept the subnet as for Shared networks; the existing `vlan` parameter carries + the routed id when the offering has `specifyVlan=true` (§6.7.2); `createVlanIpRange` gateway + handling per §5.5; zone-wide overlap validation per §6.3.2. +* `listNetworks` / `NetworkResponse` — expose the type and the `broadcasturi` (`routed://5828`), + which is how a user reads off the bridge name. The network's CIDR is meaningful (it is the + pool) and should be shown; its gateway is not. +* `listNics` / `NicResponse` — report the /32 and /128 plus the shared gateway. `netmask` is + already a dotted quad, so `255.255.255.255` needs no schema change. +* `listPublicIpAddresses` — works as for Shared networks. +* New APIs: **none.** + +### 7.2 Database + +* `network_offerings.guest_type` — new enum value (`L3`). +* `networks.broadcast_domain_type` — new enum value (`Routed`); `broadcast_uri` holds + `routed://`. Both columns are strings, so no schema change. +* `physical_network_isolation_methods` — new value `ROUTED`; a plain string, no schema change. +* `op_dc_vnet_alloc` — reused unchanged as the routed-id pool for `specifyVlan=false` (§6.7.2). +* `nics` — all needed columns exist: `ip4_address`, `netmask`, `gateway`, `ip6_address`, + `ip6_cidr`, `ip6_gateway` (`engine/schema/src/main/java/com/cloud/vm/NicVO.java:55-108`). +* `user_ip_address`, `vlan` — reused unchanged. `user_ipv6_address` stays untouched: IPv6 is + computed from subnet + MAC and stored only on the NIC (§6.3.4). +* Upgrade: new enum values only. No data migration. + +### 7.3 New components + +* **Network guru — a subclass of `DirectNetworkGuru`. DECIDED.** `DirectNetworkGuru` already + implements "operator defines a subnet, CloudStack assigns individual v4 and v6 addresses", which + is exactly the allocation behaviour wanted, so the allocate/release lifecycle is inherited rather + than duplicated. The subclass registers `IsolationMethod("ROUTED")` and overrides: + * `canHandle()` — accept `GuestType.L3` on a physical network with isolation method `ROUTED` + (`isMyIsolationMethod()`, as its siblings do — §6.7) + * the `NicProfile` after allocation — force `255.255.255.255` / `169.254.0.1` and `/128` / + `fe80::1` over the vlan row's values (`IpAddressManagerImpl.allocateDirectIp()` lines 2459–2461, + §6.3) + * `design()` — broadcast domain type `Routed`, broadcast URI `routed://` from the + operator-specified or allocated id (§6.7.2, §9.2.1) + + The known cost of subclassing is inheriting `DirectNetworkGuru`'s Shared-network assumptions. + Accepted: the alternative is duplicating the address lifecycle, which is the part most likely to + drift and the part where bugs are least visible. **TODO:** during implementation, note any + inherited behaviour that only makes sense for `GuestType.Shared` and override it explicitly rather + than letting it apply by accident. +* **Network element** — none new. `ConfigDriveNetworkElement` covers UserData/DNS; the security + group element covers filtering; host routes ride on NIC plug/unplug (§9), not on element + `implement()`. + +## 8. Guest configuration via ConfigDrive + +### 8.1 `on-link` for the link-local gateway **DECIDED** + +`ConfigDriveBuilder.getNetworksJsonArrayForNic()` +(`engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java:365`) +today emits OpenStack **network_data.json v1**: + +```json +{"id":"eth0","ip_address":"...","netmask":"...","link":"eth0","type":"ipv4", + "routes":[{"gateway":"...","netmask":"0.0.0.0","network":"0.0.0.0"}]} +``` + +A /32 address with a gateway outside its own subnet cannot be expressed here — v1 has no way to +mark a next-hop as on-link, and the guest kernel will reject the resulting config with +`Nexthop has invalid gateway`. + +**Resolved — revised 2026-09-08 after testing on Ubuntu 26.04 / cloud-init 26.1: CloudStack +emits a network-level `gateway` key for direct routed IPv4 networks.** + +The earlier resolution ("no on-link plumbing is needed; cloud-init detects a link-local gateway +itself") turned out to be wrong in a subtle way. cloud-init *does* have on-link detection +(`should_add_gateway_onlink_flag()`, gateway outside the interface subnet), but only on one path: + +1. It is applied **only to a subnet-level `gateway` key** — in the netplan renderer since 23.1 + (`network/netplan: add gateways as on-link when necessary`, LP #2000596) and the networkd + renderer since 24.2. It is **never applied to entries of the `routes` list**, in any renderer. +2. cloud-init's OpenStack converter (`cloudinit/sources/helpers/openstack.py`, + `convert_net_json()`) keeps `network_data.json` routes as subnet `routes` — a `0.0.0.0/0` + route is never promoted to the subnet `gateway`. So the v1 default-route emission always took + the flagless path, and the guest kernel rejected the IPv4 default route + (`Nexthop has invalid gateway`) while IPv6 via `fe80::1` (on-link by definition) worked. +3. The `sysconfig`, `network_manager` and `eni` renderers have no on-link logic at all. + +The fix uses the one path that works: `convert_net_json()` whitelists `gateway` as a key on the +network object (`valid_keys["subnet"]`). `ConfigDriveBuilder.getNetworksJsonArrayForNic()` +therefore emits, **for direct routed IPv4 networks only**, a network-level +`"gateway": "169.254.0.1"` *instead of* the default-route entry — instead of, not alongside: +both together would render two default routes, one of them still flagless. cloud-init carries +the key into the v1 subnet's `gateway`, and the renderer's gateway path adds `on-link: true`. +Every other network type keeps the historical `routes` emission byte-for-byte; the IPv6 route +also stays in `routes` form, since a link-local next hop needs no flag. + +The guest-side requirement is thus **cloud-init >= 23.1 with the netplan renderer, or >= 24.2 +with networkd**. Guests on the sysconfig/NetworkManager/eni renderers still lack on-link support +regardless of what CloudStack emits — an upstream cloud-init contribution (apply +`should_add_gateway_onlink_flag()` in the routes loop and the remaining renderers, the same +one-liner as LP #2000596) is the path to closing that, tracked in §15. + +Target guest config: + +```yaml +version: 2 +ethernets: + eth0: + match: {macaddress: "02:00:...:5a"} + addresses: [203.0.113.55/32, "2001:db8:1::55/128"] + routes: + - to: default + via: 169.254.0.1 + on-link: true + - to: default + via: "fe80::1" + nameservers: + addresses: [...] +``` + +The failing behaviour was verified on Ubuntu 26.04 (cloud-init 26.1, netplan renderer): the +routes-list form boots with IPv6 up and no IPv4 default route. The `gateway`-key emission was +verified on the same image in the September 2026 lab: the IPv4 default route is installed with +`on-link`, dual-stack and IPv6-only. + +### 8.2 network_data generation was gated on DHCP or DNS **IMPLEMENTED** + +```java +static boolean needForGeneratingNetworkData(Map> supportedServices) { + return supportedServices.values().stream() + .anyMatch(services -> services.contains(Network.Service.Dhcp) + || services.contains(Network.Service.Dns)); +} +``` +(`engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java`, +called from `writeNetworkData()`.) + +If neither service is supported, `writeNetworkData()` wrote an empty `{}` and **the guest received +no network configuration whatsoever** — no address, no netmask, no gateway, no routes. + +That gate was wrong for this network type. It equates "does this network have DHCP or DNS?" with +"does this NIC need its addressing written into ConfigDrive?", which held while ConfigDrive was a +supplement to a VR but does not hold when ConfigDrive is the *only* channel. This type never has +`Dhcp`, and §6.5 makes `Dns` optional, so the two conditions can both be false while the NIC still +very much needs its /32 written. + +**Implemented:** `writeNetworkData()` generates network data whenever the historical gate is met +*or* any NIC of the Instance is direct routed (recognised by its host-route address form, +`isDirectRoutedNic()`). When a direct routed NIC forces generation, **all** NICs of the Instance +are written, matching the historical all-or-nothing semantics: an explicit network config that +listed only the L3 NIC would stop cloud-init from configuring the Instance's other interfaces. +The Javadoc on `writeNetworkData()` records the coupling, because it is invisible from the +offering side. + +### 8.3 DNS **DECIDED — per network, falling back to the zone** + +DNS servers reach the guest through `network_data.json` `services` +(`getServicesJsonArrayForNic`). No resolver on the host, no VR. + +Resolvers come from the network when set, and from the zone otherwise — so an operator configures +DNS once per zone and overrides it only on the networks that need something different. + +**This already works and needs no new code.** `NetworkModelImpl.getNetworkIp4Dns()` and +`getNetworkIp6Dns()` (`server/src/main/java/com/cloud/network/NetworkModelImpl.java:3023` and +`:3037`) implement exactly that precedence: network `dns1`/`dns2` if set, then the VPC, then the +zone. The VPC branch is simply never reached here (§6.6). The `networks` table already carries +`dns1`, `dns2`, `ip6_dns1`, `ip6_dns2`, and `createNetwork`/`updateNetwork` already expose them. + +Note the interaction with §6.5: DNS is optional on the offering, but `ConfigDriveBuilder` writes +the `services` entries from the NIC profile, which carries the network's or zone's resolvers +whether or not the offering lists `Dns` (§6.4). Omitting `Dns` therefore does not suppress them. + +### 8.4 Metadata service **DECIDED — none in v1** + +With no VR there is no `data-server` at the gateway and no link-local metadata endpoint. **The +ConfigDrive ISO is the only source of metadata and user data**, and there is no +`169.254.169.254`-style HTTP endpoint. + +**Note this explicitly for operators**, because it is a real behavioural difference from other +network types rather than an omission. Tooling that expects to `curl 169.254.169.254` — cloud-init +in some configurations, Kubernetes cloud providers, various agents — will not work unmodified. A +template that reads its metadata from ConfigDrive is fine; one that assumes the HTTP endpoint is not. + +A host-side responder on the `brdr-*` bridge is entirely feasible later: the gateway address is +already there, the host already routes for the guest, and the data is already assembled for the ISO. +It is deferred rather than ruled out — **a candidate for v2** (§15). + +### 8.5 SystemVMs: boot args, not ConfigDrive **REQUIRED CHANGES — dual-stack in v1** + +Console proxy and secondary storage VM are still needed in a direct routed zone, and their public +interface is directly routed like everything else — a `/32` + `169.254.0.1` on-link gateway **and** +a `/128` + `fe80::1`. **IPv6 for the SystemVMs must work from the start**; it is not deferred. + +SystemVMs do not consume ConfigDrive: their addressing is passed down from the hypervisor as boot +arguments, parsed by `systemvm/debian/opt/cloud/bin/setup/common.sh` and applied by +`setup_common()`. For both VM types the call is `setup_common eth0 eth1 eth2` (`init.sh:166`), so +`eth2` is the public interface **and** the default-gateway device. + +Walking the chain end to end, four gaps stand between the host-route NIC form and a working +systemvm: + +1. **The IPv4 default route needs `onlink`.** The values already flow — both builders pass + `eth2ip`, `eth2mask` and `gateway` straight from the `NicProfile` + (`ConsoleProxyManagerImpl.finalizeVirtualMachineProfile()`, `SecondaryStorageManagerImpl` + likewise), so a NIC stamped in host-route form arrives correctly. But `setup_common()` then runs + `ip route add default via $GW dev $gwdev` (`common.sh:399`), which the kernel rejects with + `Nexthop has invalid gateway` when `$GW` lies outside the /32. Append `onlink` when `$GW` falls + in `169.254.0.0/16` — the same trigger rule §8.1 leans on in cloud-init, applied by hand here + because a systemvm has no cloud-init. The `/etc/network/interfaces` stanza itself (address + + `netmask 255.255.255.255`, no gateway line) is fine as is, and the VMware ping-the-gateway + workaround right after the route works once the on-link route exists. +2. **IPv6 boot args are never sent for CPVM/SSVM.** Only the VR's builder emits them + (`VirtualNetworkApplianceManagerImpl.java:1935–1946`: `ethip6`, `ethip6prelen`, + `ip6gateway`). The CPVM and SSVM builders emit IPv4 only and must add the same three appends for + NICs that carry IPv6, from values the guru already stamped — prefix length via + `NetUtils.getIp6CidrSize(nic.getIPv6Cidr())` (→ 128), `ip6gateway` from the default NIC. The + consumer side already exists: `common.sh` parses `eth0ip6`/`eth2ip6`(+`prelen`) and + `ip6gateway` → `IP6GW` today. +3. **The systemvm never installs an IPv6 default route.** `IP6GW` is parsed and then consumed + nowhere; `setup_interface_ipv6()` (`common.sh:139`) writes only the address and prefix length + and leans on `accept_ra 1` — and on a direct routed bridge **no RA ever arrives** (`accept_ra=0` + on the bridge, §9.2, and nothing sends them). Add next to the v4 default in `setup_common()`: + `ip -6 route replace default via $IP6GW dev $gwdev` whenever `IP6GW` is set. `fe80::1` is + link-local, so `dev` is mandatory and no on-link handling exists or is needed — a link-local + next-hop is on-link by definition. *Note this gap predates the feature:* a classic VLAN public + network with IPv6 also leaves CPVM/SSVM without a v6 default unless a fabric router happens to + send RAs. The fix is useful independently of direct routed and should not be gated on it. +4. **The public NIC must be stamped and plugged like a guest's.** `PublicNetworkGuru.getIp()` + (`PublicNetworkGuru.java:146–156`) sets gateway/netmask from the public range's vlan row and + hard-codes a `Vlan`/`Vxlan` broadcast URI on the NIC. For a direct routed public range the NIC + must instead receive the host-route form (as `DirectRoutedNetworkGuru.applyDirectRoutedAddressing()` + does for guests) plus `BroadcastDomainType.Routed` and a `routed://` URI — which is also + exactly what steers `BridgeVifDriver`'s Public branch into the brdr-bridge + `modifymacip.sh` + path instead of the public bridge (today that handling sits only in the Guest branch). **IPv6 + is computed directly in the guru** — `NetUtils.EUI64Address(range ip6_cidr, NIC MAC)` per + §6.3.4, then the /128 + `fe80::1` form. Deliberately *not* via `ipv6Service.updateNicIpv6()`: + auditing that path showed it is gated on the public network offering's internet protocol + (never set on the system public offering, so a no-op in practice) and reserves through **one + placeholder NIC per network** — on the shared Public network every SystemVM would receive the + *same* address. Neither the gate nor the reservation decides anything here: the MAC already + makes the address unique. Mechanism: the public IP range's vlan row carries `routed://` as + its tag, which `getIp()` copies into the broadcast URI; the zone's public network carries one + routed id (and thus one `brdr-`) of its own, and systemvm /32s and /128s are advertised by + the host's routing daemon exactly like guest addresses. + +Minor, noted for completeness: `setup_interface_ipv6()` writes `accept_ra 1` — harmless on an +RA-less bridge, but the static default must not depend on it (and may be set to 0 for direct +routed interfaces later); SSVM's apache vhost binds `$ETH2_IP` (IPv4) — pre-existing; the SSVM +serves its HTTP endpoints on IPv4, while its own outbound traffic and reachability for +management are dual-stack. + +## 9. Hypervisor (KVM) implementation + +### 9.1 The agent's entire job + +Per guest address, on NIC plug: install a static neighbour entry and a host route. On unplug / VM +stop / migrate-away: remove them. **That is the whole contract.** Nothing else on the host is +CloudStack's concern. + +#### 9.1.1 This already exists — reuse `modifymacip.sh` **DECIDED** + +The mechanism was added to `main` by `4816e059383` ("KVM: add configurable MAC/IP script hook for +static ARP/NDP and routes", PR #13495, 2026-07-10) for the VXLAN/EVPN static MAC-IP work. It does +almost exactly what this design needs: + +* `scripts/vm/network/vnet/modifymacip.sh` — `-o add -b -m [-4 ]... [-6 ]...` + runs `ip neigh replace lladdr dev nud permanent` and + `ip route replace /32 dev `, and the `-6` equivalents with `/128`. `-o delete -b + -m ` discovers the addresses to remove by querying the neighbour table for that MAC, so no + state file is needed. +* `BridgeVifDriver.executeMacIpScript()` + (`plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java:437` + for add, `:418` for delete) is already wired into NIC plug (`:291`) and unplug (`:298`). +* It also passes the MAC-derived IPv6 link-local automatically + (`NetUtils.ipv6LinkLocal(mac)`), handles secondary IPs, and sets + `net.ipv6.conf..disable_ipv6=0` before installing NDP entries. +* Gated by the agent property `vm.network.macip.static` + (`AgentProperties.VM_NETWORK_MACIP_STATIC`, default `false`). + +So §9.1 is largely an integration exercise rather than new code. Four gaps to close. + +#### 9.1.2 Bridge, not tap **DECIDED** + +Routes and neighbour entries go on the **bridge**, exactly as `modifymacip.sh` and the existing +`BridgeVifDriver` hook already do (`intf.getBrName()` is passed at `:291`). **No change to the +script or the hook for this.** The same code serves both the EVPN case and this one; there is no +reason to write it twice. + +Consequence to be aware of: the host route pins each address to the bridge, and the static neighbour +entry pins it to a **MAC** — but the MAC-to-port mapping comes from ordinary bridge FDB learning, so +the routing table alone is not an anti-spoofing boundary. §12.1 sets out why the design is still +sound (short version: the per-network bridge is the isolation boundary, so MAC-to-port pinning only +matters within one account's own network), and §12.3 for what remains exposed. + +Per-tap routes were considered and rejected for v1. Recorded for completeness in case the anti-spoof +story needs tightening later: `-b` is passed verbatim as the `dev` argument to every `ip` command, +and the delete path's `ip neigh show dev ` works on a tap too, so targeting a tap would need no +script change at all — only a rename of `-b` to something less misleading. + +#### 9.1.3 Gating is per NIC, on the broadcast type **DECIDED — revised 2026-09-04** + +`vm.network.macip.static` is resolved once in `configure()` (`BridgeVifDriver.java:87`) and is +all-or-nothing for the host. A host runs direct routed guests *and* ordinary bridged guests side by +side, so the behaviour is decided **per NIC** — not from a host property. + +**The selector is now explicit.** With §6.7 giving every direct routed network a +`routed://` broadcast domain, the `NicTO` carries `broadcastType == BroadcastDomainType.Routed` +and the URI itself — populated for every NIC by `HypervisorGuruBase.toNicTO()`. That is the test: +one decision, used twice. It picks the bridge (`brdr-` from the URI's value, §9.2.1) **and** +gates the MAC/IP hook. They are not separate decisions — if the driver has chosen a `brdr-` bridge, +the hook applies. + +**The earlier inferred contract is superseded.** Before the revision these NICs were +`BroadcastDomainType.Native` — indistinguishable from other untagged cases — so the agent inferred +"direct routed" from the address form no other network type produces: netmask `255.255.255.255` +(and/or a `/128`) with a gateway in `169.254.0.0/16`. That inference was flagged at the time as an +implicit contract a future change could trip over; the broadcast type closes it. The address form +remains meaningful where it is genuinely about addressing — cloud-init's `on-link` handling (§8.1) +keys on the link-local gateway, and `ConfigDriveBuilder` recognises the host-route shape — but the +agent's *routing-behaviour* decisions (bridge choice, MAC/IP hook, `--directrouted` to +`security_group.py`) all key on `BroadcastDomainType.Routed`. + +The agent property `vm.network.macip.static` stays as an independent host-wide opt-in for the EVPN +use case and is unaffected. + +#### 9.1.4 Silent failures are kept **DECIDED — accepted** + +Both `executeMacIpScript()` overloads catch everything and only log, deliberately — "managing host +neighbour/route entries is best-effort and must never break VM lifecycle operations" +(`BridgeVifDriver.java:432`). + +**That behaviour is kept unchanged.** No new failure handling for this network type in v1, which +also means the existing EVPN path cannot be regressed by this feature. + +The consequence, stated so it is not a surprise later: if the route or neighbour install fails, the +Instance starts normally and appears healthy, but has **no connectivity at all**, and nothing in +CloudStack reports why. Diagnosis means looking at the agent log for the warning from +`executeMacIpScript()`, or checking `ip route` / `ip neigh` on the host. + +Making it fatal to the NIC plug, or raising an alert, remains available later (§15) and would be a +small change — the call sites already distinguish success from failure, they simply do not act on it. + +#### 9.1.5 Secondary IPs **IMPLEMENTED** + +CloudStack lets an Instance hold secondary IPv4 and IPv6 addresses on a NIC, and they need the +same treatment as the primary: a host route and a static neighbour entry, or the address is not +reachable. + +**At Instance start this already worked.** `modifymacip.sh` accepts repeated `-4`/`-6`, and +`BridgeVifDriver` passes `nic.getNicSecIps()` alongside the primary addresses, so every address the +NIC holds is installed on plug. Unplug removes them all by MAC. + +**Adding or removing one on a *running* Instance did not, and was fixed.** That path +(`NetworkRulesVmSecondaryIpCommand`) only ever updated ipsets and ebtables, and the management +server only sent it when security groups were in play — three separate gates +(`SecurityGroupManagerImpl`: Instance in no security group, network without the SG service; +`NetworkServiceImpl.configureNicSecondaryIp` / `RemoveIpFromVmNicCmd`: zone without SG). On a +Direct Routed network with security groups disabled, none of them fired, so a newly added secondary +IP stayed dark until the Instance was restarted — and a removed one kept being routed and +advertised. + +The command now carries `directRouted` and `applySecurityGroupRules`; the agent installs or removes +the host route and neighbour entry for the address whenever the former is set, independently of +`canBridgeFirewall`, and skips the security group script when the latter is not. `modifymacip.sh` +gained per-address delete (`-o delete` with `-4`/`-6` removes just those; without them it keeps its +delete-everything-for-this-MAC behaviour, which is what unplug uses). + +**The allocation itself needed enabling too (found in review).** +`NetworkServiceImpl.allocateSecondaryGuestIP()` handled only Isolated and Shared networks and +logged "not supported" for anything else, so none of the above was reachable for L3. L3 now takes +the Shared branch: IPv4 secondaries come from the network's pool, an IPv6 secondary is a +user-chosen address inside `ip6cidr`, validated as for Shared networks. + +**The ipset-based dispatch pays off here.** Because to-Instance traffic on L3 matches +`--match-set dst` (§12.2), and secondary IPs are added to that same ipset, security group +dispatch covers a new secondary IP with no rule changes at all. + +#### 9.1.6 Not a gap: the shared gateway addresses + +`modifymacip.sh` does not configure `169.254.0.1` / `fe80::1` — that is `modifybrdr.sh`'s job when it +creates the network's bridge (§9.2). The two scripts have a clean split: `modifybrdr.sh` owns the +bridge and its gateway addresses, `modifymacip.sh` owns per-guest routes and neighbour entries on it. + +Minor robustness note: because the delete path derives addresses from the neighbour table, a route +leaks if its neighbour entry has already been flushed. Reconciliation (§9.6) covers this. + +### 9.2 One bridge per network **DECIDED** + +**Each direct routed network gets its own bridge on every hypervisor that runs one of its +Instances**, named `brdr-` (Bridge-DirectRouted) — for example `brdr-5828`. + +The `` is the network's **routed id** — the value of its `routed://` broadcast domain, +operator-chosen or allocated from the `ROUTED` physical network's range at creation, and stable for +the network's life (§6.7.2, §9.2.1). + +The bridge is created and removed by a new script, +`scripts/vm/network/vnet/modifybrdr.sh`, modelled on `modifyvxlan.sh`: + +``` +modifybrdr.sh -o add -n [-4 ] [-6 ] → prints the bridge name +modifybrdr.sh -o delete -b → notmine | kept | deleted +modifybrdr.sh -o query -b → mine | notmine +``` + +Every operation prints exactly one token on stdout; all diagnostics go to stderr (the agent's +`Script` runner merges the two streams, so the agent reads the **last** non-blank line and +validates it before use — an earlier version read the first line, which a stray `sysctl` warning +could turn into a bogus bridge name in the domain XML). Exit code 0 means the token is valid, 1 +that the operation failed (every `ip` and `sysctl` step is checked), 2 bad arguments. Inputs are +validated: the id is a positive integer of at most ten digits (§6.7.2), the bridge name must match +the script's own naming or the answer is `notmine` before anything is done with it, and the +gateway addresses must be well-formed. + +On `add` it creates the bridge if absent (STP off, `forward_delay 0` — there is no uplink, so no +loop to detect and no reason to hold ports down at Instance start), enables IPv4/IPv6 forwarding on +it, disables RA acceptance, and configures the gateway addresses. On `delete` it removes the bridge, +but only after confirming nothing is still attached — an Instance may have started on the network +while the last one was stopping. The whole script runs under `flock`, like `modifyvxlan.sh`, because +concurrent Instance starts on one network will race to create the bridge. + +Consequences: + +* **Networks are isolated from each other at layer 2 by the topology**, not by filtering. This is + the first reason for the design: a guest on `brdr-5828` has no L2 path of any kind to a guest on + `brdr-5829`, and no rule set has to be correct for that to hold (§12). +* **Each network becomes a named L3 interface on the hypervisor.** This is the second reason, and it + is an operational one: `brdr-5828` is something the operator can attach local policy to. Different + route-maps or redistribution filters per network, per-network policy routing, QoS, or later a VRF + per bridge — all expressible in the host's own network configuration, matched on interface name, + with no involvement from CloudStack. The routing daemon is already the operator's (§10); giving + each network its own interface is what makes per-network routing decisions possible at all. And + because the operator can choose the id (§6.7.2), that policy can be written **before** the network + exists. +* **Bridge name is identical on every host** — it derives only from the routed id — which is what + keeps migration a no-op for the guest. +* **The vif driver now needs work.** The earlier shared-bridge design could ride on + `BridgeVifDriver`'s existing `brname = trafficLabel` fallback + (`plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java:255`); + that no longer applies. The driver must take the routed id from the NIC's broadcast URI + (`BroadcastDomainType.getValue(nic.getBroadcastUri())`) and invoke `modifybrdr.sh` on plug, and on + unplug when the last interface leaves — the same shape as its existing `createVnetBr()` handling + for VXLAN. + +#### 9.2.1 How the agent learns the bridge name **DECIDED — revised 2026-09-04** + +**`NicTO` carries the broadcast URI.** The URI (`routed://5828`) and broadcast type +(`BroadcastDomainType.Routed`) are populated for every NIC by `HypervisorGuruBase.toNicTO()`, the +same way VLAN and VXLAN NICs receive theirs. + +`BridgeVifDriver` selects on the broadcast type (§9.1.3), extracts the routed id from the URI and +passes it to `modifybrdr.sh`, which creates the bridge and **prints the name it chose** — the agent +uses whatever comes back. On unplug the agent first asks the script whether the interface's bridge +is one of its own (`-o query -b ` → `mine`/`notmine`); `notmine` sends the agent down its +regular unplug path. For its own bridges the agent then removes the Instance's routes and +neighbour entries (`modifymacip.sh -o delete`, while the bridge still exists) and only then asks +the script to delete the bridge (`-o delete -b ` → `kept` or `deleted`). **How the bridges +are named is known only to the script**; no `brdr-` prefix appears anywhere in Java. The query on +every unplug costs one script execution per NIC on every host, direct routed or not; that is the +price of keeping the naming out of Java and is accepted. + +**This reverses the earlier decision**, which named the bridge from `networks.id` carried in +`NicTO.networkId`, precisely to avoid a broadcast domain, an isolation method and an id allocation. +The reversal rationale is in §6.7; the agent-side consequence is only that the number now comes from +the broadcast URI instead of the network-id field — the script contract (id in, name out, naming +private to the script) is unchanged. + +Consequences: + +* The network's `broadcast_uri` is `routed://` and its broadcast domain type is `Routed`, set + at creation and stable for the network's life. An auto-allocated id is released on network + deletion (§6.7.2); an operator-specified one was never in the pool. +* `specifyVlan` on the offering selects between operator-specified and allocated ids (§6.4, §6.7.2). +* An operator who wants `brdr-5828` to be a *specific* number **can have it** — create the network + from a `specifyVlan=true` offering with `vlan=5828`. Per-network host routing policy (§9.2) can + therefore be provisioned before the network exists. + +Per bridge, `modifybrdr.sh` sets: + +* `169.254.0.1/32` and `fe80::1/64` +* `net.ipv4.conf..forwarding=1`, `net.ipv6.conf..forwarding=1` +* `net.ipv6.conf..disable_ipv6=0` and `accept_ra=0` +* `arp_ignore=1` / `arp_announce=2` — see §9.2.1 +* **no physical uplink** — see §9.3, this is mandatory +* proxy ARP: **not needed** — the gateway addresses are local to the bridge, so the bridge answers + guest ARP/ND directly, and host→guest neighbour entries are static rather than resolved + +#### 9.2.2 The same gateway address on many bridges **DECIDED** + +Every `brdr-*` bridge on a host carries the *same* `169.254.0.1` and `fe80::1`. This is intended, +and it is what makes a guest's configuration identical no matter which network or host it lands on. + +For IPv6 it is unremarkable: link-local addresses are per-link and scoped by interface, so `fe80::1` +on twenty bridges is normal and correct. + +For IPv4 the sysctls are what make it correct, and `modifybrdr.sh` sets both: + +* `arp_ignore=1` — answer ARP only for addresses configured on the interface the request arrived on, + so a request reaching `brdr-5828` is never answered on behalf of `brdr-5829` +* `arp_announce=2` — always source ARP from the address of the interface the request goes out of + +Each bridge is its own L2 domain, so the ARP exchange stays within the right one regardless; the +sysctls remove the cases where the host might otherwise answer or source from the wrong interface. +The host's local route table gains one `local 169.254.0.1` entry per bridge, which is harmless — the +host never originates traffic from that address, it only replies on-link. + +### 9.3 The bridges have no physical uplink **DECIDED — correctness requirement** + +Unlike `cloudbr0`, a `brdr-*` bridge has **no physical port**. It is purely host-local: the only L3 +presence on it is `169.254.0.1` / `fe80::1`, and all guest traffic leaves the host via the host's own +routed uplink rather than being bridged. `modifybrdr.sh` never enslaves an interface, so this holds +by construction as long as nothing else adds one. + +If a bridge did have an uplink onto a shared L2 segment, two things break: + +1. **Duplicate gateway addresses.** Every hypervisor configures `169.254.0.1` and `fe80::1` on every + `brdr-*` bridge. Put those on a common segment and every host answers ARP/ND for the same + addresses; guests would resolve the gateway to an arbitrary host's MAC. +2. **Isolation collapses.** Guests of that network across all hosts would share one broadcast domain, + and the per-network bridge would stop being a boundary. + +### 9.4 Layer 2 isolation comes from the bridges **DECIDED** + +Separate bridges per network are the isolation mechanism. A guest on `brdr-5828` cannot send a frame of +any kind — ARP, raw L2, rogue RA, anything — to a guest on `brdr-5829`. There is no shared broadcast +domain, no shared FDB, and no path that filtering would have to police. + +This replaces the earlier shared-bridge design, in which separate networks were only administrative +and L2 separation had to be recovered with filtering. **No libvirt nwfilter is used**: the +`no-mac-spoofing` / `clean-traffic` approach was for the shared-bridge model and is dropped entirely. + +Two consequences worth stating plainly: + +* **Isolation between networks needs no filtering at all.** The boundary is the bridge rather than + a rule set. That was the motivation for this change. +* **Guests within one network still share a bridge**, so spoofing between them remains possible + (§12.3). Since a network belongs to one account, that is intra-tenant exposure rather than + cross-tenant. + +Bridge port isolation (`bridge link set dev vnetX isolated on`) remains available as a further step +if intra-network isolation is ever wanted; it is not applied in v1 (§12.3, §15). + +### 9.5 Sysctls + +Routing happens on the bridge, so the bridge is the L3 input interface for guest traffic and these +apply per `brdr-*` bridge rather than per tap. `modifybrdr.sh` sets them at creation: + +* `net.ipv4.conf..forwarding=1`, `net.ipv6.conf..forwarding=1` +* `net.ipv6.conf..disable_ipv6=0` — also set by `modifymacip.sh` before installing NDP entries +* `net.ipv6.conf..accept_ra=0` — a guest must never be able to send an RA the host acts on +* `net.ipv4.conf..arp_ignore=1` / `arp_announce=2` — required because every `brdr-*` bridge on + the host carries the same gateway address (§9.2.2) + +**`rp_filter=1` (strict) is set on every `brdr-*` bridge. DECIDED.** An Instance may only send from +an address that routes back out of the bridge it arrived on — which, given per-guest /32 routes, is +its own address. Source spoofing is therefore blocked at the host. + +Set **on the bridge only, never on `all`**, so no other interface on the host changes behaviour. The +kernel takes `max(conf.all.rp_filter, conf..rp_filter)`, so a per-bridge value of 1 is effective +regardless of what `all` is, without touching the uplink — which matters, because strict mode on a +host uplink can break legitimately asymmetric fabric routing. + +Strict mode is safe for the paths this design creates: + +* an Instance's own traffic — source `S`, route `S/32 dev brdr-N`, arrives on `brdr-N` → passes +* same-network hairpin (§5.4) — arrives on `brdr-N` from an address routed via `brdr-N` → passes +* cross-network — arrives on `brdr-N`, forwarded out `brdr-M`; the check is on ingress only → passes + +**The sysctl is IPv4 only.** The kernel has no IPv6 `rp_filter`, so `modifybrdr.sh` installs the +netfilter counterpart per bridge: `ip6tables -t raw PREROUTING -i -m rpfilter --invert +-j DROP`. An Instance can then not spoof an IPv6 source either, with or without security groups. + +**The host protects itself from its Instances (added 2026-09-10, simplified 2026-09-11).** Because +the host routes for them, an Instance's packets enter the host's own IP stack — something classic +bridging never exposed. Without a rule an Instance could reach the hypervisor's management and +storage addresses and everything the host routes to, including the `169.254.0.0/16` control +network of the SystemVMs on `cloud0`. `modifybrdr.sh` therefore inserts, per bridge and at the top +of `INPUT`, a DROP for everything arriving from the bridge preceded by an ACCEPT for ICMP +(`iptables`) and ICMPv6 (`ip6tables`), which gateway resolution and reachability checks need. TCP +and UDP from an Instance never reach the host. The rules are removed with the bridge. Forwarding +from `brdr-*` to the management and +storage subnets is **not** filtered by CloudStack — that policy is the operator's host firewall, +exactly like the routing daemon (§10), and must be stated in the operator documentation. `FORWARD` +rules remain the exclusive domain of `security_group.py` (§12.2). + +Forwarding itself needs `net.ipv4.ip_forward=1` and `net.ipv6.conf.all.forwarding=1` on the host: +the per-bridge `forwarding` sysctls the script sets do not enable it on their own (IPv6 forwards +only when `all.forwarding` is set; IPv4's per-interface flag governs packets entering the bridge, +not the uplink). A host running a routing daemon has both; they are a documented prerequisite. + +### 9.6 No reconciliation on agent restart **DECIDED** + +**The agent does not scan or rebuild routes and neighbour entries at startup.** The existing hook +fires only on NIC plug/unplug (`BridgeVifDriver.java:291,298`), and that is sufficient. + +The reasoning, which is worth writing down because "kernel state is not persistent" invites the +opposite conclusion: + +* **Host reboot** clears the routes — but it also destroys every Instance on that host. The + management server starts them again, each start goes through the plug path, and the routes are + reinstalled as a side effect. There is nothing to reconcile, because there is nothing running whose + state could be missing. +* **Agent restart without a host reboot** does not clear anything. Routes and neighbour entries live + in the kernel, not in the agent, so running Instances keep working across an agent restart with no + action at all. +* **While the agent is down**, CloudStack cannot stop or migrate Instances on that host, so host + state cannot drift from what the management server believes. + +`modifymacip.sh` uses `ip route replace` / `ip neigh replace`, so any repeated plug is idempotent +regardless. + +**Residual, accepted:** if an Instance disappears without an unplug — libvirt kills it, or it crashes +in a way that skips the normal path — its route and neighbour entry linger. The practical risk is not +the stale kernel state itself but that the routing daemon keeps advertising that /32, so if the +Instance is started on another host the fabric may see the address from two places. Rare, and not +worth a startup sweep to prevent; noted so it is recognisable if it ever shows up. + +## 10. Routing daemon — explicitly out of scope + +**CloudStack does not install, configure, or monitor the routing daemon.** + +The contract is one-directional: *CloudStack guarantees the correct routes and neighbour entries +exist in the host kernel.* The operator configures their daemon to redistribute them, e.g. with +FRR: + +``` +router bgp 65001 + address-family ipv4 unicast + redistribute kernel + address-family ipv6 unicast + redistribute kernel +``` + +Why this is the right boundary: + +* Identical behaviour for FRR, BIRD, or anything else that redistributes kernel routes. +* Identical behaviour for BGP, OSPF, or IS-IS. +* No daemon version coupling, no config-file ownership conflict with the operator's automation, no + `vtysh`/reload dependency in the agent. +* Nothing new to fail: if the route is in the kernel the guest works locally, and advertisement is + the fabric's business. + +Consequences to accept and document: + +* We ship **documentation and reference configuration**, not code. Redistribution filtering via + route-maps is the operator's job. +* **No feedback loop.** CloudStack cannot tell whether a guest's address is reachable from the + fabric. Accepted as a known gap for v1; a health signal from the agent is listed under future work + (§15) rather than treated as an open question. +* This is the **opposite** choice from the 4.20 BGP work, where CloudStack manages peers + (`BgpPeerVO`, `SetBgpPeersCommand`, `systemvm/debian/opt/cloud/bin/cs/CsBgpPeers.py`). The doc + should say why explicitly — reviewers will ask. +* Design should not preclude an optional managed mode later, but v1 does not have one. + +## 11. Live migration + +The guest's configuration is host-independent, so the guest needs no reconfiguration. What moves is +host state: + +1. Destination host installs the route and neighbour entry when the NIC is plugged. +2. Source host removes both when the NIC goes away. +3. Each host's routing daemon advertises/withdraws; the fabric reconverges. + +To work through: + +* **Ordering — confirmed.** Install-then-remove is safer than remove-then-install; a transient + duplicate advertisement is less harmful than a black hole. The destination plugs (and so + installs) in `PrepareForMigration`; the source unplugs only after the migration succeeded + (`LibvirtMigrateCommandWrapper`). A failed prepare unplugs what it plugged; a migration that + fails *after* a successful prepare leaves the destination's entries in place — deferred (§15). +* **Convergence gap.** Traffic may be black-holed until the fabric reconverges. **TODO:** quantify + on a normal iBGP/OSPF setup — is sub-second realistic? +* **No GARP needed — but the reverse direction bit us (fixed 2026-09-08).** Normally a migrating + VM sends a gratuitous ARP to update switch tables. Here there is no L2 path to update, and the + destination host's neighbour entry for the guest is installed statically by the agent — that + direction is fine. The *guest's* entry for the gateway, however, is learned: it held the source + host's bridge MAC, and after migration the destination bridge (with a different MAC) silently + dropped frames addressed to it for ~30 s until the guest's neighbour entry expired. Resolved by + giving every brdr bridge a MAC derived deterministically from the routed id + (`0e:` + 5 id bytes, set by modifybrdr.sh on every add), so the gateway MAC — like the gateway + addresses — is identical on every hypervisor and the guest's cache stays valid. +* **Per-address advertisement is mandatory.** Any aggregation scheme that pins prefixes to hosts + breaks migration. This is the price of §6.3.3. +* **Routing domain boundaries.** Migration works as long as source and destination are in the same + routing domain. **Left to the operator, not validated by CloudStack** — consistent with §10 and + §6.3.3, where fabric topology is local network design. CloudStack has no model of routing domains + and inventing one to police migration would be a larger change than the problem warrants. + +## 12. Security + +### 12.1 The isolation model + +The boundary between networks is **topological**: one bridge per network (§9.2, §9.4), with no +uplink and therefore no path between bridges except through the host's routing table. Nothing has to +be configured correctly for that to hold, and nothing degrades if filtering is disabled. + +Within one network the guests still share a bridge, so the properties there are weaker: + +* **No L3 adjacency.** Each guest is a /32 behind the host's routing table; there is no + subnet-mates relationship to exploit even between guests on the same bridge. +* **Static neighbour entries** prevent ARP-based address takeover on the host→guest path: the + IP-to-MAC mapping is asserted by `modifymacip.sh`, never learned (§5.3). The host is therefore not + susceptible to a guest claiming another guest's address. +* **`rp_filter=1` on the bridge** (§9.5) confines an Instance to sending from its own /32, since + that is the only address routed back out of that bridge. IPv4 only. + +What remains open inside a network is set out in §12.3. Because a network belongs to one account, +that is intra-tenant exposure. + +### 12.2 Security groups — supported, via the unified script rules **DECIDED** + +Security groups are supported on L3 networks, programmed by `security_group.py` as for Shared +networks. There is **no separate L3 rule function**: the classic and Direct Routed paths share one +implementation, parameterized only by how each direction identifies the Instance, plus small +conditionals for what does not exist on L3 (DHCP, DHCPv6, router advertisements towards guests). + +| Direction | Classic bridge | Direct Routed bridge | +|---|---|---| +| from the Instance | `-m physdev --physdev-in ` | same rule, identical | +| towards the Instance | `-m physdev --physdev-is-bridged --physdev-out ` | `-m set --match-set dst` | + +**The `--physdev-is-bridged` question, settled in kernel source** (`net/netfilter/xt_physdev.c`): + +* On `--physdev-in` rules the flag was **removable**. Match-time semantics: a bridged-then-routed + packet carries bridge info with the ingress port set and no bridged egress port, so a plain + `--physdev-in` matches it while `--physdev-is-bridged` excludes it. On classic bridges removing + the flag changes nothing observable — the BF- framework hook (which keeps the flag) already + restricts what reaches the per-VM chains to bridged traffic. Removing it is what lets both paths + share the from-Instance rules verbatim. The golden test was regenerated once, deliberately, for + exactly this diff. +* On `--physdev-out` rules the flag **stays**: for a routed packet no bridge info exists at all at + FORWARD time (`nf_bridge_info_exists()` is false and every physdev variant returns false), so the + kernel is structurally incapable of identifying the bridged egress port there. Removing the flag + would change nothing and merely deviate from upstream convention. This is why the to-Instance + direction matches the destination against the Instance's ipsets instead — exact, since L3 + addresses are /32s and /128s. + +The IPv6 source-spoof drop in the shared rules doubles as the missing IPv6 `rp_filter` (§9.5). +Teardown uses one awk pattern on the chain names, which also covers rules created by older versions +that still carry the flag on `physdev-in` lines. + +**Framework setup is shared too.** `enable_bridge_netfilter()`, `create_bridge_fw_chains()` and +`add_notrack_ipset_rules()` were extracted from `add_fw_framework()` and are used by both it and +`add_l3_fw_framework()`; the two differ only in their FORWARD hooks (classic gates on +`physdev-is-bridged` and consults chain reference counts; L3 jumps unconditionally, with the same +default-deny backstop). A second golden test pins `add_fw_framework`'s command stream, proving the +extraction left it byte-identical — 44 commands, unchanged. + +**How the script knows: the Agent tells it.** `security_group.py` performs no classification of its +own — no bridge-name check, no gateway inspection. The Agent already identifies these NICs for the +bridge and MAC/IP hook (§9.1.3), so it passes `--directrouted` on `default_network_rules` and +`add_network_rules`; the script's own re-entry points thread the flag through. This keeps the +decision in exactly one host-side place and leaves the script with no inference to get wrong. + +`network_rules_for_rebooted_vm` is dead code (its only caller has been commented out upstream for +years) and carries no L3 handling; a rebooted Instance is reprogrammed by the Agent through the +normal `default_network_rules`/`add_network_rules` path, which receives the flag. + +**Two-pass filtering on the routed path (added 2026-09-10, simplified 2026-09-11).** A routed +packet between two Instances on one host enters on one `brdr-*` bridge and leaves on another (or +the same), so a single per-bridge FORWARD jump can only ever evaluate one of them — whichever +bridge's rule came first decided terminally, which let an Instance bypass the ingress rules of an +Instance in another account. FORWARD therefore runs two passes over shared chains, source first: + +``` +-A FORWARD -j BF-L3-IN per bridge: -i -j BF--IN +-A FORWARD -j BF-L3-OUT per bridge: -o -j BF--OUT, then -o -j DROP +-A FORWARD -i -j ACCEPT per bridge: passed the source pass, not for an L3 Instance here +``` + +Source-side rules never ACCEPT: an allowed packet RETURNs so the destination pass still runs, a +denied one is dropped. Egress security group rules therefore end in RETURN and the egress chain +itself drops what no rule returned (with no egress rules it returns everything); until +`add_network_rules` has run the chain holds a single DROP. The destination pass gives the final +verdict, and an address on an L3 bridge no Instance claims is dropped. A bridge port with no +programmed rules is not dropped — the same gap classic bridges have, bounded by the per-Instance +route and `rp_filter`. Non-L3 traffic matches no per-bridge rule and falls through, so classic +bridges are unaffected. When the Agent deletes a bridge, the periodic `cleanup_rules` removes the +rules and chains that named it. Unit tests walk packets through the generated rules for every +case in both bridge creation orders (`scripts/vm/network/tests/test_security_group.py`). + +**Still to verify in the lab:** the four traffic cases end to end with two Instances in +different accounts and networks on one host, including the same-bridge hairpin and IPv6 DAD, and +that `iptables-save` renders the rules in the form `verify_network_rules` expects. + +**Investigated and rejected: libvirt nwfilters.** An nwfilter-based implementation was built and +then discarded. Its ebtables layer would have served anti-spoofing well (it sees routed delivery), +but stateful ingress cannot work: libvirt's own to-Instance iptables hook is hard-coded as +`-m physdev --physdev-is-bridged --physdev-out` (`src/nwfilter/nwfilter_ebiptables_driver.c`), the +same structural blindness to routed delivery — inside libvirt, where it cannot be patched or +augmented with destination matching. The script approach filters correctly in both directions and, +after the unification above, without duplicate code. + +### 12.3 Residual risk within one network **ACCEPTED for v1** + +Guests of the same network share a bridge. With security groups enabled, RA and gateway +impersonation between them are handled by the shared rules (ebtables ARP pinning, NDP source +checks, RA drop) and source spoofing is bounded by the ipset checks plus `rp_filter` (§9.5). An +operator who disables security groups accepts RA and gateway impersonation between guests of that +one network — a deliberate operator choice; isolation between *tenants* is topological and +unaffected (§12.1). Source-address spoofing is bounded with or without security groups: strict +`rp_filter` for IPv4 and the `rpfilter` netfilter rule for IPv6 (§9.5). The host itself is immune +either way: its neighbour entries are static (§5.3) and its `INPUT` path from `brdr-*` bridges is +closed (§9.5). + +### 12.4 Sharp edge + +**Guests are directly reachable from the fabric.** No NAT, no VR firewall. Whatever the fabric +permits reaches the guest, subject only to the guest's security groups if they are in use. This is a +meaningful change in default posture versus an Isolated network and must be prominent in the +documentation. + +## 13. Orchestration touchpoints + +Status of the implementation checklist (September 2026): + +- [x] `NetworkOrchestrator.allocate()` / `prepare()` / `release()` — address lifecycle inherited + from `DirectNetworkGuru` (§7.3) +- [x] `NetworkOrchestrator` and `VirtualMachineManagerImpl` — `L2` and `Shared` branches reviewed; + L3 follows Shared where a subnet exists and L2 nowhere +- [x] `UserVmManagerImpl` — `addNicToVm` works; `updateVmNicIp` is rejected for L3 (host-route + form would need re-stamping); secondary IPs via `allocateSecondaryGuestIP` (§9.1.5) +- [x] `NetworkModelImpl` — `canUseForDeploy()`, `checkSecurityGroupSupportForNetwork()` extended +- [x] `IpAddressManagerImpl.allocateDirectIp()` / `Ipv6AddressManagerImpl.setNicIp6Address()` — + gate on the cidr for L3; the guru forces /32 and /128 + link-local gateways (§6.3) +- [x] `Networks.BroadcastDomainType.Routed` and `getRoutedId()` (§6.7, §6.7.2) +- [x] `BridgeVifDriver` — selects on `BroadcastDomainType.Routed` (or a `routed://` URI, §9.1.3), + creates the bridge via `modifybrdr.sh`, runs the MAC/IP hook, unplugs in query → MAC/IP + delete → bridge delete order (§9.2.1) +- [x] `scripts/vm/network/vnet/modifybrdr.sh` — bridge lifecycle, host protection (§9.2, §9.5) +- [x] Guru registers `IsolationMethod("ROUTED")`; `canHandle()` on guest type + isolation method; + `design()` sets `Routed` (§6.7) +- [x] `NetworkOrchestrator.encodeVlanIdIntoBroadcastUri()` — `ROUTED` physical network → + `routed://` URI; broadcast domain type derived from the URI scheme (§6.7.2) +- [x] `NetworkServiceImpl.commitNetwork()` — vnet auto-allocation and release extended to L3 +- [x] Offering validation — `validateL3NetworkOffering()` (§6.4) +- [x] `ConfigDriveBuilder.writeNetworkData()` — always generated with a direct routed NIC, for all + NICs (§8.2); network-level `gateway` key for direct routed IPv4 (§8.1) +- [x] CPVM/SSVM boot args carry IPv6; `common.sh` installs `onlink` v4 and static v6 defaults (§8.5) +- [x] `PublicNetworkGuru` — routed public ranges, EUI-64 IPv6 in the guru; routed ids guarded + against guest/public collisions in both directions and against `ROUTED` vnet ranges (§8.5, + §6.7.2) +- [x] IPv6 CIDR longer than /64 rejected for L3 (§6.3.4) +- [x] DNS on L3 offerings allowed; zone IPv6 DNS not required for L3 (§6.4, §6.5) +- [x] `security_group.py` — unified rules with the L3 dispatch structure of §12.2; **lab + verification of live traffic in both directions still required after the September 2026 + restructure** +- [x] Zone-wide IPv4 overlap validation, both directions (§6.3.2) +- [x] `.0`/`.255` assignable on the explicit range path; no `NetUtils` change needed (§6.3.1) +- [ ] VM import (`importNic` selects IPs the Isolated way for L3) — deferred (§15) +- [x] Network restart — no VR, nothing to restart; the element implementations are no-ops +- [ ] IP capacity reporting and usage records — L3 addresses are `user_ip_address` rows like + Shared-network addresses and are reported and billed the same way; not separately reviewed +- [x] UI: `ROUTED` in the isolation method lists, L3 offering form (specifyVlan as routed id), + L3 network form with physical network selector, Instance list addresses + +## 14. Upgrade and compatibility + +* Additive: new enum values (`GuestType.L3`, `BroadcastDomainType.Routed`), new isolation method + string, new offering type. No change to existing networks, no data migration. +* The pre-revision implementation on this branch (L3 networks with `Native` broadcast domain and an + empty `broadcast_uri`) was never released, so no migration from it is provided; development + deployments recreate their L3 networks. +* **Agent version gating is out of scope. DECIDED.** No capability flag, no version check, and no + management-server logic to keep Instances of this type away from agents that predate it. Operators + are expected to upgrade their agents as part of upgrading CloudStack, as they already are. The + failure mode if they do not is that `modifybrdr.sh` is missing on the old host and the Instance + fails to get connectivity — visible in the agent log, consistent with §9.1.4. +* Downgrade unsupported once networks of this type exist, as usual. + +## 15. Future work / explicitly deferred + +* Non-KVM hypervisors +* Optional CloudStack-managed routing daemon configuration +* **Host-side metadata service** (`169.254.169.254`) — a v2 candidate; the gateway address and the + data are both already present, so it is a natural addition (§8.4) +* Multiple addresses per NIC — additional /32s fit the model naturally, but v1 is one v4 + one v6 +* Reachability/health feedback from the routing daemon +* Network Config v2 `network-config` emission for NoCloud-configured images — a later PR (§8.1) +* Upstream cloud-init contribution: apply the on-link flag to routes-list entries and in the + sysconfig/NetworkManager/eni renderers, so guests beyond netplan/networkd work too (§8.1) +* Making a failed route/neighbour install fatal or alert-raising, rather than silent (§9.1.4) +* Closing the §12.3 intra-network spoofing gaps — bridge port isolation (§9.4) first, or libvirt + nwfilter if a narrower fix is preferred +* **Per-tenant VRFs**, which would lift the non-overlapping-subnet constraint of §6.3.2 + +Found in the September 2026 review and consciously deferred, not forgotten: + +* **Failed live migration** after a successful `PrepareForMigration`: the destination keeps the + host route and neighbour entry (and its daemon keeps advertising) for an Instance that stayed on + the source, so the fabric may black-hole it until the next start/stop. The orchestrator should + send the rollback form of `PrepareForMigrationCommand` on any migrate failure and the agent's + rollback should unplug the NICs (§11). +* **Who may create L3 networks.** `validateNetworkOfferingForNonRootAdminUser()` admits Isolated, + L2 and Shared-without-specifyVlan only, so L3 networks are root-admin-only today. Security-group + enabled Advanced zones reject L3 in two places (`NetworkOrchestrator`, `UserVmManagerImpl`). + Decide and either add L3 or document root-admin-only. +* **KVM-only is not enforced** (§6.8): no hypervisor check at network creation or placement. +* **IPv6 start/end ranges** are accepted for L3 though §6.3 says `ip6cidr` alone; with a range + two networks can share a /64 and EUI-64 addresses could collide. Reject the range. +* **Routed public ranges** still type an IPv4 and IPv6 gateway that nothing uses (§5.5). +* `deleteVlanIpRange`/`updateVlanIpRange` key on gateway presence and so mishandle gateway-less L3 + rows (stale `networks.cidr`, "IPv4 is not supported in this IP range"). +* `updateNetwork` cannot set IPv6 DNS on an L3 network (`isIpv6` keys on Shared). +* `network_rules_for_rebooted_vm` in `security_group.py` is dead code; `verify_network_rules` + expects the pre-2026 rule stream (§12.2). +* Per-interface `forwarding` sysctls do not enable forwarding by themselves: `net.ipv4.ip_forward=1` + and `net.ipv6.conf.all.forwarding=1` are host prerequisites (a host running a routing daemon has + them), to be stated in the operator documentation (§9.5). +* SystemVMs with a security-group NIC on a routed public range would hit the classic framework in + `default_network_rules_systemvm`; VRs are not expected on routed public ranges. +* The routed-id pool of a `ROUTED` physical network is capped at 1–4094 because the vnet-range + validation treats it like VLAN (§6.7.2). +* `handleVmStartFailure` in the KVM start wrapper unplugs the NICs of a domain that may have + started before a late `RuntimeException`; checking the domain state first would avoid removing + a running Instance's routes ahead of the orchestrator's Stop. +* VM import (`importNic`) treats L3 like Isolated when selecting an IP; it should follow Shared. + +## 16. Decision log and open questions + +### Revised 2026-09-04 — isolation model + +Three decisions from the 2026-07-30 design were reversed together; rationale in §6.7: + +* **Isolation method:** was "none to register"; now the guru registers `IsolationMethod("ROUTED")` + and direct routed networks live on a dedicated physical network carrying it. The network operator + opts the zone in explicitly. +* **Broadcast domain:** was "`Native`, empty URI"; now `BroadcastDomainType.Routed` with + `routed://`, set at creation and stable for the network's life. +* **Bridge naming:** was `brdr-` (unchoosable); now `brdr-`, with the id + operator-specified (`specifyVlan=true` + `vlan` parameter) or allocated from the physical + network's range (`specifyVlan=false`) — the Shared network's VLAN mechanics, reused (§6.7.2). + +The agent's inferred /32-plus-link-local-gateway gating became explicit gating on the broadcast +type as a consequence (§9.1.3). Entries below are updated in place; superseded wording is kept in +the relevant sections as "reverses the earlier decision" notes. + +### Settled + +* Network type is "no DHCP", not "like L2" (§1) +* IPv4 addresses come from the `user_ip_address` pool with an operator-supplied subnet (§6.3); + subnets must not overlap zone-wide (§6.3.2) +* **IPv6 is calculated (EUI-64) from the subnet and the NIC's MAC** — never drawn from a stored + pool; only the allocated result is stored, on the NIC. Already how both the guest and the public + path behave; requires the subnet to be /64 or larger, to be validated at creation (§6.3.4) +* New `GuestType.L3`, chosen over overloading `Shared`/`NetworkMode` (§6.1) +* Gateway is a static, non-configurable `169.254.0.1` / `fe80::1`; a /32 means the guest must treat + its gateway as on-link regardless, so configurability would buy nothing (§6.2) +* ConfigDrive emits a network-level `gateway` key for direct routed IPv4, from which cloud-init + derives `on-link` (§8.1) +* The agent writes only routes and neighbour entries; FRR is out of scope (§9.1, §10) +* That work is done by reusing `modifymacip.sh` + the `BridgeVifDriver` hook already in main from + `4816e059383` / PR #13495, rather than new code (§9.1.1) +* Routes and neighbour entries go on the **bridge**, not the tap — the script and hook are reused + unchanged (§9.1.2) +* **One bridge per network**, named `brdr-`, created and removed by the new + `scripts/vm/network/vnet/modifybrdr.sh` (§9.2) +* **Isolation method `ROUTED` on a dedicated physical network** selects the guru; the network's + broadcast domain is `routed://` and the id is operator-specified or allocated from the + physical network's range — reversing the earlier "no isolation method, no broadcast domain" + decision (§6.7, §6.7.2, §9.2.1; revision block above) +* Per-network bridges also give the operator a named interface per network to hang local routing + policy off — a deliberate benefit, not just a side effect (§9.2) +* Those bridges have no physical uplink (§9.3) +* L2 isolation between networks is topological — separate bridges — not filtering. **No libvirt + nwfilter is used**; the earlier `no-mac-spoofing` / `clean-traffic` plan is dropped (§9.4) +* Security groups are supported on L3 via **one unified rule implementation** shared with classic + bridges: `--physdev-is-bridged` dropped from `physdev-in` rules (verified harmless in kernel + source), destination-ipset matching towards the Instance. nwfilter was investigated and rejected + — libvirt's own to-Instance hook has the same physdev-is-bridged blindness (§12.2) +* Within one network, RA and gateway-impersonation protection depends on security groups being + enabled; leaving them off is a deliberate operator choice (§12.3) +* The `--physdev-is-bridged` rework is **required** — L3 filtering must work — and must be strictly + additive so existing Basic-zone and Shared-network rules are unchanged (§12.2) +* A network is direct routed when its offering's guest type is `L3` **and** it lives on a + physical network with isolation method `ROUTED` — guru selection follows the standard + isolation-method contract. Changing a network's offering afterwards is not guarded (§6.7, §6.7.1) +* The offering requires ConfigDrive `UserData`; `Dns` is optional but strongly recommended, and + `SecurityGroup` is optional. `Dhcp` is rejected — not just unsupported but unnecessary (§6.4, §6.5) +* `rp_filter=1` is set on each `brdr-*` bridge, bounding IPv4 source spoofing to the Instance's own + address; IPv6 has no kernel equivalent (§9.5) +* **No reconciliation at agent startup** — a host reboot destroys the Instances too, and an agent + restart does not clear kernel state (§9.6) +* The MAC/IP hook is gated **per NIC, on `BroadcastDomainType.Routed`** rather than a host + property; the same test picks the bridge. (Originally inferred from the /32 + link-local-gateway + address form; superseded by the explicit broadcast type — §9.1.3) +* The same `169.254.0.1` on every `brdr-*` bridge is **correct as designed**; `arp_ignore=1` and + `arp_announce=2` handle it (§9.2.2) +* The guru is a **subclass of `DirectNetworkGuru`**, inheriting the address lifecycle rather than + duplicating it (§7.3) +* **On-link via the network-level `gateway` key** — cloud-init applies its on-link detection only + to that key, never to routes-list entries, so CloudStack emits it for direct routed IPv4 + (verified on Ubuntu 26.04 / cloud-init 26.1); the v2 `network-config` file is deferred to a + later PR (§8.1, §15) +* DNS is **per network, falling back to the zone**; already implemented by + `NetworkModelImpl.getNetworkIp4Dns()` (§8.3) +* **No metadata service** in v1 — ConfigDrive only, no `169.254.169.254`; a v2 candidate (§8.4) +* Route/neighbour install failures **stay silent** for v1, as they are for EVPN (§9.1.4) +* Route scale is **out of scope** — fabric capacity and aggregation are local network design; + ~100k routes is not usually a problem on modern equipment, but CloudStack states no ceiling (§6.3.3) +* Network and broadcast addresses **must be assignable** — true on the explicit start/end range + path and asserted by the smoke test; the `cidr`-only form defaults to the usable range (§6.3.1) +* **No subnet gateway** for guest L3 networks: `gateway`/`ip6gateway` are ignored and stored as + NULL; routed public ranges still type one, which the guru replaces (§5.5, §6.3) +* The routed id is a **positive integer of at most ten digits**, canonicalised on input; public + ranges may not take an id inside a `ROUTED` physical network's range (§6.7.2) +* IPv4 range overlap is checked **zone-wide in both directions** on the range-creation path L3 + networks actually take (§6.3.2) +* The `--physdev-is-bridged` rework **lands in v1**; security groups are not shipped half-working + (§12.2) +* Zone-wide subnet overlap validation is **required** — an overlap is an address conflict (§6.3.2) +* **No warning** when a template ignores ConfigDrive; cloud-init inside the guest is the operator's + responsibility (§6.5) +* **Agent version gating is out of scope** — operators upgrade agents with CloudStack (§14) +* KVM only for v1 (§6.8); **VPC never** — it is a separate use case already served by VPC's own + BGP-routed subnets (§6.6) +* **SystemVMs stay, configured via boot args, dual-stack from the start** — the v4 default route + gains `onlink` for link-local gateways, the CPVM/SSVM builders emit the IPv6 args the VR builder + already emits, the systemvm installs the v6 default from `ip6gateway` instead of hoping for an + RA, and the public NIC is stamped/plugged in the same host-route + `routed://` form as a guest + NIC — its IPv6 coming from the same EUI-64 computation (§8.5, §6.3.4) + +### Still open + +**None.** Every design question raised in this document has been decided. + +What remains is implementation work, tracked in §13 and as `TODO` markers in the sections above. +Three of those are verification rather than coding, and are the ones most likely to change a +decision if they come out badly: + +* §12.2 — confirm on a real host that the unified rules match live traffic in both directions on + both bridge types, and that ARP for the gateway and neighbour discovery pass +* §6.3.1 — confirm nothing downstream of `createVlanIpRange` re-derives the usable range and + re-excludes `.0` and `.255` +* §6.3.2 — confirm the existing overlap checks are zone-wide, and widen them if not diff --git a/pom.xml b/pom.xml index 0dc2b34208c0..5834ebf33d57 100644 --- a/pom.xml +++ b/pom.xml @@ -1072,6 +1072,7 @@ debian/dirs debian/rules debian/source/format + docs/design/direct-routed-networks.md dist/console-proxy/js/jquery.js engine/schema/dist/** plugins/hypervisors/hyperv/conf/agent.properties