Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@
import java.util.Set;
import java.util.stream.Collectors;

import javax.inject.Inject;

import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.storage.VMTemplateVO;
Expand All @@ -43,16 +45,21 @@
import com.cloud.exception.ManagementServerException;
import com.cloud.exception.NetworkRuleConflictException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.exception.VirtualMachineMigrationException;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.kubernetes.cluster.KubernetesCluster;
import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl;
import com.cloud.kubernetes.cluster.KubernetesClusterService;
import com.cloud.kubernetes.cluster.KubernetesClusterVO;
import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO;
import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil;
import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeCapacityReconciler;
import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeCapacityReconciler.NodeAccess;
import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeCapacityReconciler.NodeCapacitySnapshot;
import com.cloud.kubernetes.cluster.utils.KubernetesClusterNodeCapacityReconcilerImpl;
import com.cloud.network.IpAddress;
import com.cloud.network.Network;
import com.cloud.network.rules.FirewallRule;
import com.cloud.network.rules.PortForwardingRuleVO;
import com.cloud.offering.ServiceOffering;
import com.cloud.storage.LaunchPermissionVO;
import com.cloud.uservm.UserVm;
Expand All @@ -79,6 +86,9 @@
private Boolean isAutoscalingEnabled;
private long scaleTimeoutTime;

@Inject

Check warning on line 89 in plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Remove this field injection and use constructor injection instead.

See more on https://sonarcloud.io/project/issues?id=apache_cloudstack&issues=AaDJI_DaJBZ3696PvGsa&open=AaDJI_DaJBZ3696PvGsa&pullRequest=14223
protected KubernetesClusterNodeCapacityReconciler kubernetesClusterNodeCapacityReconciler;

protected KubernetesClusterScaleWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) {
super(kubernetesCluster, clusterManager);
}
Expand Down Expand Up @@ -342,7 +352,7 @@
}
}

private void scaleKubernetesClusterOffering(KubernetesClusterNodeType nodeType, ServiceOffering serviceOffering,

Check failure on line 355 in plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Refactor this method to reduce its Cognitive Complexity from 43 to the 15 allowed.

See more on https://sonarcloud.io/project/issues?id=apache_cloudstack&issues=AaDJI_DaJBZ3696PvGsb&open=AaDJI_DaJBZ3696PvGsb&pullRequest=14223
boolean updateNodeOffering, boolean updateClusterOffering) throws CloudRuntimeException {
validateKubernetesClusterScaleOfferingParameters();
List<KubernetesCluster.State> scalingStates = List.of(KubernetesCluster.State.Scaling, KubernetesCluster.State.ScalingStoppedCluster);
Expand All @@ -359,14 +369,66 @@
for (long i = 0; i < tobeScaledVMCount; i++) {
KubernetesClusterVmMapVO vmMapVO = vmList.get((int) i);
UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId());
if (userVM == null) {
logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster : %s failed, unable to find cluster VM %s",
kubernetesCluster.getName(), vmMapVO.getVmId()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
}
ServiceOffering oldOffering = serviceOfferingDao.findById(userVM.getServiceOfferingId());
boolean capacityChanged = KubernetesClusterNodeCapacityReconcilerImpl.capacityChanged(oldOffering, serviceOffering);
if (capacityChanged && KubernetesCluster.State.Running == originalState && ETCD == nodeType
&& KubernetesCluster.ClusterType.CloudManaged == kubernetesCluster.getClusterType()) {
logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster : %s cannot live-resize dedicated etcd VM %s; " +
"etcd health reconciliation is not implemented", kubernetesCluster.getName(), userVM.getDisplayName()),
kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
}
boolean reconcileKubernetes = shouldReconcileNodeCapacity(vmMapVO, userVM, oldOffering, serviceOffering, nodeType);
NodeAccess nodeAccess = null;
NodeCapacitySnapshot before = null;
boolean result = false;
boolean cksCordonCompleted = false;
boolean resizeSucceeded = serviceOffering.getId() == userVM.getServiceOfferingId();
try {
result = userVmManager.upgradeVirtualMachine(userVM.getId(), serviceOffering.getId(), new HashMap<String, String>());
} catch (RuntimeException | ResourceUnavailableException | ManagementServerException | VirtualMachineMigrationException e) {
logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster : %s failed, unable to scale cluster VM : %s due to %s", kubernetesCluster.getName(), userVM.getDisplayName(), e.getMessage()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e);
}
if (!result) {
logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster : %s failed, unable to scale cluster VM : %s", kubernetesCluster.getName(), userVM.getDisplayName()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
if (reconcileKubernetes) {
nodeAccess = resolveNodeAccess(userVM);
before = kubernetesClusterNodeCapacityReconciler.captureBefore(kubernetesCluster, userVM, nodeAccess);
kubernetesClusterNodeCapacityReconciler.cordonIfNeeded(kubernetesCluster, userVM, before, nodeAccess, scaleTimeoutTime);
cksCordonCompleted = !before.isUnschedulable() || before.isCloudStackResizeCordon();
}
if (serviceOffering.getId() != userVM.getServiceOfferingId()) {
result = userVmManager.upgradeVirtualMachine(userVM.getId(), serviceOffering.getId(), new HashMap<String, String>());
if (!result) {
logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster : %s failed, unable to scale cluster VM : %s", kubernetesCluster.getName(), userVM.getDisplayName()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
}
resizeSucceeded = true;
userVM = userVmDao.findById(userVM.getId());
if (userVM == null || serviceOffering.getId() != userVM.getServiceOfferingId()) {
logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster : %s failed, VM %s did not reach target offering", kubernetesCluster.getName(), vmMapVO.getVmId()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
}
}
if (reconcileKubernetes) {
kubernetesClusterNodeCapacityReconciler.verifyGuestResources(userVM, serviceOffering, before, nodeAccess, scaleTimeoutTime);
if (!kubernetesClusterNodeCapacityReconciler.isKubernetesResourcesCurrent(before, serviceOffering)) {
kubernetesClusterNodeCapacityReconciler.restartKubelet(userVM, nodeAccess, scaleTimeoutTime);
kubernetesClusterNodeCapacityReconciler.waitForKubernetesResources(kubernetesCluster, userVM, serviceOffering, before, nodeAccess, scaleTimeoutTime);
}
kubernetesClusterNodeCapacityReconciler.restoreSchedulability(kubernetesCluster, userVM, before, nodeAccess, scaleTimeoutTime);
}
} catch (Exception e) {
if (reconcileKubernetes && cksCordonCompleted && !resizeSucceeded) {
try {
kubernetesClusterNodeCapacityReconciler.restoreSchedulability(kubernetesCluster, userVM, before, nodeAccess, scaleTimeoutTime);
} catch (Exception cleanupException) {
logger.warn("Unable to restore schedulability for CKS node {} after a pre-resize failure", userVM.getUuid(), cleanupException);
}
}
String recovery = resizeSucceeded
? "The node remains cordoned; check kubelet on the VM and retry the CKS scale operation."
: "The VM was not resized; CKS attempted to restore its temporary cordon.";
String message = String.format("Scaling Kubernetes cluster %s (UUID %s) failed for %s node VM %s (UUID %s), " +
"offering %s to %s: %s %s", kubernetesCluster.getName(), kubernetesCluster.getUuid(), nodeType,
userVM.getDisplayName(), userVM.getUuid(), oldOffering == null ? null : oldOffering.getId(),
serviceOffering.getId(), e.getMessage(), recovery);
logTransitStateAndThrow(Level.ERROR, message, kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e);
}
if (System.currentTimeMillis() > scaleTimeoutTime) {
logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster : %s failed, scaling action timed out", kubernetesCluster.getName()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
Expand All @@ -375,6 +437,42 @@
kubernetesCluster = updateKubernetesClusterEntryForNodeType(null, nodeType, serviceOffering, updateNodeOffering, updateClusterOffering);
}

private NodeAccess resolveNodeAccess(UserVm userVM) {
Pair<String, Integer> controlAccess = getKubernetesClusterServerIpSshPort(null);
if (StringUtils.isBlank(controlAccess.first())) {
throw new CloudRuntimeException(String.format("Unable to resolve control-plane SSH for Kubernetes cluster %s", kubernetesCluster.getUuid()));
}
if (manager.isDirectAccess(network)) {
if (StringUtils.isBlank(userVM.getPrivateIpAddress())) {
throw new CloudRuntimeException(String.format("Unable to resolve private SSH address for VM %s", userVM.getUuid()));
}
return new NodeAccess(controlAccess.first(), controlAccess.second(), userVM.getPrivateIpAddress(), DEFAULT_SSH_PORT,
getControlNodeLoginUser(), getManagementServerSshPublicKeyFile());
}
PortForwardingRuleVO sshRule = portForwardingRulesDao.listByVm(userVM.getId()).stream()
.filter(rule -> rule.getDestinationPortStart() == DEFAULT_SSH_PORT)
.filter(rule -> !FirewallRule.State.Revoke.equals(rule.getState()))
.findFirst().orElse(null);
if (sshRule == null) {
throw new CloudRuntimeException(String.format("Unable to resolve SSH port-forwarding rule for VM %s", userVM.getUuid()));
}
IpAddress targetPublicIp = network.getVpcId() == null ? getNetworkSourceNatIp(network) : getVpcTierKubernetesPublicIp(network);
if (targetPublicIp == null) {
throw new CloudRuntimeException(String.format("Unable to resolve target-node public IP for VM %s", userVM.getUuid()));
}
return new NodeAccess(controlAccess.first(), controlAccess.second(), targetPublicIp.getAddress().addr(), sshRule.getSourcePortStart(),
getControlNodeLoginUser(), getManagementServerSshPublicKeyFile());
}

protected boolean shouldReconcileNodeCapacity(KubernetesClusterVmMapVO vmMapVO, UserVm userVM,
ServiceOffering oldOffering, ServiceOffering targetOffering,
KubernetesClusterNodeType nodeType) {
return !vmMapVO.isExternalNode()
&& KubernetesCluster.ClusterType.CloudManaged == kubernetesCluster.getClusterType()
&& Hypervisor.HypervisorType.KVM == userVM.getHypervisorType()
&& kubernetesClusterNodeCapacityReconciler.requiresKubeletRefresh(oldOffering, targetOffering, nodeType, originalState);
}

private void removeNodesFromCluster(List<KubernetesClusterVmMapVO> vmMaps) throws CloudRuntimeException {
for (KubernetesClusterVmMapVO vmMapVO : vmMaps) {
UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId());
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.kubernetes.cluster.utils;

import java.io.File;

import com.cloud.kubernetes.cluster.KubernetesCluster;
import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType;
import com.cloud.offering.ServiceOffering;
import com.cloud.uservm.UserVm;

/** Reconciles the guest and Kubernetes resource views after a live VM resize. */
public interface KubernetesClusterNodeCapacityReconciler {
NodeCapacitySnapshot captureBefore(KubernetesCluster cluster, UserVm vm, NodeAccess access) throws Exception;

Check warning on line 28 in plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Replace generic exceptions with specific library exceptions or a custom exception.

See more on https://sonarcloud.io/project/issues?id=apache_cloudstack&issues=AaDJI_XyJBZ3696PvGsj&open=AaDJI_XyJBZ3696PvGsj&pullRequest=14223
boolean requiresKubeletRefresh(ServiceOffering oldOffering, ServiceOffering newOffering,
KubernetesClusterNodeType nodeType, KubernetesCluster.State originalState);
void cordonIfNeeded(KubernetesCluster cluster, UserVm vm, NodeCapacitySnapshot before, NodeAccess access, long deadline) throws Exception;

Check warning on line 31 in plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Replace generic exceptions with specific library exceptions or a custom exception.

See more on https://sonarcloud.io/project/issues?id=apache_cloudstack&issues=AaDJI_XyJBZ3696PvGsk&open=AaDJI_XyJBZ3696PvGsk&pullRequest=14223
void verifyGuestResources(UserVm vm, ServiceOffering target, NodeCapacitySnapshot before, NodeAccess access, long deadline) throws Exception;

Check warning on line 32 in plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Replace generic exceptions with specific library exceptions or a custom exception.

See more on https://sonarcloud.io/project/issues?id=apache_cloudstack&issues=AaDJI_XyJBZ3696PvGsl&open=AaDJI_XyJBZ3696PvGsl&pullRequest=14223
boolean isKubernetesResourcesCurrent(NodeCapacitySnapshot snapshot, ServiceOffering target);
void restartKubelet(UserVm vm, NodeAccess access, long deadline) throws Exception;

Check warning on line 34 in plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Replace generic exceptions with specific library exceptions or a custom exception.

See more on https://sonarcloud.io/project/issues?id=apache_cloudstack&issues=AaDJI_XyJBZ3696PvGsm&open=AaDJI_XyJBZ3696PvGsm&pullRequest=14223
NodeCapacitySnapshot waitForKubernetesResources(KubernetesCluster cluster, UserVm vm, ServiceOffering target,
NodeCapacitySnapshot before, NodeAccess access, long deadline) throws Exception;

Check warning on line 36 in plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Replace generic exceptions with specific library exceptions or a custom exception.

See more on https://sonarcloud.io/project/issues?id=apache_cloudstack&issues=AaDJI_XyJBZ3696PvGsn&open=AaDJI_XyJBZ3696PvGsn&pullRequest=14223
void restoreSchedulability(KubernetesCluster cluster, UserVm vm, NodeCapacitySnapshot before, NodeAccess access, long deadline) throws Exception;

Check warning on line 37 in plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Replace generic exceptions with specific library exceptions or a custom exception.

See more on https://sonarcloud.io/project/issues?id=apache_cloudstack&issues=AaDJI_XyJBZ3696PvGso&open=AaDJI_XyJBZ3696PvGso&pullRequest=14223

final class NodeAccess {
private final String controlAddress; private final int controlPort; private final String nodeAddress; private final int nodePort;

Check warning on line 40 in plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Declare "controlPort" and all following declarations on a separate line.

See more on https://sonarcloud.io/project/issues?id=apache_cloudstack&issues=AaDJI_XyJBZ3696PvGsp&open=AaDJI_XyJBZ3696PvGsp&pullRequest=14223
private final String user; private final File sshKeyFile;

Check warning on line 41 in plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Declare "sshKeyFile" on a separate line.

See more on https://sonarcloud.io/project/issues?id=apache_cloudstack&issues=AaDJI_XyJBZ3696PvGsq&open=AaDJI_XyJBZ3696PvGsq&pullRequest=14223
public NodeAccess(String controlAddress, int controlPort, String nodeAddress, int nodePort, String user, File sshKeyFile) {
this.controlAddress = controlAddress; this.controlPort = controlPort; this.nodeAddress = nodeAddress; this.nodePort = nodePort;
this.user = user; this.sshKeyFile = sshKeyFile;
}
public String getControlAddress() { return controlAddress; } public int getControlPort() { return controlPort; }
public String getNodeAddress() { return nodeAddress; } public int getNodePort() { return nodePort; }
public String getUser() { return user; } public File getSshKeyFile() { return sshKeyFile; }
}

final class NodeCapacitySnapshot {
private final boolean unschedulable; private final boolean cloudStackResizeCordon; private final long guestOnlineCpuCount;

Check warning on line 52 in plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Declare "cloudStackResizeCordon" and all following declarations on a separate line.

See more on https://sonarcloud.io/project/issues?id=apache_cloudstack&issues=AaDJI_XyJBZ3696PvGsr&open=AaDJI_XyJBZ3696PvGsr&pullRequest=14223
private final long guestMemoryKiB; private final long capacityCpuMillis; private final long capacityMemoryBytes;

Check warning on line 53 in plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Declare "capacityCpuMillis" and all following declarations on a separate line.

See more on https://sonarcloud.io/project/issues?id=apache_cloudstack&issues=AaDJI_XyJBZ3696PvGss&open=AaDJI_XyJBZ3696PvGss&pullRequest=14223
private final long allocatableCpuMillis; private final long allocatableMemoryBytes; private final boolean ready;

Check warning on line 54 in plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Declare "allocatableMemoryBytes" and all following declarations on a separate line.

See more on https://sonarcloud.io/project/issues?id=apache_cloudstack&issues=AaDJI_XyJBZ3696PvGst&open=AaDJI_XyJBZ3696PvGst&pullRequest=14223
public NodeCapacitySnapshot(boolean unschedulable, boolean cloudStackResizeCordon, long guestOnlineCpuCount, long guestMemoryKiB,

Check warning on line 55 in plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterNodeCapacityReconciler.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Constructor has 9 parameters, which is greater than 7 authorized.

See more on https://sonarcloud.io/project/issues?id=apache_cloudstack&issues=AaDJI_XyJBZ3696PvGsu&open=AaDJI_XyJBZ3696PvGsu&pullRequest=14223
long capacityCpuMillis, long capacityMemoryBytes, long allocatableCpuMillis, long allocatableMemoryBytes, boolean ready) {
this.unschedulable = unschedulable; this.cloudStackResizeCordon = cloudStackResizeCordon; this.guestOnlineCpuCount = guestOnlineCpuCount;
this.guestMemoryKiB = guestMemoryKiB; this.capacityCpuMillis = capacityCpuMillis; this.capacityMemoryBytes = capacityMemoryBytes;
this.allocatableCpuMillis = allocatableCpuMillis; this.allocatableMemoryBytes = allocatableMemoryBytes; this.ready = ready;
}
public boolean isUnschedulable() { return unschedulable; } public boolean isCloudStackResizeCordon() { return cloudStackResizeCordon; }
public long getGuestOnlineCpuCount() { return guestOnlineCpuCount; } public long getGuestMemoryKiB() { return guestMemoryKiB; }
public long getCapacityCpuMillis() { return capacityCpuMillis; } public long getCapacityMemoryBytes() { return capacityMemoryBytes; }
public long getAllocatableCpuMillis() { return allocatableCpuMillis; } public long getAllocatableMemoryBytes() { return allocatableMemoryBytes; }
public boolean isReady() { return ready; }
}
}
Loading
Loading