From b810503a6da4b5a407e65dee9cf853f89519dd84 Mon Sep 17 00:00:00 2001 From: Brad House Date: Fri, 6 Nov 2026 21:23:09 +0000 Subject: [PATCH] Allow scaling a Kubernetes cluster that is in Alert When any node VM of a running cluster isn't Running, the status scanner moves the cluster to Alert, and scaleKubernetesCluster then refuses every request: Kubernetes cluster is in Alert state and can not be scaled The cluster-autoscaler scales through the same API, so one stopped node blocks adding capacity, autoscaling, and removing that node with nodeids, which is when scaling is most needed. Allow a size change, or removing nodes by nodeids, in Alert: - state machine: Alert -> Scaling on ScaleUpRequested and ScaleDownRequested. Scaling -> Running / -> Alert are unchanged, so the scanner puts the cluster back in Alert if a node is still down. - the state check moves to validateKubernetesClusterStateForScale. In Alert it refuses an offering or autoscaling change, which act on every node (a different minsize/maxsize counts, as in the worker); a request that changes neither the size nor the nodes, which would only move the cluster to Running; removing control, etcd or external nodes; any request while the first control node VM, which scaling reaches over SSH, isn't running; any request while the cluster has more or fewer VMs than nodes, for example VMs left by a failed scale-up, which a scale-up would only add to; any request while a control or etcd VM is down, as etcd may have lost quorum and new nodes can't join; and any request while none of its (non-external) worker VMs is down, as the cluster is then in Alert for another reason, such as a failed start or upgrade or a VM on a disconnected host, that scaling won't fix. Sending the offerings and autoscaling setting the cluster already has is allowed. - in Alert the worker gets no offerings, so it only changes the size and can't apply the cluster offering to another node type. The request is carried out against the cluster as validated, not a fresh read the scanner may have changed, and fails if the cluster's state has changed since it was validated (to Alert, or by another operation starting). - scale-up waits for the nodes whose VMs aren't down to be Ready, rather than every node: a stopped node is never Ready, so the scale-up always timed out. The new nodes must all be Ready, and the count is re-read on every check, in case a VM is started or stopped meanwhile. Capacity is planned for the new VMs only, as in Running. - scale-down by size removes workers whose VMs are down first. - a node whose VM is down is not drained before it is deleted. The drain waits for pod deletions that a NotReady node can never confirm, so removing the stopped node always failed. - "down" is Stopped, Shutdown, Error, Destroyed or Expunging. Migrating, Unknown and other states may still be running the VM and are treated as before. - the UI shows Scale for a cluster in Alert that doesn't autoscale, with only the size field. Signed-off-by: Brad House --- .../kubernetes/cluster/KubernetesCluster.java | 2 + .../cluster/KubernetesClusterManagerImpl.java | 109 +++++++++++- .../KubernetesClusterScaleWorker.java | 78 +++++++-- .../cluster/utils/KubernetesClusterUtil.java | 29 ++- .../KubernetesClusterManagerImplTest.java | 165 ++++++++++++++++++ .../KubernetesClusterScaleWorkerTest.java | 72 ++++++++ ui/src/config/section/compute.js | 2 +- .../views/compute/ScaleKubernetesCluster.vue | 69 +++++--- 8 files changed, 469 insertions(+), 57 deletions(-) diff --git a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java index ce905b293ff3..a7b74d9fe455 100644 --- a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java +++ b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java @@ -96,6 +96,8 @@ enum State { s_fsm.addTransition(State.Running, Event.AutoscaleRequested, State.Scaling); s_fsm.addTransition(State.Running, Event.ScaleUpRequested, State.Scaling); s_fsm.addTransition(State.Running, Event.ScaleDownRequested, State.Scaling); + s_fsm.addTransition(State.Alert, Event.ScaleUpRequested, State.Scaling); + s_fsm.addTransition(State.Alert, Event.ScaleDownRequested, State.Scaling); s_fsm.addTransition(State.Stopped, Event.ScaleUpRequested, State.ScalingStoppedCluster); s_fsm.addTransition(State.Scaling, Event.OperationSucceeded, State.Running); s_fsm.addTransition(State.Scaling, Event.OperationFailed, State.Alert); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 035554e38237..2136aae62d58 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -1271,7 +1271,98 @@ protected void validateKubernetesClusterScaleSize(final KubernetesClusterVO kube } } - private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd cmd) { + /** + * A cluster in Alert (for example with one worker VM stopped) can still change its size or have + * specific nodes removed, which is often what is needed to get it healthy again. A service + * offering or autoscaling change acts on every node and is refused until the cluster is + * recovered. Passing the offerings and the autoscaling setting the cluster already has is + * allowed, so a client that always sends them with a new size still works. The first control + * node VM, which scaling reaches over SSH, must be running, no control or etcd VM may be down, + * some worker VM must be down (see KubernetesClusterScaleWorker.isVmDown), and only worker nodes + * can be removed. + */ + protected void validateKubernetesClusterStateForScale(final KubernetesClusterVO kubernetesCluster, final Long clusterSize, final List nodeIds, + final Long serviceOfferingId, final Map serviceOfferingNodeTypeMap, + final Boolean isAutoscalingEnabled, final Long minSize, final Long maxSize) { + List validClusterStates = Arrays.asList(KubernetesCluster.State.Created, KubernetesCluster.State.Running, + KubernetesCluster.State.Stopped, KubernetesCluster.State.Alert); + if (!(validClusterStates.contains(kubernetesCluster.getState()))) { + throw new PermissionDeniedException(String.format("Kubernetes cluster %s is in %s state and can not be scaled", kubernetesCluster.getName(), kubernetesCluster.getState().toString())); + } + if (!KubernetesCluster.State.Alert.equals(kubernetesCluster.getState())) { + return; + } + boolean offeringChange = serviceOfferingId != null && !serviceOfferingId.equals(kubernetesCluster.getServiceOfferingId()); + if (MapUtils.isNotEmpty(serviceOfferingNodeTypeMap)) { + for (Map.Entry entry : serviceOfferingNodeTypeMap.entrySet()) { + Long existingOfferingId = getExistingServiceOfferingIdForNodeType(entry.getKey(), kubernetesCluster); + if (existingOfferingId == null) { + existingOfferingId = kubernetesCluster.getServiceOfferingId(); + } + if (entry.getValue() != null && !entry.getValue().equals(existingOfferingId)) { + offeringChange = true; + } + } + } + // As KubernetesClusterScaleWorker.isAutoscalingChanged() + boolean autoscalingChange = isAutoscalingEnabled != null && (isAutoscalingEnabled != kubernetesCluster.getAutoscalingEnabled() || + (minSize != null && !minSize.equals(kubernetesCluster.getMinSize())) || (maxSize != null && !maxSize.equals(kubernetesCluster.getMaxSize()))); + if (offeringChange || autoscalingChange) { + throw new PermissionDeniedException(String.format("Kubernetes cluster %s is in %s state; only its size can be changed or nodes removed. " + + "Recover the cluster before changing its service offering or autoscaling", kubernetesCluster.getName(), kubernetesCluster.getState().toString())); + } + boolean sizeChange = (clusterSize != null && clusterSize != kubernetesCluster.getNodeCount()) || CollectionUtils.isNotEmpty(nodeIds); + if (!sizeChange) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster %s is in %s state; only its size can be changed or nodes removed, " + + "and the request does neither", kubernetesCluster.getName(), kubernetesCluster.getState().toString())); + } + List clusterVmMaps = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + if (clusterVmMaps.size() != kubernetesCluster.getTotalNodeCount()) { + // For example VMs left by a failed scale-up: scaling again would only add to them + throw new PermissionDeniedException(String.format("Kubernetes cluster %s is in %s state and has %d VMs for %d nodes; " + + "recover the cluster before scaling it", kubernetesCluster.getName(), kubernetesCluster.getState().toString(), + clusterVmMaps.size(), kubernetesCluster.getTotalNodeCount())); + } + KubernetesClusterVmMapVO firstControlVmMap = clusterVmMaps.stream().filter(KubernetesClusterVmMapVO::isControlNode).findFirst().orElse(null); + VMInstanceVO firstControlVm = firstControlVmMap != null ? vmInstanceDao.findById(firstControlVmMap.getVmId()) : null; + if (firstControlVm == null || !VirtualMachine.State.Running.equals(firstControlVm.getState())) { + throw new PermissionDeniedException(String.format("Kubernetes cluster %s is in %s state and its first control node VM is not running; " + + "recover the cluster before scaling it", kubernetesCluster.getName(), kubernetesCluster.getState().toString())); + } + boolean controlPlaneVmDown = false; + boolean workerVmDown = false; + for (KubernetesClusterVmMapVO vmMap : clusterVmMaps) { + VMInstanceVO vm = vmInstanceDao.findById(vmMap.getVmId()); + if (vm == null || !KubernetesClusterScaleWorker.isVmDown(vm.getState())) { + continue; + } + if (vmMap.isControlNode() || vmMap.isEtcdNode()) { + controlPlaneVmDown = true; + } else if (!vmMap.isExternalNode()) { + // External nodes are left alone by scaling (see removeVirtualMachinesFromKubernetesCluster) + workerVmDown = true; + } + } + if (controlPlaneVmDown) { + // With a control or etcd node down, etcd may have lost quorum and new nodes may not join + throw new PermissionDeniedException(String.format("Kubernetes cluster %s is in %s state and a control or etcd node VM is down; " + + "recover the control plane before scaling the cluster", kubernetesCluster.getName(), kubernetesCluster.getState().toString())); + } + if (!workerVmDown) { + // Scaling in Alert works around worker VMs that are down (stopped, in error, ...). With none + // down, the cluster is in Alert for another reason, such as a failed start or upgrade, or a + // VM on a disconnected host, which scaling won't fix. + throw new PermissionDeniedException(String.format("Kubernetes cluster %s is in %s state but none of its worker VMs is stopped or in error, so it is in %s " + + "for another reason, such as a failed start or upgrade, or a VM on a disconnected host; investigate and recover the cluster before scaling it", + kubernetesCluster.getName(), kubernetesCluster.getState().toString(), kubernetesCluster.getState().toString())); + } + if (CollectionUtils.isNotEmpty(nodeIds) && clusterVmMaps.stream().anyMatch(vm -> nodeIds.contains(vm.getVmId()) && (vm.isControlNode() || vm.isEtcdNode() || vm.isExternalNode()))) { + throw new PermissionDeniedException(String.format("Kubernetes cluster %s is in %s state; only worker nodes can be removed from it", + kubernetesCluster.getName(), kubernetesCluster.getState().toString())); + } + } + + private KubernetesClusterVO validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd cmd) { final Long kubernetesClusterId = cmd.getId(); final Long clusterSize = cmd.getClusterSize(); final List nodeIds = cmd.getNodeIds(); @@ -1310,10 +1401,8 @@ private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd if (clusterVersion == null) { throw new CloudRuntimeException(String.format("Invalid Kubernetes version associated with Kubernetes cluster : %s", kubernetesCluster.getName())); } - List validClusterStates = Arrays.asList(KubernetesCluster.State.Created, KubernetesCluster.State.Running, KubernetesCluster.State.Stopped); - if (!(validClusterStates.contains(kubernetesCluster.getState()))) { - throw new PermissionDeniedException(String.format("Kubernetes cluster %s is in %s state and can not be scaled", kubernetesCluster.getName(), kubernetesCluster.getState().toString())); - } + validateKubernetesClusterStateForScale(kubernetesCluster, clusterSize, nodeIds, defaultServiceOfferingId, serviceOfferingNodeTypeMap, + isAutoscalingEnabled, minSize, maxSize); int maxClusterSize = KubernetesMaxClusterSize.valueIn(kubernetesCluster.getAccountId()); if (isAutoscalingEnabled != null && isAutoscalingEnabled) { @@ -1372,6 +1461,7 @@ private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd ensureResourceLimitsForScale(kubernetesCluster, serviceOfferingNodeTypeMap, clusterSize != null ? clusterSize : null, kubernetesCluster.getAccountId()); + return kubernetesCluster; } protected void ensureResourceLimitsForScale(final KubernetesClusterVO cluster, @@ -2190,9 +2280,12 @@ public boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws Clou if (!KubernetesServiceEnabled.value()) { logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); } - validateKubernetesClusterScaleParameters(cmd); - KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(cmd.getId()); - Map nodeToOfferingMap = createNodeTypeToServiceOfferingMap(cmd.getServiceOfferingNodeTypeMap(), cmd.getServiceOfferingId(), kubernetesCluster); + // Use the state the request was validated in, not a fresh read the status scanner may have changed + KubernetesClusterVO kubernetesCluster = validateKubernetesClusterScaleParameters(cmd); + // Offering changes are refused in Alert. With no offerings, the worker only changes the size, + // and doesn't apply the cluster offering to node types whose VMs use another one. + Map nodeToOfferingMap = KubernetesCluster.State.Alert.equals(kubernetesCluster.getState()) ? new HashMap<>() : + createNodeTypeToServiceOfferingMap(cmd.getServiceOfferingNodeTypeMap(), cmd.getServiceOfferingId(), kubernetesCluster); String[] keys = getServiceUserKeys(kubernetesCluster); KubernetesClusterScaleWorker scaleWorker = diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 8f8d3aff87eb..49da7397af0a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -21,9 +21,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import javax.inject.Inject; @@ -294,24 +296,28 @@ private boolean removeKubernetesClusterNode(final String ipAddress, final int po if (StringUtils.isNotEmpty(hostName)) { hostName = hostName.toLowerCase(); } + // A down VM runs no pods, and draining its NotReady node would wait for pod deletions + // that the node can never confirm. Deleting the node object is enough. + final boolean drain = !isVmDown(userVm.getState()); while (retryCounter < retries) { retryCounter++; try { - Pair result = SshHelper.sshExecute(ipAddress, port, getControlNodeLoginUser(), - pkFile, null, String.format("sudo /opt/bin/kubectl drain %s --ignore-daemonsets --delete-emptydir-data", hostName), - 10000, 10000, 60000); - if (!result.first()) { - logger.warn("Draining node: {} on VM: {} in Kubernetes cluster: {} unsuccessful", hostName, userVm, kubernetesCluster); - } else { - result = SshHelper.sshExecute(ipAddress, port, getControlNodeLoginUser(), - pkFile, null, String.format("sudo /opt/bin/kubectl delete node %s", hostName), - 10000, 10000, 30000); - if (result.first()) { - return true; - } else { - logger.warn("Deleting node: {} on VM: {} in Kubernetes cluster: {} unsuccessful", hostName, userVm, kubernetesCluster); + if (drain) { + Pair result = SshHelper.sshExecute(ipAddress, port, getControlNodeLoginUser(), + pkFile, null, String.format("sudo /opt/bin/kubectl drain %s --ignore-daemonsets --delete-emptydir-data", hostName), + 10000, 10000, 60000); + if (!result.first()) { + logger.warn("Draining node: {} on VM: {} in Kubernetes cluster: {} unsuccessful", hostName, userVm, kubernetesCluster); + break; } } + Pair result = SshHelper.sshExecute(ipAddress, port, getControlNodeLoginUser(), + pkFile, null, String.format("sudo /opt/bin/kubectl delete node %s", hostName), + 10000, 10000, 30000); + if (result.first()) { + return true; + } + logger.warn("Deleting node: {} on VM: {} in Kubernetes cluster: {} unsuccessful", hostName, userVm, kubernetesCluster); break; } catch (Exception e) { String msg = String.format("Failed to remove Kubernetes cluster: %s node: %s on VM: %s", kubernetesCluster, hostName, userVm); @@ -363,7 +369,7 @@ private void validateKubernetesClusterScaleSizeParameters() throws CloudRuntimeE final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); VMTemplateVO clusterTemplate = templateDao.findById(kubernetesCluster.getTemplateId()); try { - if (originalState.equals(KubernetesCluster.State.Running)) { + if (KubernetesCluster.State.Running.equals(originalState) || KubernetesCluster.State.Alert.equals(originalState)) { plan(newVmRequiredCount, zone, clusterServiceOffering, kubernetesCluster.getDomainId(), kubernetesCluster.getAccountId(), clusterTemplate.getHypervisorType(), clusterTemplate.getArch()); } else { plan(kubernetesCluster.getTotalNodeCount() + newVmRequiredCount, zone, clusterServiceOffering, kubernetesCluster.getDomainId(), kubernetesCluster.getAccountId(), clusterTemplate.getHypervisorType(), clusterTemplate.getArch()); @@ -474,10 +480,41 @@ private void scaleDownKubernetesClusterSize() throws CloudRuntimeException { removeNodesFromCluster(vmList); } + /** + * Whether the VM is known not to be running, so its node can't be Ready and runs no pods. + * Other states that aren't Running, such as Migrating or Unknown (a disconnected host), are + * not treated as down, as the VM may still be running. + */ + public static boolean isVmDown(VirtualMachine.State state) { + return VirtualMachine.State.Stopped.equals(state) || VirtualMachine.State.Shutdown.equals(state) || + VirtualMachine.State.Error.equals(state) || VirtualMachine.State.Destroyed.equals(state) || + VirtualMachine.State.Expunging.equals(state); + } + + private boolean isVmDown(long vmId) { + VMInstanceVO vm = vmInstanceDao.findById(vmId); + return vm != null && isVmDown(vm.getState()); + } + + /** + * The number of the cluster's Kubernetes nodes whose VMs are down, leaving out etcd nodes, + * which aren't Kubernetes nodes, and the given VMs. + */ + protected long getDownNodeCount(Set excludedVmIds) { + return getKubernetesClusterVMMaps().stream() + .filter(vm -> !vm.isEtcdNode() && !excludedVmIds.contains(vm.getVmId()) && isVmDown(vm.getVmId())) + .count(); + } + public List getWorkerNodesToRemove() { List workerVMsMap = getKubernetesClusterVMMaps().stream() .filter(vm -> !vm.isExternalNode() && !vm.isControlNode() && !vm.isEtcdNode()) .collect(Collectors.toList()); + // Nodes are removed from the end of the list: put workers whose VMs are down last, so a + // scale-down removes those first. The sort is stable, so the order is otherwise kept. + Set downVmIds = workerVMsMap.stream().map(KubernetesClusterVmMapVO::getVmId) + .filter(this::isVmDown).collect(Collectors.toSet()); + workerVMsMap.sort(Comparator.comparing(vm -> downVmIds.contains(vm.getVmId()))); int totalWorkerNodes = workerVMsMap.size(); int desiredWorkerNodes = clusterSize == null ? (int) kubernetesCluster.getNodeCount() : clusterSize.intValue(); int toRemoveCount = Math.max(0, totalWorkerNodes - desiredWorkerNodes); @@ -514,8 +551,11 @@ private void scaleUpKubernetesClusterSize(final long newVmCount) throws CloudRun attachIsoKubernetesVMs(clusterVMs); KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(kubernetesCluster.getId()); kubernetesClusterVO.setNodeCount(clusterSize); + // Existing nodes whose VMs are down (the usual reason a cluster is in Alert) can't become + // ready. The new nodes are always expected to become ready, even if a new VM goes down. + final Set newVmIds = clusterVMs.stream().map(InternalIdentity::getId).collect(Collectors.toSet()); boolean readyNodesCountValid = KubernetesClusterUtil.validateKubernetesClusterReadyNodesCount(kubernetesClusterVO, publicIpAddress, sshPort, - getControlNodeLoginUser(), sshKeyFile, scaleTimeoutTime, 15000); + getControlNodeLoginUser(), sshKeyFile, () -> kubernetesClusterVO.getTotalNodeCount() - getDownNodeCount(newVmIds), scaleTimeoutTime, 15000); detachIsoKubernetesVMs(clusterVMs); deleteTemplateLaunchPermission(); if (!readyNodesCountValid) { // Scaling failed @@ -574,6 +614,14 @@ public boolean scaleCluster() throws CloudRuntimeException { scaleTimeoutTime = System.currentTimeMillis() + KubernetesClusterService.KubernetesClusterScaleTimeout.value() * 1000; final long originalClusterSize = kubernetesCluster.getNodeCount(); + // The request was validated for the state it was accepted in, for example what Alert allows, + // and another operation may have started since + KubernetesClusterVO currentCluster = kubernetesClusterDao.findById(kubernetesCluster.getId()); + if (currentCluster != null && !currentCluster.getState().equals(originalState)) { + logAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster : %s failed, it went from %s to %s state after the request was accepted, please retry", + kubernetesCluster.getName(), originalState, currentCluster.getState())); + } + // DEFAULT node type means only the global service offering has been set for the Kubernetes cluster boolean scaleClusterDefaultOffering = serviceOfferingNodeTypeMap.containsKey(DEFAULT.name()); if (scaleClusterDefaultOffering) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java index 00625f6e076d..8f67f373fad7 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java @@ -25,6 +25,7 @@ import java.net.Socket; import java.net.URL; import java.security.SecureRandom; +import java.util.function.LongSupplier; import java.util.stream.Collectors; import javax.net.ssl.HttpsURLConnection; @@ -283,21 +284,37 @@ public static boolean validateKubernetesClusterReadyNodesCount(final KubernetesC final String ipAddress, final int port, final String user, final File sshKeyFile, final long timeoutTime, final long waitDuration) { + return validateKubernetesClusterReadyNodesCount(kubernetesCluster, ipAddress, port, user, sshKeyFile, + kubernetesCluster::getTotalNodeCount, timeoutTime, waitDuration); + } + + /** + * Waits until the cluster has the expected number of ready nodes, for when some of the + * cluster's nodes can't be ready (their VMs are down). The expected count is read again on + * every check, as such a VM may be started, or another one stopped, while this waits. + */ + public static boolean validateKubernetesClusterReadyNodesCount(final KubernetesCluster kubernetesCluster, + final String ipAddress, final int port, + final String user, final File sshKeyFile, + final LongSupplier expectedReadyNodesCountSupplier, + final long timeoutTime, final long waitDuration) { while (System.currentTimeMillis() < timeoutTime) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Checking ready nodes for the Kubernetes cluster %s with total %d provisioned nodes", kubernetesCluster, kubernetesCluster.getTotalNodeCount())); - } try { + long expectedReadyNodesCount = expectedReadyNodesCountSupplier.getAsLong(); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Checking ready nodes for the Kubernetes cluster %s with total %d provisioned nodes, %d expected to be ready", + kubernetesCluster, kubernetesCluster.getTotalNodeCount(), expectedReadyNodesCount)); + } int nodesCount = KubernetesClusterUtil.getKubernetesClusterReadyNodesCount(kubernetesCluster, ipAddress, port, user, sshKeyFile); - if (nodesCount == kubernetesCluster.getTotalNodeCount()) { + if (nodesCount == expectedReadyNodesCount) { if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Kubernetes cluster %s has %d ready nodes now", kubernetesCluster, kubernetesCluster.getTotalNodeCount())); + LOGGER.info(String.format("Kubernetes cluster %s has %d ready nodes now", kubernetesCluster, expectedReadyNodesCount)); } return true; } else { if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Kubernetes cluster %s has total %d provisioned nodes while %d ready now", kubernetesCluster, kubernetesCluster.getTotalNodeCount(), nodesCount)); + LOGGER.debug(String.format("Kubernetes cluster %s has %d nodes expected to be ready while %d ready now", kubernetesCluster, expectedReadyNodesCount, nodesCount)); } } } catch (Exception e) { diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java index 2a381f282de2..849d9a0ac225 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java @@ -45,6 +45,7 @@ import com.cloud.utils.Pair; import com.cloud.utils.net.NetUtils; import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachinesToKubernetesClusterCmd; @@ -441,4 +442,168 @@ public void testGetCksClusterPreferredArchSameArch() { String cksClusterPreferredArch = kubernetesClusterManager.getCksClusterPreferredArch(systemVMArch, cksIso); Assert.assertEquals(CPU.CPUArch.amd64.getType(), cksClusterPreferredArch); } + + private KubernetesClusterVO mockClusterForScaleState(KubernetesCluster.State state, VirtualMachine.State controlVmState) { + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + Mockito.when(cluster.getState()).thenReturn(state); + Mockito.lenient().when(cluster.getId()).thenReturn(1L); + Mockito.lenient().when(cluster.getServiceOfferingId()).thenReturn(1L); + Mockito.lenient().when(cluster.getWorkerNodeServiceOfferingId()).thenReturn(2L); + Mockito.lenient().when(cluster.getNodeCount()).thenReturn(2L); + Mockito.lenient().when(cluster.getTotalNodeCount()).thenReturn(2L); + Mockito.lenient().when(cluster.getAutoscalingEnabled()).thenReturn(false); + KubernetesClusterVmMapVO controlVmMap = Mockito.mock(KubernetesClusterVmMapVO.class); + Mockito.lenient().when(controlVmMap.getVmId()).thenReturn(100L); + Mockito.lenient().when(controlVmMap.isControlNode()).thenReturn(true); + KubernetesClusterVmMapVO workerVmMap = Mockito.mock(KubernetesClusterVmMapVO.class); + Mockito.lenient().when(workerVmMap.getVmId()).thenReturn(10L); + Mockito.lenient().when(kubernetesClusterVmMapDao.listByClusterId(1L)).thenReturn(List.of(controlVmMap, workerVmMap)); + VMInstanceVO controlVm = Mockito.mock(VMInstanceVO.class); + Mockito.lenient().when(controlVm.getState()).thenReturn(controlVmState); + Mockito.lenient().when(vmInstanceDao.findById(100L)).thenReturn(controlVm); + // The usual reason for Alert: a worker VM is stopped + VMInstanceVO workerVm = Mockito.mock(VMInstanceVO.class); + Mockito.lenient().when(workerVm.getState()).thenReturn(VirtualMachine.State.Stopped); + Mockito.lenient().when(vmInstanceDao.findById(10L)).thenReturn(workerVm); + return cluster; + } + + private KubernetesClusterVO mockClusterForScaleState(KubernetesCluster.State state) { + return mockClusterForScaleState(state, VirtualMachine.State.Running); + } + + @Test + public void testValidateKubernetesClusterStateForScaleAlertSize() { + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 3L, null, null, null, null, null, null); + } + + @Test + public void testValidateKubernetesClusterStateForScaleAlertNodeIds() { + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, null, List.of(10L), null, null, null, null, null); + } + + @Test + public void testValidateKubernetesClusterStateForScaleAlertSizeWithCurrentSettings() { + // The offerings and autoscaling setting the cluster already has may be sent along with the new size + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 3L, null, 1L, Map.of(WORKER.name(), 2L), false, null, null); + } + + @Test(expected = PermissionDeniedException.class) + public void testValidateKubernetesClusterStateForScaleAlertOfferingChange() { + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 3L, null, 5L, null, null, null, null); + } + + @Test(expected = PermissionDeniedException.class) + public void testValidateKubernetesClusterStateForScaleAlertNodeOfferingChange() { + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 3L, null, null, Map.of(WORKER.name(), 5L), null, null, null); + } + + @Test(expected = PermissionDeniedException.class) + public void testValidateKubernetesClusterStateForScaleAlertAutoscaling() { + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, null, null, null, null, true, null, null); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidateKubernetesClusterStateForScaleAlertNothingToDo() { + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, null, null, 1L, null, null, null, null); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidateKubernetesClusterStateForScaleAlertSameSize() { + // Would do nothing, then move the cluster from Alert to Running + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 2L, null, null, null, null, null, null); + } + + @Test(expected = PermissionDeniedException.class) + public void testValidateKubernetesClusterStateForScaleAlertControlNodeDown() { + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert, VirtualMachine.State.Stopped); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 3L, null, null, null, null, null, null); + } + + @Test(expected = PermissionDeniedException.class) + public void testValidateKubernetesClusterStateForScaleAlertAutoscalingSizeChange() { + // Autoscaling stays disabled, but minsize/maxsize differ: the worker treats it as an autoscaling change + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 3L, null, null, null, false, 1L, 3L); + } + + @Test(expected = PermissionDeniedException.class) + public void testValidateKubernetesClusterStateForScaleAlertExtraVms() { + // For example a worker VM left by a failed scale-up + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + Mockito.when(cluster.getTotalNodeCount()).thenReturn(1L); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 3L, null, null, null, null, null, null); + } + + @Test(expected = PermissionDeniedException.class) + public void testValidateKubernetesClusterStateForScaleAlertAllVmsRunning() { + // In Alert for another reason, such as a failed upgrade + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + VMInstanceVO workerVm = Mockito.mock(VMInstanceVO.class); + Mockito.when(workerVm.getState()).thenReturn(VirtualMachine.State.Running); + Mockito.when(vmInstanceDao.findById(10L)).thenReturn(workerVm); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 3L, null, null, null, null, null, null); + } + + @Test(expected = PermissionDeniedException.class) + public void testValidateKubernetesClusterStateForScaleAlertNoVmDown() { + // A VM on a disconnected host may still be running: the worker can't work around it + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + VMInstanceVO workerVm = Mockito.mock(VMInstanceVO.class); + Mockito.when(workerVm.getState()).thenReturn(VirtualMachine.State.Unknown); + Mockito.when(vmInstanceDao.findById(10L)).thenReturn(workerVm); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 3L, null, null, null, null, null, null); + } + + @Test(expected = PermissionDeniedException.class) + public void testValidateKubernetesClusterStateForScaleAlertEtcdVmDown() { + // etcd may have lost quorum, so new nodes can't join + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + Mockito.when(cluster.getTotalNodeCount()).thenReturn(3L); + KubernetesClusterVmMapVO etcdVmMap = Mockito.mock(KubernetesClusterVmMapVO.class); + Mockito.when(etcdVmMap.getVmId()).thenReturn(200L); + Mockito.when(etcdVmMap.isEtcdNode()).thenReturn(true); + List vmMaps = new ArrayList<>(kubernetesClusterVmMapDao.listByClusterId(1L)); + vmMaps.add(etcdVmMap); + Mockito.when(kubernetesClusterVmMapDao.listByClusterId(1L)).thenReturn(vmMaps); + VMInstanceVO etcdVm = Mockito.mock(VMInstanceVO.class); + Mockito.when(etcdVm.getState()).thenReturn(VirtualMachine.State.Stopped); + Mockito.when(vmInstanceDao.findById(200L)).thenReturn(etcdVm); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 3L, null, null, null, null, null, null); + } + + @Test(expected = PermissionDeniedException.class) + public void testValidateKubernetesClusterStateForScaleAlertOnlyExternalNodeDown() { + // Scaling never acts on external nodes, so it can't work around one that is down + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + KubernetesClusterVmMapVO workerVmMap = kubernetesClusterVmMapDao.listByClusterId(1L).get(1); + Mockito.when(workerVmMap.isExternalNode()).thenReturn(true); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 3L, null, null, null, null, null, null); + } + + @Test(expected = PermissionDeniedException.class) + public void testValidateKubernetesClusterStateForScaleAlertRemoveControlNode() { + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, null, List.of(100L), null, null, null, null, null); + } + + @Test + public void testValidateKubernetesClusterStateForScaleRunningOfferingChange() { + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Running); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, null, null, 5L, null, null, null, null); + } + + @Test(expected = PermissionDeniedException.class) + public void testValidateKubernetesClusterStateForScaleUpgrading() { + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Upgrading); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 3L, null, null, null, null, null, null); + } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java index c9299bdbaa67..4c8cacf89eef 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java @@ -25,7 +25,10 @@ import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.utils.Pair; import com.cloud.vm.UserVmVO; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.UserVmDao; +import com.cloud.vm.dao.VMInstanceDao; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -36,7 +39,9 @@ import org.mockito.junit.MockitoJUnitRunner; import java.util.Arrays; +import java.util.Collections; import java.util.List; +import java.util.Set; import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL; import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.DEFAULT; @@ -54,6 +59,8 @@ public class KubernetesClusterScaleWorkerTest { private KubernetesClusterVmMapDao kubernetesClusterVmMapDao; @Mock private UserVmDao userVmDao; + @Mock + private VMInstanceDao vmInstanceDao; private KubernetesClusterScaleWorker worker; @@ -134,6 +141,7 @@ public void testGetWorkerNodesToRemoveForDownsize_singleRemoval() { KubernetesCluster kubernetesCluster = Mockito.mock(KubernetesCluster.class); KubernetesClusterManagerImpl clusterManager = Mockito.mock(KubernetesClusterManagerImpl.class); KubernetesClusterScaleWorker worker = new KubernetesClusterScaleWorker(kubernetesCluster, new java.util.HashMap<>(), 2L, null, false, null, null, clusterManager); + worker.vmInstanceDao = vmInstanceDao; KubernetesClusterScaleWorker spyWorker = Mockito.spy(worker); KubernetesClusterVmMapVO vm1 = Mockito.mock(KubernetesClusterVmMapVO.class); @@ -164,6 +172,7 @@ public void testGetWorkerNodesToRemoveForDownsize_noRemoval() { KubernetesCluster kubernetesCluster = Mockito.mock(KubernetesCluster.class); KubernetesClusterScaleWorker worker = new KubernetesClusterScaleWorker(kubernetesCluster, new java.util.HashMap<>(), 3L, null, false, null, null, clusterManager); + worker.vmInstanceDao = vmInstanceDao; KubernetesClusterScaleWorker spyWorker = Mockito.spy(worker); KubernetesClusterVmMapVO vm1 = Mockito.mock(KubernetesClusterVmMapVO.class); @@ -187,4 +196,67 @@ public void testGetWorkerNodesToRemoveForDownsize_noRemoval() { Assert.assertTrue(toRemove.isEmpty()); } + + private KubernetesClusterVmMapVO mockWorkerVmMap(long vmId, VirtualMachine.State state) { + KubernetesClusterVmMapVO vmMap = Mockito.mock(KubernetesClusterVmMapVO.class); + Mockito.lenient().when(vmMap.isExternalNode()).thenReturn(false); + Mockito.lenient().when(vmMap.isControlNode()).thenReturn(false); + Mockito.lenient().when(vmMap.isEtcdNode()).thenReturn(false); + Mockito.lenient().when(vmMap.getVmId()).thenReturn(vmId); + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.lenient().when(vm.getState()).thenReturn(state); + Mockito.lenient().when(vmInstanceDao.findById(vmId)).thenReturn(vm); + return vmMap; + } + + @Test + public void testGetWorkerNodesToRemovePrefersNodesDown() { + KubernetesClusterScaleWorker worker = new KubernetesClusterScaleWorker(kubernetesCluster, new java.util.HashMap<>(), 2L, null, false, null, null, clusterManager); + worker.vmInstanceDao = vmInstanceDao; + KubernetesClusterScaleWorker spyWorker = Mockito.spy(worker); + + KubernetesClusterVmMapVO vm1 = mockWorkerVmMap(1L, VirtualMachine.State.Running); + KubernetesClusterVmMapVO vm2 = mockWorkerVmMap(2L, VirtualMachine.State.Stopped); + KubernetesClusterVmMapVO vm3 = mockWorkerVmMap(3L, VirtualMachine.State.Running); + KubernetesClusterVmMapVO vm4 = mockWorkerVmMap(4L, VirtualMachine.State.Running); + Mockito.doReturn(Arrays.asList(vm1, vm2, vm3, vm4)).when(spyWorker).getKubernetesClusterVMMaps(); + + List toRemove = spyWorker.getWorkerNodesToRemove(); + + // The stopped node goes first, then the last running node as before + Assert.assertEquals(2, toRemove.size()); + Assert.assertTrue(toRemove.contains(vm2)); + Assert.assertTrue(toRemove.contains(vm4)); + } + + @Test + public void testGetDownNodeCountIgnoresEtcdNodes() { + KubernetesClusterScaleWorker worker = new KubernetesClusterScaleWorker(kubernetesCluster, new java.util.HashMap<>(), 2L, null, false, null, null, clusterManager); + worker.vmInstanceDao = vmInstanceDao; + KubernetesClusterScaleWorker spyWorker = Mockito.spy(worker); + + KubernetesClusterVmMapVO running = mockWorkerVmMap(1L, VirtualMachine.State.Running); + KubernetesClusterVmMapVO stopped = mockWorkerVmMap(2L, VirtualMachine.State.Stopped); + KubernetesClusterVmMapVO etcd = mockWorkerVmMap(3L, VirtualMachine.State.Stopped); + Mockito.when(etcd.isEtcdNode()).thenReturn(true); + Mockito.doReturn(Arrays.asList(running, stopped, etcd)).when(spyWorker).getKubernetesClusterVMMaps(); + + Assert.assertEquals(1L, spyWorker.getDownNodeCount(Collections.emptySet())); + // Newly provisioned VMs aren't left out of the ready nodes + Assert.assertEquals(0L, spyWorker.getDownNodeCount(Set.of(2L))); + } + + @Test + public void testIsVmDown() { + Assert.assertTrue(KubernetesClusterScaleWorker.isVmDown(VirtualMachine.State.Stopped)); + Assert.assertTrue(KubernetesClusterScaleWorker.isVmDown(VirtualMachine.State.Shutdown)); + Assert.assertTrue(KubernetesClusterScaleWorker.isVmDown(VirtualMachine.State.Error)); + Assert.assertTrue(KubernetesClusterScaleWorker.isVmDown(VirtualMachine.State.Destroyed)); + Assert.assertTrue(KubernetesClusterScaleWorker.isVmDown(VirtualMachine.State.Expunging)); + // The VM may still be running in these + Assert.assertFalse(KubernetesClusterScaleWorker.isVmDown(VirtualMachine.State.Running)); + Assert.assertFalse(KubernetesClusterScaleWorker.isVmDown(VirtualMachine.State.Migrating)); + Assert.assertFalse(KubernetesClusterScaleWorker.isVmDown(VirtualMachine.State.Unknown)); + Assert.assertFalse(KubernetesClusterScaleWorker.isVmDown(VirtualMachine.State.Starting)); + } } diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index 4575cf2a0572..070b8e316e9e 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -630,7 +630,7 @@ export default { message: 'message.kubernetes.cluster.scale', docHelp: 'plugins/cloudstack-kubernetes-service.html#scaling-kubernetes-cluster', dataView: true, - show: (record) => { return ['Created', 'Running', 'Stopped'].includes(record.state) && record.clustertype === 'CloudManaged' }, + show: (record) => { return (['Created', 'Running', 'Stopped'].includes(record.state) || (record.state === 'Alert' && !record.autoscalingenabled)) && record.clustertype === 'CloudManaged' }, popup: true, component: shallowRef(defineAsyncComponent(() => import('@/views/compute/ScaleKubernetesCluster.vue'))) }, diff --git a/ui/src/views/compute/ScaleKubernetesCluster.vue b/ui/src/views/compute/ScaleKubernetesCluster.vue index 837860061684..e3a11288fd4c 100644 --- a/ui/src/views/compute/ScaleKubernetesCluster.vue +++ b/ui/src/views/compute/ScaleKubernetesCluster.vue @@ -28,7 +28,7 @@ :rules="rules" @finish="handleSubmit" layout="vertical"> - + @@ -47,7 +47,7 @@ - + @@ -66,7 +66,7 @@ - + @@ -85,7 +85,7 @@ - + @@ -104,13 +104,13 @@ - + - +