diff --git a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java index ce905b293ff3..a7b74d9fe455 100644 --- a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java +++ b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java @@ -96,6 +96,8 @@ enum State { s_fsm.addTransition(State.Running, Event.AutoscaleRequested, State.Scaling); s_fsm.addTransition(State.Running, Event.ScaleUpRequested, State.Scaling); s_fsm.addTransition(State.Running, Event.ScaleDownRequested, State.Scaling); + s_fsm.addTransition(State.Alert, Event.ScaleUpRequested, State.Scaling); + s_fsm.addTransition(State.Alert, Event.ScaleDownRequested, State.Scaling); s_fsm.addTransition(State.Stopped, Event.ScaleUpRequested, State.ScalingStoppedCluster); s_fsm.addTransition(State.Scaling, Event.OperationSucceeded, State.Running); s_fsm.addTransition(State.Scaling, Event.OperationFailed, State.Alert); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 035554e38237..2136aae62d58 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -1271,7 +1271,98 @@ protected void validateKubernetesClusterScaleSize(final KubernetesClusterVO kube } } - private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd cmd) { + /** + * A cluster in Alert (for example with one worker VM stopped) can still change its size or have + * specific nodes removed, which is often what is needed to get it healthy again. A service + * offering or autoscaling change acts on every node and is refused until the cluster is + * recovered. Passing the offerings and the autoscaling setting the cluster already has is + * allowed, so a client that always sends them with a new size still works. The first control + * node VM, which scaling reaches over SSH, must be running, no control or etcd VM may be down, + * some worker VM must be down (see KubernetesClusterScaleWorker.isVmDown), and only worker nodes + * can be removed. + */ + protected void validateKubernetesClusterStateForScale(final KubernetesClusterVO kubernetesCluster, final Long clusterSize, final List nodeIds, + final Long serviceOfferingId, final Map serviceOfferingNodeTypeMap, + final Boolean isAutoscalingEnabled, final Long minSize, final Long maxSize) { + List validClusterStates = Arrays.asList(KubernetesCluster.State.Created, KubernetesCluster.State.Running, + KubernetesCluster.State.Stopped, KubernetesCluster.State.Alert); + if (!(validClusterStates.contains(kubernetesCluster.getState()))) { + throw new PermissionDeniedException(String.format("Kubernetes cluster %s is in %s state and can not be scaled", kubernetesCluster.getName(), kubernetesCluster.getState().toString())); + } + if (!KubernetesCluster.State.Alert.equals(kubernetesCluster.getState())) { + return; + } + boolean offeringChange = serviceOfferingId != null && !serviceOfferingId.equals(kubernetesCluster.getServiceOfferingId()); + if (MapUtils.isNotEmpty(serviceOfferingNodeTypeMap)) { + for (Map.Entry entry : serviceOfferingNodeTypeMap.entrySet()) { + Long existingOfferingId = getExistingServiceOfferingIdForNodeType(entry.getKey(), kubernetesCluster); + if (existingOfferingId == null) { + existingOfferingId = kubernetesCluster.getServiceOfferingId(); + } + if (entry.getValue() != null && !entry.getValue().equals(existingOfferingId)) { + offeringChange = true; + } + } + } + // As KubernetesClusterScaleWorker.isAutoscalingChanged() + boolean autoscalingChange = isAutoscalingEnabled != null && (isAutoscalingEnabled != kubernetesCluster.getAutoscalingEnabled() || + (minSize != null && !minSize.equals(kubernetesCluster.getMinSize())) || (maxSize != null && !maxSize.equals(kubernetesCluster.getMaxSize()))); + if (offeringChange || autoscalingChange) { + throw new PermissionDeniedException(String.format("Kubernetes cluster %s is in %s state; only its size can be changed or nodes removed. " + + "Recover the cluster before changing its service offering or autoscaling", kubernetesCluster.getName(), kubernetesCluster.getState().toString())); + } + boolean sizeChange = (clusterSize != null && clusterSize != kubernetesCluster.getNodeCount()) || CollectionUtils.isNotEmpty(nodeIds); + if (!sizeChange) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster %s is in %s state; only its size can be changed or nodes removed, " + + "and the request does neither", kubernetesCluster.getName(), kubernetesCluster.getState().toString())); + } + List clusterVmMaps = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + if (clusterVmMaps.size() != kubernetesCluster.getTotalNodeCount()) { + // For example VMs left by a failed scale-up: scaling again would only add to them + throw new PermissionDeniedException(String.format("Kubernetes cluster %s is in %s state and has %d VMs for %d nodes; " + + "recover the cluster before scaling it", kubernetesCluster.getName(), kubernetesCluster.getState().toString(), + clusterVmMaps.size(), kubernetesCluster.getTotalNodeCount())); + } + KubernetesClusterVmMapVO firstControlVmMap = clusterVmMaps.stream().filter(KubernetesClusterVmMapVO::isControlNode).findFirst().orElse(null); + VMInstanceVO firstControlVm = firstControlVmMap != null ? vmInstanceDao.findById(firstControlVmMap.getVmId()) : null; + if (firstControlVm == null || !VirtualMachine.State.Running.equals(firstControlVm.getState())) { + throw new PermissionDeniedException(String.format("Kubernetes cluster %s is in %s state and its first control node VM is not running; " + + "recover the cluster before scaling it", kubernetesCluster.getName(), kubernetesCluster.getState().toString())); + } + boolean controlPlaneVmDown = false; + boolean workerVmDown = false; + for (KubernetesClusterVmMapVO vmMap : clusterVmMaps) { + VMInstanceVO vm = vmInstanceDao.findById(vmMap.getVmId()); + if (vm == null || !KubernetesClusterScaleWorker.isVmDown(vm.getState())) { + continue; + } + if (vmMap.isControlNode() || vmMap.isEtcdNode()) { + controlPlaneVmDown = true; + } else if (!vmMap.isExternalNode()) { + // External nodes are left alone by scaling (see removeVirtualMachinesFromKubernetesCluster) + workerVmDown = true; + } + } + if (controlPlaneVmDown) { + // With a control or etcd node down, etcd may have lost quorum and new nodes may not join + throw new PermissionDeniedException(String.format("Kubernetes cluster %s is in %s state and a control or etcd node VM is down; " + + "recover the control plane before scaling the cluster", kubernetesCluster.getName(), kubernetesCluster.getState().toString())); + } + if (!workerVmDown) { + // Scaling in Alert works around worker VMs that are down (stopped, in error, ...). With none + // down, the cluster is in Alert for another reason, such as a failed start or upgrade, or a + // VM on a disconnected host, which scaling won't fix. + throw new PermissionDeniedException(String.format("Kubernetes cluster %s is in %s state but none of its worker VMs is stopped or in error, so it is in %s " + + "for another reason, such as a failed start or upgrade, or a VM on a disconnected host; investigate and recover the cluster before scaling it", + kubernetesCluster.getName(), kubernetesCluster.getState().toString(), kubernetesCluster.getState().toString())); + } + if (CollectionUtils.isNotEmpty(nodeIds) && clusterVmMaps.stream().anyMatch(vm -> nodeIds.contains(vm.getVmId()) && (vm.isControlNode() || vm.isEtcdNode() || vm.isExternalNode()))) { + throw new PermissionDeniedException(String.format("Kubernetes cluster %s is in %s state; only worker nodes can be removed from it", + kubernetesCluster.getName(), kubernetesCluster.getState().toString())); + } + } + + private KubernetesClusterVO validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd cmd) { final Long kubernetesClusterId = cmd.getId(); final Long clusterSize = cmd.getClusterSize(); final List nodeIds = cmd.getNodeIds(); @@ -1310,10 +1401,8 @@ private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd if (clusterVersion == null) { throw new CloudRuntimeException(String.format("Invalid Kubernetes version associated with Kubernetes cluster : %s", kubernetesCluster.getName())); } - List validClusterStates = Arrays.asList(KubernetesCluster.State.Created, KubernetesCluster.State.Running, KubernetesCluster.State.Stopped); - if (!(validClusterStates.contains(kubernetesCluster.getState()))) { - throw new PermissionDeniedException(String.format("Kubernetes cluster %s is in %s state and can not be scaled", kubernetesCluster.getName(), kubernetesCluster.getState().toString())); - } + validateKubernetesClusterStateForScale(kubernetesCluster, clusterSize, nodeIds, defaultServiceOfferingId, serviceOfferingNodeTypeMap, + isAutoscalingEnabled, minSize, maxSize); int maxClusterSize = KubernetesMaxClusterSize.valueIn(kubernetesCluster.getAccountId()); if (isAutoscalingEnabled != null && isAutoscalingEnabled) { @@ -1372,6 +1461,7 @@ private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd ensureResourceLimitsForScale(kubernetesCluster, serviceOfferingNodeTypeMap, clusterSize != null ? clusterSize : null, kubernetesCluster.getAccountId()); + return kubernetesCluster; } protected void ensureResourceLimitsForScale(final KubernetesClusterVO cluster, @@ -2190,9 +2280,12 @@ public boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws Clou if (!KubernetesServiceEnabled.value()) { logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); } - validateKubernetesClusterScaleParameters(cmd); - KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(cmd.getId()); - Map nodeToOfferingMap = createNodeTypeToServiceOfferingMap(cmd.getServiceOfferingNodeTypeMap(), cmd.getServiceOfferingId(), kubernetesCluster); + // Use the state the request was validated in, not a fresh read the status scanner may have changed + KubernetesClusterVO kubernetesCluster = validateKubernetesClusterScaleParameters(cmd); + // Offering changes are refused in Alert. With no offerings, the worker only changes the size, + // and doesn't apply the cluster offering to node types whose VMs use another one. + Map nodeToOfferingMap = KubernetesCluster.State.Alert.equals(kubernetesCluster.getState()) ? new HashMap<>() : + createNodeTypeToServiceOfferingMap(cmd.getServiceOfferingNodeTypeMap(), cmd.getServiceOfferingId(), kubernetesCluster); String[] keys = getServiceUserKeys(kubernetesCluster); KubernetesClusterScaleWorker scaleWorker = diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 8f8d3aff87eb..49da7397af0a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -21,9 +21,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import javax.inject.Inject; @@ -294,24 +296,28 @@ private boolean removeKubernetesClusterNode(final String ipAddress, final int po if (StringUtils.isNotEmpty(hostName)) { hostName = hostName.toLowerCase(); } + // A down VM runs no pods, and draining its NotReady node would wait for pod deletions + // that the node can never confirm. Deleting the node object is enough. + final boolean drain = !isVmDown(userVm.getState()); while (retryCounter < retries) { retryCounter++; try { - Pair result = SshHelper.sshExecute(ipAddress, port, getControlNodeLoginUser(), - pkFile, null, String.format("sudo /opt/bin/kubectl drain %s --ignore-daemonsets --delete-emptydir-data", hostName), - 10000, 10000, 60000); - if (!result.first()) { - logger.warn("Draining node: {} on VM: {} in Kubernetes cluster: {} unsuccessful", hostName, userVm, kubernetesCluster); - } else { - result = SshHelper.sshExecute(ipAddress, port, getControlNodeLoginUser(), - pkFile, null, String.format("sudo /opt/bin/kubectl delete node %s", hostName), - 10000, 10000, 30000); - if (result.first()) { - return true; - } else { - logger.warn("Deleting node: {} on VM: {} in Kubernetes cluster: {} unsuccessful", hostName, userVm, kubernetesCluster); + if (drain) { + Pair result = SshHelper.sshExecute(ipAddress, port, getControlNodeLoginUser(), + pkFile, null, String.format("sudo /opt/bin/kubectl drain %s --ignore-daemonsets --delete-emptydir-data", hostName), + 10000, 10000, 60000); + if (!result.first()) { + logger.warn("Draining node: {} on VM: {} in Kubernetes cluster: {} unsuccessful", hostName, userVm, kubernetesCluster); + break; } } + Pair result = SshHelper.sshExecute(ipAddress, port, getControlNodeLoginUser(), + pkFile, null, String.format("sudo /opt/bin/kubectl delete node %s", hostName), + 10000, 10000, 30000); + if (result.first()) { + return true; + } + logger.warn("Deleting node: {} on VM: {} in Kubernetes cluster: {} unsuccessful", hostName, userVm, kubernetesCluster); break; } catch (Exception e) { String msg = String.format("Failed to remove Kubernetes cluster: %s node: %s on VM: %s", kubernetesCluster, hostName, userVm); @@ -363,7 +369,7 @@ private void validateKubernetesClusterScaleSizeParameters() throws CloudRuntimeE final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); VMTemplateVO clusterTemplate = templateDao.findById(kubernetesCluster.getTemplateId()); try { - if (originalState.equals(KubernetesCluster.State.Running)) { + if (KubernetesCluster.State.Running.equals(originalState) || KubernetesCluster.State.Alert.equals(originalState)) { plan(newVmRequiredCount, zone, clusterServiceOffering, kubernetesCluster.getDomainId(), kubernetesCluster.getAccountId(), clusterTemplate.getHypervisorType(), clusterTemplate.getArch()); } else { plan(kubernetesCluster.getTotalNodeCount() + newVmRequiredCount, zone, clusterServiceOffering, kubernetesCluster.getDomainId(), kubernetesCluster.getAccountId(), clusterTemplate.getHypervisorType(), clusterTemplate.getArch()); @@ -474,10 +480,41 @@ private void scaleDownKubernetesClusterSize() throws CloudRuntimeException { removeNodesFromCluster(vmList); } + /** + * Whether the VM is known not to be running, so its node can't be Ready and runs no pods. + * Other states that aren't Running, such as Migrating or Unknown (a disconnected host), are + * not treated as down, as the VM may still be running. + */ + public static boolean isVmDown(VirtualMachine.State state) { + return VirtualMachine.State.Stopped.equals(state) || VirtualMachine.State.Shutdown.equals(state) || + VirtualMachine.State.Error.equals(state) || VirtualMachine.State.Destroyed.equals(state) || + VirtualMachine.State.Expunging.equals(state); + } + + private boolean isVmDown(long vmId) { + VMInstanceVO vm = vmInstanceDao.findById(vmId); + return vm != null && isVmDown(vm.getState()); + } + + /** + * The number of the cluster's Kubernetes nodes whose VMs are down, leaving out etcd nodes, + * which aren't Kubernetes nodes, and the given VMs. + */ + protected long getDownNodeCount(Set excludedVmIds) { + return getKubernetesClusterVMMaps().stream() + .filter(vm -> !vm.isEtcdNode() && !excludedVmIds.contains(vm.getVmId()) && isVmDown(vm.getVmId())) + .count(); + } + public List getWorkerNodesToRemove() { List workerVMsMap = getKubernetesClusterVMMaps().stream() .filter(vm -> !vm.isExternalNode() && !vm.isControlNode() && !vm.isEtcdNode()) .collect(Collectors.toList()); + // Nodes are removed from the end of the list: put workers whose VMs are down last, so a + // scale-down removes those first. The sort is stable, so the order is otherwise kept. + Set downVmIds = workerVMsMap.stream().map(KubernetesClusterVmMapVO::getVmId) + .filter(this::isVmDown).collect(Collectors.toSet()); + workerVMsMap.sort(Comparator.comparing(vm -> downVmIds.contains(vm.getVmId()))); int totalWorkerNodes = workerVMsMap.size(); int desiredWorkerNodes = clusterSize == null ? (int) kubernetesCluster.getNodeCount() : clusterSize.intValue(); int toRemoveCount = Math.max(0, totalWorkerNodes - desiredWorkerNodes); @@ -514,8 +551,11 @@ private void scaleUpKubernetesClusterSize(final long newVmCount) throws CloudRun attachIsoKubernetesVMs(clusterVMs); KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(kubernetesCluster.getId()); kubernetesClusterVO.setNodeCount(clusterSize); + // Existing nodes whose VMs are down (the usual reason a cluster is in Alert) can't become + // ready. The new nodes are always expected to become ready, even if a new VM goes down. + final Set newVmIds = clusterVMs.stream().map(InternalIdentity::getId).collect(Collectors.toSet()); boolean readyNodesCountValid = KubernetesClusterUtil.validateKubernetesClusterReadyNodesCount(kubernetesClusterVO, publicIpAddress, sshPort, - getControlNodeLoginUser(), sshKeyFile, scaleTimeoutTime, 15000); + getControlNodeLoginUser(), sshKeyFile, () -> kubernetesClusterVO.getTotalNodeCount() - getDownNodeCount(newVmIds), scaleTimeoutTime, 15000); detachIsoKubernetesVMs(clusterVMs); deleteTemplateLaunchPermission(); if (!readyNodesCountValid) { // Scaling failed @@ -574,6 +614,14 @@ public boolean scaleCluster() throws CloudRuntimeException { scaleTimeoutTime = System.currentTimeMillis() + KubernetesClusterService.KubernetesClusterScaleTimeout.value() * 1000; final long originalClusterSize = kubernetesCluster.getNodeCount(); + // The request was validated for the state it was accepted in, for example what Alert allows, + // and another operation may have started since + KubernetesClusterVO currentCluster = kubernetesClusterDao.findById(kubernetesCluster.getId()); + if (currentCluster != null && !currentCluster.getState().equals(originalState)) { + logAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster : %s failed, it went from %s to %s state after the request was accepted, please retry", + kubernetesCluster.getName(), originalState, currentCluster.getState())); + } + // DEFAULT node type means only the global service offering has been set for the Kubernetes cluster boolean scaleClusterDefaultOffering = serviceOfferingNodeTypeMap.containsKey(DEFAULT.name()); if (scaleClusterDefaultOffering) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java index 00625f6e076d..8f67f373fad7 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java @@ -25,6 +25,7 @@ import java.net.Socket; import java.net.URL; import java.security.SecureRandom; +import java.util.function.LongSupplier; import java.util.stream.Collectors; import javax.net.ssl.HttpsURLConnection; @@ -283,21 +284,37 @@ public static boolean validateKubernetesClusterReadyNodesCount(final KubernetesC final String ipAddress, final int port, final String user, final File sshKeyFile, final long timeoutTime, final long waitDuration) { + return validateKubernetesClusterReadyNodesCount(kubernetesCluster, ipAddress, port, user, sshKeyFile, + kubernetesCluster::getTotalNodeCount, timeoutTime, waitDuration); + } + + /** + * Waits until the cluster has the expected number of ready nodes, for when some of the + * cluster's nodes can't be ready (their VMs are down). The expected count is read again on + * every check, as such a VM may be started, or another one stopped, while this waits. + */ + public static boolean validateKubernetesClusterReadyNodesCount(final KubernetesCluster kubernetesCluster, + final String ipAddress, final int port, + final String user, final File sshKeyFile, + final LongSupplier expectedReadyNodesCountSupplier, + final long timeoutTime, final long waitDuration) { while (System.currentTimeMillis() < timeoutTime) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Checking ready nodes for the Kubernetes cluster %s with total %d provisioned nodes", kubernetesCluster, kubernetesCluster.getTotalNodeCount())); - } try { + long expectedReadyNodesCount = expectedReadyNodesCountSupplier.getAsLong(); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Checking ready nodes for the Kubernetes cluster %s with total %d provisioned nodes, %d expected to be ready", + kubernetesCluster, kubernetesCluster.getTotalNodeCount(), expectedReadyNodesCount)); + } int nodesCount = KubernetesClusterUtil.getKubernetesClusterReadyNodesCount(kubernetesCluster, ipAddress, port, user, sshKeyFile); - if (nodesCount == kubernetesCluster.getTotalNodeCount()) { + if (nodesCount == expectedReadyNodesCount) { if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Kubernetes cluster %s has %d ready nodes now", kubernetesCluster, kubernetesCluster.getTotalNodeCount())); + LOGGER.info(String.format("Kubernetes cluster %s has %d ready nodes now", kubernetesCluster, expectedReadyNodesCount)); } return true; } else { if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Kubernetes cluster %s has total %d provisioned nodes while %d ready now", kubernetesCluster, kubernetesCluster.getTotalNodeCount(), nodesCount)); + LOGGER.debug(String.format("Kubernetes cluster %s has %d nodes expected to be ready while %d ready now", kubernetesCluster, expectedReadyNodesCount, nodesCount)); } } } catch (Exception e) { diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java index 2a381f282de2..849d9a0ac225 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java @@ -45,6 +45,7 @@ import com.cloud.utils.Pair; import com.cloud.utils.net.NetUtils; import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachinesToKubernetesClusterCmd; @@ -441,4 +442,168 @@ public void testGetCksClusterPreferredArchSameArch() { String cksClusterPreferredArch = kubernetesClusterManager.getCksClusterPreferredArch(systemVMArch, cksIso); Assert.assertEquals(CPU.CPUArch.amd64.getType(), cksClusterPreferredArch); } + + private KubernetesClusterVO mockClusterForScaleState(KubernetesCluster.State state, VirtualMachine.State controlVmState) { + KubernetesClusterVO cluster = Mockito.mock(KubernetesClusterVO.class); + Mockito.when(cluster.getState()).thenReturn(state); + Mockito.lenient().when(cluster.getId()).thenReturn(1L); + Mockito.lenient().when(cluster.getServiceOfferingId()).thenReturn(1L); + Mockito.lenient().when(cluster.getWorkerNodeServiceOfferingId()).thenReturn(2L); + Mockito.lenient().when(cluster.getNodeCount()).thenReturn(2L); + Mockito.lenient().when(cluster.getTotalNodeCount()).thenReturn(2L); + Mockito.lenient().when(cluster.getAutoscalingEnabled()).thenReturn(false); + KubernetesClusterVmMapVO controlVmMap = Mockito.mock(KubernetesClusterVmMapVO.class); + Mockito.lenient().when(controlVmMap.getVmId()).thenReturn(100L); + Mockito.lenient().when(controlVmMap.isControlNode()).thenReturn(true); + KubernetesClusterVmMapVO workerVmMap = Mockito.mock(KubernetesClusterVmMapVO.class); + Mockito.lenient().when(workerVmMap.getVmId()).thenReturn(10L); + Mockito.lenient().when(kubernetesClusterVmMapDao.listByClusterId(1L)).thenReturn(List.of(controlVmMap, workerVmMap)); + VMInstanceVO controlVm = Mockito.mock(VMInstanceVO.class); + Mockito.lenient().when(controlVm.getState()).thenReturn(controlVmState); + Mockito.lenient().when(vmInstanceDao.findById(100L)).thenReturn(controlVm); + // The usual reason for Alert: a worker VM is stopped + VMInstanceVO workerVm = Mockito.mock(VMInstanceVO.class); + Mockito.lenient().when(workerVm.getState()).thenReturn(VirtualMachine.State.Stopped); + Mockito.lenient().when(vmInstanceDao.findById(10L)).thenReturn(workerVm); + return cluster; + } + + private KubernetesClusterVO mockClusterForScaleState(KubernetesCluster.State state) { + return mockClusterForScaleState(state, VirtualMachine.State.Running); + } + + @Test + public void testValidateKubernetesClusterStateForScaleAlertSize() { + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 3L, null, null, null, null, null, null); + } + + @Test + public void testValidateKubernetesClusterStateForScaleAlertNodeIds() { + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, null, List.of(10L), null, null, null, null, null); + } + + @Test + public void testValidateKubernetesClusterStateForScaleAlertSizeWithCurrentSettings() { + // The offerings and autoscaling setting the cluster already has may be sent along with the new size + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 3L, null, 1L, Map.of(WORKER.name(), 2L), false, null, null); + } + + @Test(expected = PermissionDeniedException.class) + public void testValidateKubernetesClusterStateForScaleAlertOfferingChange() { + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 3L, null, 5L, null, null, null, null); + } + + @Test(expected = PermissionDeniedException.class) + public void testValidateKubernetesClusterStateForScaleAlertNodeOfferingChange() { + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 3L, null, null, Map.of(WORKER.name(), 5L), null, null, null); + } + + @Test(expected = PermissionDeniedException.class) + public void testValidateKubernetesClusterStateForScaleAlertAutoscaling() { + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, null, null, null, null, true, null, null); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidateKubernetesClusterStateForScaleAlertNothingToDo() { + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, null, null, 1L, null, null, null, null); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidateKubernetesClusterStateForScaleAlertSameSize() { + // Would do nothing, then move the cluster from Alert to Running + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 2L, null, null, null, null, null, null); + } + + @Test(expected = PermissionDeniedException.class) + public void testValidateKubernetesClusterStateForScaleAlertControlNodeDown() { + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert, VirtualMachine.State.Stopped); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 3L, null, null, null, null, null, null); + } + + @Test(expected = PermissionDeniedException.class) + public void testValidateKubernetesClusterStateForScaleAlertAutoscalingSizeChange() { + // Autoscaling stays disabled, but minsize/maxsize differ: the worker treats it as an autoscaling change + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 3L, null, null, null, false, 1L, 3L); + } + + @Test(expected = PermissionDeniedException.class) + public void testValidateKubernetesClusterStateForScaleAlertExtraVms() { + // For example a worker VM left by a failed scale-up + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + Mockito.when(cluster.getTotalNodeCount()).thenReturn(1L); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 3L, null, null, null, null, null, null); + } + + @Test(expected = PermissionDeniedException.class) + public void testValidateKubernetesClusterStateForScaleAlertAllVmsRunning() { + // In Alert for another reason, such as a failed upgrade + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + VMInstanceVO workerVm = Mockito.mock(VMInstanceVO.class); + Mockito.when(workerVm.getState()).thenReturn(VirtualMachine.State.Running); + Mockito.when(vmInstanceDao.findById(10L)).thenReturn(workerVm); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 3L, null, null, null, null, null, null); + } + + @Test(expected = PermissionDeniedException.class) + public void testValidateKubernetesClusterStateForScaleAlertNoVmDown() { + // A VM on a disconnected host may still be running: the worker can't work around it + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + VMInstanceVO workerVm = Mockito.mock(VMInstanceVO.class); + Mockito.when(workerVm.getState()).thenReturn(VirtualMachine.State.Unknown); + Mockito.when(vmInstanceDao.findById(10L)).thenReturn(workerVm); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 3L, null, null, null, null, null, null); + } + + @Test(expected = PermissionDeniedException.class) + public void testValidateKubernetesClusterStateForScaleAlertEtcdVmDown() { + // etcd may have lost quorum, so new nodes can't join + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + Mockito.when(cluster.getTotalNodeCount()).thenReturn(3L); + KubernetesClusterVmMapVO etcdVmMap = Mockito.mock(KubernetesClusterVmMapVO.class); + Mockito.when(etcdVmMap.getVmId()).thenReturn(200L); + Mockito.when(etcdVmMap.isEtcdNode()).thenReturn(true); + List vmMaps = new ArrayList<>(kubernetesClusterVmMapDao.listByClusterId(1L)); + vmMaps.add(etcdVmMap); + Mockito.when(kubernetesClusterVmMapDao.listByClusterId(1L)).thenReturn(vmMaps); + VMInstanceVO etcdVm = Mockito.mock(VMInstanceVO.class); + Mockito.when(etcdVm.getState()).thenReturn(VirtualMachine.State.Stopped); + Mockito.when(vmInstanceDao.findById(200L)).thenReturn(etcdVm); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 3L, null, null, null, null, null, null); + } + + @Test(expected = PermissionDeniedException.class) + public void testValidateKubernetesClusterStateForScaleAlertOnlyExternalNodeDown() { + // Scaling never acts on external nodes, so it can't work around one that is down + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + KubernetesClusterVmMapVO workerVmMap = kubernetesClusterVmMapDao.listByClusterId(1L).get(1); + Mockito.when(workerVmMap.isExternalNode()).thenReturn(true); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 3L, null, null, null, null, null, null); + } + + @Test(expected = PermissionDeniedException.class) + public void testValidateKubernetesClusterStateForScaleAlertRemoveControlNode() { + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Alert); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, null, List.of(100L), null, null, null, null, null); + } + + @Test + public void testValidateKubernetesClusterStateForScaleRunningOfferingChange() { + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Running); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, null, null, 5L, null, null, null, null); + } + + @Test(expected = PermissionDeniedException.class) + public void testValidateKubernetesClusterStateForScaleUpgrading() { + KubernetesClusterVO cluster = mockClusterForScaleState(KubernetesCluster.State.Upgrading); + kubernetesClusterManager.validateKubernetesClusterStateForScale(cluster, 3L, null, null, null, null, null, null); + } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java index c9299bdbaa67..4c8cacf89eef 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java @@ -25,7 +25,10 @@ import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.utils.Pair; import com.cloud.vm.UserVmVO; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.UserVmDao; +import com.cloud.vm.dao.VMInstanceDao; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -36,7 +39,9 @@ import org.mockito.junit.MockitoJUnitRunner; import java.util.Arrays; +import java.util.Collections; import java.util.List; +import java.util.Set; import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL; import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.DEFAULT; @@ -54,6 +59,8 @@ public class KubernetesClusterScaleWorkerTest { private KubernetesClusterVmMapDao kubernetesClusterVmMapDao; @Mock private UserVmDao userVmDao; + @Mock + private VMInstanceDao vmInstanceDao; private KubernetesClusterScaleWorker worker; @@ -134,6 +141,7 @@ public void testGetWorkerNodesToRemoveForDownsize_singleRemoval() { KubernetesCluster kubernetesCluster = Mockito.mock(KubernetesCluster.class); KubernetesClusterManagerImpl clusterManager = Mockito.mock(KubernetesClusterManagerImpl.class); KubernetesClusterScaleWorker worker = new KubernetesClusterScaleWorker(kubernetesCluster, new java.util.HashMap<>(), 2L, null, false, null, null, clusterManager); + worker.vmInstanceDao = vmInstanceDao; KubernetesClusterScaleWorker spyWorker = Mockito.spy(worker); KubernetesClusterVmMapVO vm1 = Mockito.mock(KubernetesClusterVmMapVO.class); @@ -164,6 +172,7 @@ public void testGetWorkerNodesToRemoveForDownsize_noRemoval() { KubernetesCluster kubernetesCluster = Mockito.mock(KubernetesCluster.class); KubernetesClusterScaleWorker worker = new KubernetesClusterScaleWorker(kubernetesCluster, new java.util.HashMap<>(), 3L, null, false, null, null, clusterManager); + worker.vmInstanceDao = vmInstanceDao; KubernetesClusterScaleWorker spyWorker = Mockito.spy(worker); KubernetesClusterVmMapVO vm1 = Mockito.mock(KubernetesClusterVmMapVO.class); @@ -187,4 +196,67 @@ public void testGetWorkerNodesToRemoveForDownsize_noRemoval() { Assert.assertTrue(toRemove.isEmpty()); } + + private KubernetesClusterVmMapVO mockWorkerVmMap(long vmId, VirtualMachine.State state) { + KubernetesClusterVmMapVO vmMap = Mockito.mock(KubernetesClusterVmMapVO.class); + Mockito.lenient().when(vmMap.isExternalNode()).thenReturn(false); + Mockito.lenient().when(vmMap.isControlNode()).thenReturn(false); + Mockito.lenient().when(vmMap.isEtcdNode()).thenReturn(false); + Mockito.lenient().when(vmMap.getVmId()).thenReturn(vmId); + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.lenient().when(vm.getState()).thenReturn(state); + Mockito.lenient().when(vmInstanceDao.findById(vmId)).thenReturn(vm); + return vmMap; + } + + @Test + public void testGetWorkerNodesToRemovePrefersNodesDown() { + KubernetesClusterScaleWorker worker = new KubernetesClusterScaleWorker(kubernetesCluster, new java.util.HashMap<>(), 2L, null, false, null, null, clusterManager); + worker.vmInstanceDao = vmInstanceDao; + KubernetesClusterScaleWorker spyWorker = Mockito.spy(worker); + + KubernetesClusterVmMapVO vm1 = mockWorkerVmMap(1L, VirtualMachine.State.Running); + KubernetesClusterVmMapVO vm2 = mockWorkerVmMap(2L, VirtualMachine.State.Stopped); + KubernetesClusterVmMapVO vm3 = mockWorkerVmMap(3L, VirtualMachine.State.Running); + KubernetesClusterVmMapVO vm4 = mockWorkerVmMap(4L, VirtualMachine.State.Running); + Mockito.doReturn(Arrays.asList(vm1, vm2, vm3, vm4)).when(spyWorker).getKubernetesClusterVMMaps(); + + List toRemove = spyWorker.getWorkerNodesToRemove(); + + // The stopped node goes first, then the last running node as before + Assert.assertEquals(2, toRemove.size()); + Assert.assertTrue(toRemove.contains(vm2)); + Assert.assertTrue(toRemove.contains(vm4)); + } + + @Test + public void testGetDownNodeCountIgnoresEtcdNodes() { + KubernetesClusterScaleWorker worker = new KubernetesClusterScaleWorker(kubernetesCluster, new java.util.HashMap<>(), 2L, null, false, null, null, clusterManager); + worker.vmInstanceDao = vmInstanceDao; + KubernetesClusterScaleWorker spyWorker = Mockito.spy(worker); + + KubernetesClusterVmMapVO running = mockWorkerVmMap(1L, VirtualMachine.State.Running); + KubernetesClusterVmMapVO stopped = mockWorkerVmMap(2L, VirtualMachine.State.Stopped); + KubernetesClusterVmMapVO etcd = mockWorkerVmMap(3L, VirtualMachine.State.Stopped); + Mockito.when(etcd.isEtcdNode()).thenReturn(true); + Mockito.doReturn(Arrays.asList(running, stopped, etcd)).when(spyWorker).getKubernetesClusterVMMaps(); + + Assert.assertEquals(1L, spyWorker.getDownNodeCount(Collections.emptySet())); + // Newly provisioned VMs aren't left out of the ready nodes + Assert.assertEquals(0L, spyWorker.getDownNodeCount(Set.of(2L))); + } + + @Test + public void testIsVmDown() { + Assert.assertTrue(KubernetesClusterScaleWorker.isVmDown(VirtualMachine.State.Stopped)); + Assert.assertTrue(KubernetesClusterScaleWorker.isVmDown(VirtualMachine.State.Shutdown)); + Assert.assertTrue(KubernetesClusterScaleWorker.isVmDown(VirtualMachine.State.Error)); + Assert.assertTrue(KubernetesClusterScaleWorker.isVmDown(VirtualMachine.State.Destroyed)); + Assert.assertTrue(KubernetesClusterScaleWorker.isVmDown(VirtualMachine.State.Expunging)); + // The VM may still be running in these + Assert.assertFalse(KubernetesClusterScaleWorker.isVmDown(VirtualMachine.State.Running)); + Assert.assertFalse(KubernetesClusterScaleWorker.isVmDown(VirtualMachine.State.Migrating)); + Assert.assertFalse(KubernetesClusterScaleWorker.isVmDown(VirtualMachine.State.Unknown)); + Assert.assertFalse(KubernetesClusterScaleWorker.isVmDown(VirtualMachine.State.Starting)); + } } diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index 4575cf2a0572..070b8e316e9e 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -630,7 +630,7 @@ export default { message: 'message.kubernetes.cluster.scale', docHelp: 'plugins/cloudstack-kubernetes-service.html#scaling-kubernetes-cluster', dataView: true, - show: (record) => { return ['Created', 'Running', 'Stopped'].includes(record.state) && record.clustertype === 'CloudManaged' }, + show: (record) => { return (['Created', 'Running', 'Stopped'].includes(record.state) || (record.state === 'Alert' && !record.autoscalingenabled)) && record.clustertype === 'CloudManaged' }, popup: true, component: shallowRef(defineAsyncComponent(() => import('@/views/compute/ScaleKubernetesCluster.vue'))) }, diff --git a/ui/src/views/compute/ScaleKubernetesCluster.vue b/ui/src/views/compute/ScaleKubernetesCluster.vue index 837860061684..e3a11288fd4c 100644 --- a/ui/src/views/compute/ScaleKubernetesCluster.vue +++ b/ui/src/views/compute/ScaleKubernetesCluster.vue @@ -28,7 +28,7 @@ :rules="rules" @finish="handleSubmit" layout="vertical"> - + @@ -47,7 +47,7 @@ - + @@ -66,7 +66,7 @@ - + @@ -85,7 +85,7 @@ - + @@ -104,13 +104,13 @@ - + - +