Allow exclusion of destination addresses from usage on KVM shared networks

debian/control

@@ -24,7 +24,7 @@ Description: CloudStack server library
 
 Package: cloudstack-agent
 Architecture: all
-Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17 | openjdk-21-jre-headless | java21-runtime-headless | java21-runtime | zulu-21, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat, python3-libnbd, socat
+Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17 | openjdk-21-jre-headless | java21-runtime-headless | java21-runtime | zulu-21, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, nftables, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, ovmf, swtpm, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd, sysstat, python3-libnbd, socat
 Recommends: init-system-helpers
 Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts
 Description: CloudStack agent

engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java

@@ -16,6 +16,7 @@
 // under the License.
 package org.apache.cloudstack.engine.orchestration.service;
 
+import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -53,12 +54,14 @@
 import com.cloud.user.User;
 import com.cloud.utils.fsm.NoTransitionException;
 import com.cloud.utils.Pair;
+import com.cloud.utils.net.NetUtils;
 import com.cloud.vm.Nic;
 import com.cloud.vm.NicProfile;
 import com.cloud.vm.ReservationContext;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachine.Type;
 import com.cloud.vm.VirtualMachineProfile;
+import com.googlecode.ipv6.IPv6Network;
 
 /**
  * NetworkManager manages the network for the different end users.
@@ -138,6 +141,55 @@ public interface NetworkOrchestrationService {
     ConfigKey<Integer> VmNetworkThrottlingRate = new ConfigKey<Integer>("Network", Integer.class, "vm.network.throttling.rate", "200",
             "Default data transfer rate in megabits per second allowed in User vm's default network.", true, ConfigKey.Scope.Zone);
 
+    ConfigKey<String> NetworkUsageExemptCidrs = new ConfigKey<>(String.class, "network.usage.exempt.cidrs", "Usage", "",
+            "Comma-separated IPv4 or IPv6 addresses or CIDRs. Traffic between guest VMs and these addresses doesn't count " +
+            "towards network usage. Virtual routers only count IPv4 traffic, so they ignore IPv6 entries, and pick up a change " +
+            "the next time they start or restart. On shared networks, KVM hosts exempt both IPv4 and IPv6 and pick up a change " +
+            "straight away.",
+            true, ConfigKey.Scope.Zone, null, null, null, null, null, ConfigKey.Kind.CSV, null);
+
+    /**
+     * Tidies one entry of {@link #NetworkUsageExemptCidrs} into a CIDR, or returns null if it isn't a valid address or CIDR.
+     * For example, 192.0.2.5 becomes 192.0.2.5/32, 198.51.100.70/24 becomes 198.51.100.0/24 and 2001:db8::5 becomes
+     * 2001:db8::5/128. Callers refuse a /0 result, as it would exempt all traffic.
+     */
+    static String normaliseUsageExemptCidr(String entry) {
+        if (NetUtils.isValidIp4(entry)) {
+            entry = entry + "/32";
+        } else if (NetUtils.isValidIp6(entry)) {
+            entry = entry + "/128";
+        }
+        if (NetUtils.isValidIp4Cidr(entry)) {
+            return NetUtils.getCidrSubNet(entry) + "/" + NetUtils.getCidr(entry).second();
+        }
+        if (entry.contains(":") && NetUtils.isValidIp6Cidr(entry)) {
+            return IPv6Network.fromString(entry).toString();
+        }
+        return null;
+    }
+
+    /**
+     * Turns a {@link #NetworkUsageExemptCidrs} value into a tidy list with no repeats, using {@link #normaliseUsageExemptCidr}.
+     * Entries that aren't valid, and /0, are left out. Saving the setting already refuses them.
+     */
+    static List<String> parseUsageExemptCidrs(String value) {
+        final List<String> cidrs = new ArrayList<>();
+        if (value == null) {
+            return cidrs;
+        }
+        for (String entry : value.split(",")) {
+            entry = entry.trim();
+            if (entry.isEmpty()) {
+                continue;
+            }
+            final String cidr = normaliseUsageExemptCidr(entry);
+            if (cidr != null && !cidr.endsWith("/0") && !cidrs.contains(cidr)) {
+                cidrs.add(cidr);
+            }
+        }
+        return cidrs;
+    }
+
     List<? extends Network> setupNetwork(Account owner, NetworkOffering offering, DeploymentPlan plan, String name, String displayText, boolean isDefault)
         throws ConcurrentOperationException;
 

engine/api/src/test/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationServiceTest.java

@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cloudstack.engine.orchestration.service;
+
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class NetworkOrchestrationServiceTest {
+
+    @Test
+    public void parseUsageExemptCidrsTestNotSet() {
+        Assert.assertTrue(NetworkOrchestrationService.parseUsageExemptCidrs(null).isEmpty());
+        Assert.assertTrue(NetworkOrchestrationService.parseUsageExemptCidrs(" ").isEmpty());
+    }
+
+    @Test
+    public void parseUsageExemptCidrsTestNormalisesEntries() {
+        List<String> cidrs = NetworkOrchestrationService.parseUsageExemptCidrs("192.0.2.5, 198.51.100.70/24,,192.0.2.5/32");
+        Assert.assertEquals(List.of("192.0.2.5/32", "198.51.100.0/24"), cidrs);
+    }
+
+    @Test
+    public void parseUsageExemptCidrsTestNormalisesIpv6Entries() {
+        List<String> cidrs = NetworkOrchestrationService.parseUsageExemptCidrs("2001:db8::5, 2001:db8:0:1::7/64,192.0.2.5");
+        Assert.assertEquals(List.of("2001:db8::5/128", "2001:db8:0:1::/64", "192.0.2.5/32"), cidrs);
+    }
+
+    @Test
+    public void parseUsageExemptCidrsTestSkipsInvalidEntries() {
+        List<String> cidrs = NetworkOrchestrationService.parseUsageExemptCidrs("not-an-ip,0.0.0.0/0,::/0,10.0.0.0/33,192.168.1.300,172.16.0.0/12");
+        Assert.assertEquals(List.of("172.16.0.0/12"), cidrs);
+    }
+}

engine/components-api/src/main/java/com/cloud/agent/AgentManager.java

@@ -175,6 +175,12 @@ enum TapAgentsAction {
 
     void propagateChangeToAgents(Map<String, String> params);
 
+    /**
+     * Sends network.usage.exempt.cidrs to the KVM hosts in one zone, or in every zone when zoneId is null,
+     * using the value for each host's zone.
+     */
+    void propagateUsageExemptCidrsToAgents(Long zoneId);
+
     boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs, boolean excludeHostsInMaintenance);
 
     int getHostSshPort(HostVO host);

engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java

@@ -23,6 +23,7 @@
 import java.nio.channels.ClosedChannelException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
@@ -2275,6 +2276,7 @@ public void processConnect(final Host host, final StartupCommand cmd, final bool
                 params.put(Config.MigrateWait.toString(), _configDao.getValue(Config.MigrateWait.toString()));
                 params.put(NetworkOrchestrationService.TUNGSTEN_ENABLED.key(), String.valueOf(NetworkOrchestrationService.TUNGSTEN_ENABLED.valueIn(host.getDataCenterId())));
                 params.put(ReconcileCommandService.ReconcileCommandsEnabled.key(), String.valueOf(_reconcileCommandsEnabled));
+                params.put(NetworkOrchestrationService.NetworkUsageExemptCidrs.key(), getUsageExemptCidrsParam(host.getDataCenterId()));
 
                     try {
                         SetHostParamsCommand cmds = new SetHostParamsCommand(params);
@@ -2353,6 +2355,31 @@ public void propagateChangeToAgents(Map<String, String> params) {
         }
     }
 
+    @Override
+    public void propagateUsageExemptCidrsToAgents(Long zoneId) {
+        final Map<Long, List<Long>> hostsPerZone = getHostsPerZone();
+        if (hostsPerZone == null) {
+            return;
+        }
+        logger.debug("Sending {} to the agents", NetworkOrchestrationService.NetworkUsageExemptCidrs.key());
+        for (Map.Entry<Long, List<Long>> zone : hostsPerZone.entrySet()) {
+            if (zoneId != null && !zoneId.equals(zone.getKey())) {
+                continue;
+            }
+            final Map<String, String> params = new HashMap<>();
+            params.put(NetworkOrchestrationService.NetworkUsageExemptCidrs.key(), getUsageExemptCidrsParam(zone.getKey()));
+            sendCommandToAgents(Collections.singletonMap(zone.getKey(), zone.getValue()), params);
+        }
+    }
+
+    /**
+     * The zone's exempt list, tidied and joined with commas. An empty string tells the host the list is empty.
+     */
+    protected String getUsageExemptCidrsParam(long zoneId) {
+        return String.join(",", NetworkOrchestrationService.parseUsageExemptCidrs(
+                NetworkOrchestrationService.NetworkUsageExemptCidrs.valueIn(zoneId)));
+    }
+
     @Override
     public boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs, boolean excludeHostsInMaintenance) {
         return true;

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java

@@ -5117,6 +5117,6 @@ public ConfigKey<?>[] getConfigKeys() {
                 GuestDomainSuffix, NetworkThrottlingRate, VmNetworkThrottlingRate, MinVRVersion, DhcpLeaseTimeout,
                 PromiscuousMode, MacAddressChanges, ForgedTransmits, MacLearning, RollingRestartEnabled,
                 TUNGSTEN_ENABLED, NSX_ENABLED, NETRIS_ENABLED, NETWORK_LB_HAPROXY_MAX_CONN,
-                NETWORK_LB_HAPROXY_IDLE_TIMEOUT};
+                NETWORK_LB_HAPROXY_IDLE_TIMEOUT, NetworkUsageExemptCidrs};
     }
 }

engine/orchestration/src/test/java/com/cloud/agent/manager/AgentManagerImplTest.java

@@ -20,6 +20,7 @@
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.ReadyAnswer;
 import com.cloud.agent.api.ReadyCommand;
+import com.cloud.agent.api.SetHostParamsCommand;
 import com.cloud.agent.api.StartupCommand;
 import com.cloud.agent.api.StartupRoutingCommand;
 import com.cloud.exception.ConnectionException;
@@ -37,7 +38,10 @@
 import org.mockito.Mockito;
 
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 public class AgentManagerImplTest {
@@ -172,4 +176,33 @@ public void testGetHostSshPortWithKVMHostCustomPort() {
         int hostSshPort = mgr.getHostSshPort(host);
         Assert.assertEquals(3922, hostSshPort);
     }
+
+    private void mockUsageExemptPush() {
+        Map<Long, List<Long>> hostsPerZone = new HashMap<>();
+        hostsPerZone.put(1L, Arrays.asList(11L, 12L));
+        hostsPerZone.put(2L, Collections.singletonList(21L));
+        Mockito.doReturn(hostsPerZone).when(mgr).getHostsPerZone();
+        Mockito.doReturn("192.0.2.5/32").when(mgr).getUsageExemptCidrsParam(Mockito.anyLong());
+        Answer answer = Mockito.mock(Answer.class);
+        Mockito.when(answer.getResult()).thenReturn(true);
+        Mockito.doReturn(answer).when(mgr).easySend(Mockito.anyLong(), Mockito.any(SetHostParamsCommand.class));
+    }
+
+    @Test
+    public void propagateUsageExemptCidrsToAgentsTestOneZone() {
+        mockUsageExemptPush();
+        mgr.propagateUsageExemptCidrsToAgents(1L);
+        Mockito.verify(mgr).easySend(Mockito.eq(11L), Mockito.any(SetHostParamsCommand.class));
+        Mockito.verify(mgr).easySend(Mockito.eq(12L), Mockito.any(SetHostParamsCommand.class));
+        Mockito.verify(mgr, Mockito.never()).easySend(Mockito.eq(21L), Mockito.any(SetHostParamsCommand.class));
+    }
+
+    @Test
+    public void propagateUsageExemptCidrsToAgentsTestAllZones() {
+        mockUsageExemptPush();
+        mgr.propagateUsageExemptCidrsToAgents(null);
+        Mockito.verify(mgr, Mockito.times(3)).easySend(Mockito.anyLong(), Mockito.any(SetHostParamsCommand.class));
+        Mockito.verify(mgr).getUsageExemptCidrsParam(1L);
+        Mockito.verify(mgr).getUsageExemptCidrsParam(2L);
+    }
 }

packaging/el8/cloud.spec

@@ -114,6 +114,7 @@ Requires: ethtool
 Requires: (net-tools or net-tools-deprecated)
 Requires: iproute
 Requires: ipset
+Requires: nftables
 Requires: perl
 Requires: rsync
 Requires: cifs-utils

packaging/suse15/cloud.spec

@@ -114,6 +114,7 @@ Requires: ethtool
 Requires: (net-tools or net-tools-deprecated)
 Requires: iproute
 Requires: ipset
+Requires: nftables
 Requires: perl
 Requires: rsync
 Requires: cifs-utils

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -613,6 +613,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
     protected MemStat memStat = new MemStat(dom0MinMem, dom0OvercommitMem);
     private final LibvirtUtilitiesHelper libvirtUtilitiesHelper = new LibvirtUtilitiesHelper();
     private LibvirtDomainListener libvirtDomainListener;
+    private final UsageExemptCounters usageExemptCounters = new UsageExemptCounters();
 
     protected Boolean enableManuallySettingCpuTopologyOnKvmVm = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.ENABLE_MANUALLY_SETTING_CPU_TOPOLOGY_ON_KVM_VM);
 
@@ -1780,10 +1781,19 @@ public boolean configureHostParams(final Map<String, String> params) {
             isTungstenEnabled = Boolean.parseBoolean(params.get(NetworkOrchestrationService.TUNGSTEN_ENABLED.key()));
         }
 
-        if (params.get(ReconcileCommandService.ReconcileCommandsEnabled.key()) != null) {
-            isReconcileCommandsEnabled = Boolean.parseBoolean(params.get(ReconcileCommandService.ReconcileCommandsEnabled.key()));
+        final String usageExemptCidrs = params.get(NetworkOrchestrationService.NetworkUsageExemptCidrs.key());
+        if (usageExemptCidrs != null && hypervisorType == HypervisorType.KVM) {
+            usageExemptCounters.setTapOwners(this::getTapOwners);
+            usageExemptCounters.setCidrs(NetworkOrchestrationService.parseUsageExemptCidrs(usageExemptCidrs));
         }
-        if (isReconcileCommandsEnabled) {
+
+        final String reconcileCommandsEnabled = params.get(ReconcileCommandService.ReconcileCommandsEnabled.key());
+        if (reconcileCommandsEnabled != null) {
+            isReconcileCommandsEnabled = Boolean.parseBoolean(reconcileCommandsEnabled);
+        }
+        // Only on connect, which is the only time the server sends this setting. Settings pushed while the agent is
+        // running must leave the logs of commands still in progress alone.
+        if (isReconcileCommandsEnabled && reconcileCommandsEnabled != null) {
             File commandsLogPath = new File(COMMANDS_LOG_PATH);
             if (!commandsLogPath.exists()) {
                 commandsLogPath.mkdirs();
@@ -5180,6 +5190,38 @@ private int executeBashScriptAndRetrieveExitValue(final String script) {
         return command.getExitValue();
     }
 
+    public UsageExemptCounters getUsageExemptCounters() {
+        return usageExemptCounters;
+    }
+
+    /**
+     * Maps each running VM's NIC device name to the VM's name, so the usage exemption counters can skip system VMs.
+     * Returns null if libvirt can't be asked.
+     */
+    protected Map<String, String> getTapOwners() {
+        try {
+            final Connect conn = LibvirtConnection.getConnection();
+            final Map<String, String> owners = new HashMap<>();
+            for (final int id : conn.listDomains()) {
+                final Domain dm = conn.domainLookupByID(id);
+                try {
+                    final String vmName = dm.getName();
+                    for (final InterfaceDef nic : getInterfaces(conn, vmName)) {
+                        if (nic.getDevName() != null) {
+                            owners.put(nic.getDevName(), vmName);
+                        }
+                    }
+                } finally {
+                    dm.free();
+                }
+            }
+            return owners;
+        } catch (final LibvirtException e) {
+            LOGGER.debug("Failed to list the running VMs' NICs for the usage exemption counters", e);
+            return null;
+        }
+    }
+
     public List<VmNetworkStatsEntry> getVmNetworkStat(Connect conn, String vmName) throws LibvirtException {
         Domain dm = null;
         try {
@@ -5192,7 +5234,8 @@ public List<VmNetworkStatsEntry> getVmNetworkStat(Connect conn, String vmName) t
             for (InterfaceDef nic : nics) {
                 DomainInterfaceStats nicStats = dm.interfaceStats(nic.getDevName());
                 String macAddress = nic.getMacAddress();
-                VmNetworkStatsEntry stat = new VmNetworkStatsEntry(vmName, macAddress, nicStats.tx_bytes, nicStats.rx_bytes);
+                long[] usage = usageExemptCounters.subtractExempt(nic.getDevName(), nicStats.tx_bytes, nicStats.rx_bytes);
+                VmNetworkStatsEntry stat = new VmNetworkStatsEntry(vmName, macAddress, usage[0], usage[1]);
                 stats.add(stat);
             }