diff --git a/conf/globalConfig/lb.xml b/conf/globalConfig/lb.xml index 5b3a9e3f5d9..4eaee9776b8 100755 --- a/conf/globalConfig/lb.xml +++ b/conf/globalConfig/lb.xml @@ -109,4 +109,20 @@ 40 java.lang.Long + + + loadBalancer + ipvs.defaultMode + Default IPVS forwarding mode for IPVS listeners. Options: dr (Direct Routing), fullnat. Empty means haproxy/gobetween path. + + java.lang.String + + + + loadBalancer + ipvs.defaultScheduler + Default IPVS scheduling algorithm. Options: rr (round-robin), wrr (weighted round-robin), lc (least-connections), sh (source-hash). + rr + java.lang.String + diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java index f2f8271674b..90bf9bb6b4c 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java @@ -779,12 +779,35 @@ private void validate(APICreateLoadBalancerListenerMsg msg) { validateAcl(msg.getAclUuids(),new ArrayList<>(), msg.getLoadBalancerUuid()); } - insertTagIfNotExisting( - msg, LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT, - LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT.instantiateTag( - map(e(LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT_TOKEN, LoadBalancerGlobalConfig.CONNECTION_IDLE_TIMEOUT.value(Long.class))) - ) - ); + // Extract ipvsMode early to gate insertion of tags that are incompatible with IPVS listeners. + // Also apply IPVS_DEFAULT_MODE global config if operator has configured one and no explicit tag is given. + String earlyIpvsMode = null; + if (msg.getSystemTags() != null) { + for (String tag : msg.getSystemTags()) { + if (LoadBalancerSystemTags.IPVS_MODE.isMatch(tag)) { + earlyIpvsMode = LoadBalancerSystemTags.IPVS_MODE.getTokenByTag(tag, LoadBalancerSystemTags.IPVS_MODE_TOKEN); + break; + } + } + } + if (earlyIpvsMode == null || earlyIpvsMode.isEmpty()) { + String defaultMode = LoadBalancerGlobalConfig.IPVS_DEFAULT_MODE.value(); + if (defaultMode != null && !defaultMode.isEmpty()) { + earlyIpvsMode = defaultMode; + insertTagIfNotExisting(msg, LoadBalancerSystemTags.IPVS_MODE, + LoadBalancerSystemTags.IPVS_MODE.instantiateTag( + map(e(LoadBalancerSystemTags.IPVS_MODE_TOKEN, defaultMode)))); + } + } + + if (earlyIpvsMode == null || earlyIpvsMode.isEmpty()) { + insertTagIfNotExisting( + msg, LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT, + LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT.instantiateTag( + map(e(LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT_TOKEN, LoadBalancerGlobalConfig.CONNECTION_IDLE_TIMEOUT.value(Long.class))) + ) + ); + } insertTagIfNotExisting( msg, LoadBalancerSystemTags.HEALTHY_THRESHOLD, @@ -821,12 +844,14 @@ private void validate(APICreateLoadBalancerListenerMsg msg) { ) ); - insertTagIfNotExisting( - msg, LoadBalancerSystemTags.MAX_CONNECTION, - LoadBalancerSystemTags.MAX_CONNECTION.instantiateTag( - map(e(LoadBalancerSystemTags.MAX_CONNECTION_TOKEN, LoadBalancerGlobalConfig.MAX_CONNECTION.value(Long.class))) - ) - ); + if (earlyIpvsMode == null || earlyIpvsMode.isEmpty()) { + insertTagIfNotExisting( + msg, LoadBalancerSystemTags.MAX_CONNECTION, + LoadBalancerSystemTags.MAX_CONNECTION.instantiateTag( + map(e(LoadBalancerSystemTags.MAX_CONNECTION_TOKEN, LoadBalancerGlobalConfig.MAX_CONNECTION.value(Long.class))) + ) + ); + } insertTagIfNotExisting( msg, LoadBalancerSystemTags.BALANCER_ALGORITHM, @@ -868,7 +893,7 @@ private void validate(APICreateLoadBalancerListenerMsg msg) { } } - String algorithm = null, seessionPersistence = null, httpRedirectHttps = null, redirectPort = null, statusCode = null; + String algorithm = null, seessionPersistence = null, httpRedirectHttps = null, redirectPort = null, statusCode = null, ipvsMode = null; for (String tag : msg.getSystemTags()) { if (LoadBalancerSystemTags.BALANCER_ALGORITHM.isMatch(tag)) { algorithm = LoadBalancerSystemTags.BALANCER_ALGORITHM.getTokenByTag(tag, @@ -898,6 +923,10 @@ private void validate(APICreateLoadBalancerListenerMsg msg) { "could not create the loadbalancer listener with systemTag httpCompressAlgos::disable, please remove this tag")); } } + if (LoadBalancerSystemTags.IPVS_MODE.isMatch(tag)) { + ipvsMode = LoadBalancerSystemTags.IPVS_MODE.getTokenByTag(tag, + LoadBalancerSystemTags.IPVS_MODE_TOKEN); + } } if ((redirectPort != null || statusCode != null) && (httpRedirectHttps == null || HttpRedirectHttps.disable.toString().equals(httpRedirectHttps))) { @@ -1179,6 +1208,8 @@ private void validate(APICreateLoadBalancerListenerMsg msg) { ); } } + + validateIpvsMode(msg.getLoadBalancerUuid(), ipvsMode, msg.getProtocol(), msg.getSystemTags(), algorithm, null); } private void validate(APIDeleteLoadBalancerListenerMsg msg) { @@ -1192,6 +1223,85 @@ private void validate(APIDeleteLoadBalancerListenerMsg msg) { msg.setLoadBalancerUuid(lbUuid); } + + private void validateIpvsMode(String lbUuid, String ipvsMode, String protocol, List systemTags, String algorithm, String excludeListenerUuid) { + boolean isIpvs = ipvsMode != null && !ipvsMode.isEmpty(); + + // Always enforce consistency: all listeners on the same LB must use the same forwarding path + List listenerUuids = Q.New(LoadBalancerListenerVO.class) + .select(LoadBalancerListenerVO_.uuid) + .eq(LoadBalancerListenerVO_.loadBalancerUuid, lbUuid) + .listValues(); + for (String listenerUuid : listenerUuids) { + if (listenerUuid.equals(excludeListenerUuid)) { + continue; + } + String existingMode = LoadBalancerSystemTags.IPVS_MODE.getTokenByResourceUuid(listenerUuid, LoadBalancerSystemTags.IPVS_MODE_TOKEN); + boolean existingIsIpvs = existingMode != null && !existingMode.isEmpty(); + if (isIpvs != existingIsIpvs) { + throw new ApiMessageInterceptionException(argerr( + "cannot mix ipvs and non-ipvs (haproxy/gobetween) listeners on the same load balancer; " + + "existing listener[uuid:%s] uses [%s]", + listenerUuid, existingIsIpvs ? existingMode : "haproxy/gobetween")); + } + if (isIpvs && !ipvsMode.equals(existingMode)) { + throw new ApiMessageInterceptionException(argerr( + "all listeners on the same load balancer must have the same ipvsMode; " + + "listener[uuid:%s] has ipvsMode[%s] but the new listener specifies [%s]", + listenerUuid, existingMode, ipvsMode)); + } + } + + if (!isIpvs) { + return; + } + + // ipvsMode value must be recognized + if (!LoadBalancerConstants.IPVS_MODES.contains(ipvsMode)) { + throw new ApiMessageInterceptionException(argerr("invalid ipvsMode [%s], supported values: %s", ipvsMode, LoadBalancerConstants.IPVS_MODES)); + } + + // Only TCP/UDP listeners may use ipvs + if (!LB_PROTOCOL_TCP.equals(protocol) && !LB_PROTOCOL_UDP.equals(protocol)) { + throw new ApiMessageInterceptionException(argerr("ipvsMode is only supported for tcp/udp listeners, but protocol is [%s]", protocol)); + } + + // ipvs listeners forbid L7 features + for (String tag : systemTags) { + if (LoadBalancerSystemTags.HTTP_MODE.isMatch(tag)) { + throw new ApiMessageInterceptionException(argerr("ipvs listener does not support httpMode")); + } + if (LoadBalancerSystemTags.HTTP_REDIRECT_HTTPS.isMatch(tag)) { + throw new ApiMessageInterceptionException(argerr("ipvs listener does not support http redirect https")); + } + if (LoadBalancerSystemTags.SESSION_PERSISTENCE.isMatch(tag)) { + String sp = LoadBalancerSystemTags.SESSION_PERSISTENCE.getTokenByTag(tag, LoadBalancerSystemTags.SESSION_PERSISTENCE_TOKEN); + if (!LoadBalancerSessionPersistence.disable.toString().equals(sp)) { + throw new ApiMessageInterceptionException(argerr("ipvs listener does not support session persistence")); + } + } + if (LoadBalancerSystemTags.COOKIE_NAME.isMatch(tag)) { + throw new ApiMessageInterceptionException(argerr("ipvs listener does not support cookie-based session persistence")); + } + if (LoadBalancerSystemTags.TCP_PROXYPROTOCOL.isMatch(tag)) { + throw new ApiMessageInterceptionException(argerr("ipvs listener does not support tcp proxy protocol")); + } + if (LoadBalancerSystemTags.MAX_CONNECTION.isMatch(tag)) { + throw new ApiMessageInterceptionException(argerr("ipvs listener does not support maxConnection")); + } + if (LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT.isMatch(tag)) { + throw new ApiMessageInterceptionException(argerr("ipvs listener does not support connectionIdleTimeout")); + } + } + + // balancerAlgorithm must be in IPVS whitelist when specified + if (algorithm != null && !LoadBalancerConstants.IPVS_ALLOWED_BALANCE_ALGORITHMS.contains(algorithm)) { + throw new ApiMessageInterceptionException(argerr( + "balancerAlgorithm [%s] is not supported in ipvs mode; allowed values: %s", + algorithm, LoadBalancerConstants.IPVS_ALLOWED_BALANCE_ALGORITHMS)); + } + } + private void validate(APIUpdateLoadBalancerListenerMsg msg) { String loadBalancerUuid = Q.New(LoadBalancerListenerVO.class). select(LoadBalancerListenerVO_.loadBalancerUuid). @@ -1199,6 +1309,20 @@ private void validate(APIUpdateLoadBalancerListenerMsg msg) { getLoadBalancerListenerUuid()).findValue(); msg.setLoadBalancerUuid(loadBalancerUuid); bus.makeTargetServiceIdByResourceUuid(msg, LoadBalancerConstants.SERVICE_ID, loadBalancerUuid); + // Validate ipvsMode consistency if the update carries a new ipvsMode tag + String newIpvsMode = null; + for (String tag : msg.getSystemTags()) { + if (LoadBalancerSystemTags.IPVS_MODE.isMatch(tag)) { + newIpvsMode = LoadBalancerSystemTags.IPVS_MODE.getTokenByTag(tag, LoadBalancerSystemTags.IPVS_MODE_TOKEN); + } + } + if (newIpvsMode != null) { + String protocol = Q.New(LoadBalancerListenerVO.class) + .select(LoadBalancerListenerVO_.protocol) + .eq(LoadBalancerListenerVO_.uuid, msg.getLoadBalancerListenerUuid()) + .findValue(); + validateIpvsMode(loadBalancerUuid, newIpvsMode, protocol, msg.getSystemTags(), null, msg.getLoadBalancerListenerUuid()); + } } private void validate(APIAddCertificateToLoadBalancerListenerMsg msg) { diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerConstants.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerConstants.java index fe147c11cf4..2968fdf7d33 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerConstants.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerConstants.java @@ -171,6 +171,28 @@ public static enum Param { public static final String HEALTH_CHECK_TARGET_DEFAULT = "default"; + // IPVS forwarding modes for LoadBalancerListenerVO systemTag "ipvsMode::{mode}" + public static final String IPVS_MODE_DR = "dr"; + public static final String IPVS_MODE_FULLNAT = "fullnat"; + public static final List IPVS_MODES = asList(IPVS_MODE_DR, IPVS_MODE_FULLNAT); + + // IPVS scheduler names that map 1:1 from BALANCE_ALGORITHM_* values + public static final String IPVS_SCHEDULER_RR = "rr"; + public static final String IPVS_SCHEDULER_WRR = "wrr"; + public static final String IPVS_SCHEDULER_LC = "lc"; + public static final String IPVS_SCHEDULER_SH = "sh"; + // Algorithms allowed when ipvsMode is set (enforced by interceptor) + public static final List IPVS_ALLOWED_BALANCE_ALGORITHMS = asList( + BALANCE_ALGORITHM_ROUND_ROBIN, + BALANCE_ALGORITHM_WEIGHT_ROUND_ROBIN, + BALANCE_ALGORITHM_LEAST_CONN, + BALANCE_ALGORITHM_LEAST_SOURCE + ); + + // IPVS connection type flags passed to ipvsadm (-g = DR gate, -m = masquerade/fullnat) + public static final String IPVS_CONNECTION_TYPE_DR = "-g"; + public static final String IPVS_CONNECTION_TYPE_FULLNAT = "-m"; + public static final List vmOperationForDetachListener = asList( VmInstanceConstant.VmOperation.Destroy, VmInstanceConstant.VmOperation.DetachNic, diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerGlobalConfig.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerGlobalConfig.java index 011f0a1c350..a9e4db84d39 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerGlobalConfig.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerGlobalConfig.java @@ -36,5 +36,9 @@ public class LoadBalancerGlobalConfig { @GlobalConfigValidation public static GlobalConfig HTTP_MODE = new GlobalConfig(CATEGORY, "httpMode"); + @GlobalConfigValidation + public static GlobalConfig IPVS_DEFAULT_MODE = new GlobalConfig(CATEGORY, "ipvs.defaultMode"); + @GlobalConfigValidation + public static GlobalConfig IPVS_DEFAULT_SCHEDULER = new GlobalConfig(CATEGORY, "ipvs.defaultScheduler"); } diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerSystemTags.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerSystemTags.java index 021f525b8fb..2d38d3a0aa5 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerSystemTags.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerSystemTags.java @@ -87,4 +87,7 @@ public class LoadBalancerSystemTags { public static final String HTTP_COMPRESS_ALGOS_TOKEN = "httpCompressAlgos"; public static PatternedSystemTag HTTP_COMPRESS_ALGOS= new PatternedSystemTag(String.format("httpCompressAlgos::{%s}", HTTP_COMPRESS_ALGOS_TOKEN), LoadBalancerListenerVO.class); + + public static final String IPVS_MODE_TOKEN = "ipvsMode"; + public static PatternedSystemTag IPVS_MODE = new PatternedSystemTag(String.format("ipvsMode::{%s}", IPVS_MODE_TOKEN), LoadBalancerListenerVO.class); } diff --git a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/lb/VirtualRouterLoadBalancerBackend.java b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/lb/VirtualRouterLoadBalancerBackend.java index 1b4266d6e05..48711076498 100755 --- a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/lb/VirtualRouterLoadBalancerBackend.java +++ b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/lb/VirtualRouterLoadBalancerBackend.java @@ -278,6 +278,11 @@ public static class LbTO { boolean enableStatsLog; + // IPVS fields: populated from ipvsMode systemTag; empty string = haproxy/gobetween path + String ipvsMode; // "dr" | "fullnat" | "" + String scheduler; // "rr" | "wrr" | "lc" | "sh" | "" + String connectionType; // "-g" (DR) | "-m" (fullnat) | "" + public static class ServerGroup { private String name; private String serverGroupUuid; @@ -525,6 +530,30 @@ public String getVipL3Uuid() { public void setVipL3Uuid(String vipL3Uuid) { this.vipL3Uuid = vipL3Uuid; } + + public String getIpvsMode() { + return ipvsMode; + } + + public void setIpvsMode(String ipvsMode) { + this.ipvsMode = ipvsMode; + } + + public String getScheduler() { + return scheduler; + } + + public void setScheduler(String scheduler) { + this.scheduler = scheduler; + } + + public String getConnectionType() { + return connectionType; + } + + public void setConnectionType(String connectionType) { + this.connectionType = connectionType; + } } public static class RefreshLbCmd extends AgentCommand { @@ -660,6 +689,23 @@ public boolean enableStatsLog(LbTO to) { } } + // Maps ZStack balancerAlgorithm names to IPVS scheduler names (1:1 mapping) + private static String mapBalancerAlgorithmToIpvsScheduler(String algorithm) { + String configDefault = LoadBalancerGlobalConfig.IPVS_DEFAULT_SCHEDULER.value(); + String fallback = (configDefault != null && !configDefault.isEmpty()) + ? configDefault : LoadBalancerConstants.IPVS_SCHEDULER_RR; + if (algorithm == null) { + return fallback; + } + switch (algorithm) { + case LoadBalancerConstants.BALANCE_ALGORITHM_ROUND_ROBIN: return LoadBalancerConstants.IPVS_SCHEDULER_RR; + case LoadBalancerConstants.BALANCE_ALGORITHM_WEIGHT_ROUND_ROBIN: return LoadBalancerConstants.IPVS_SCHEDULER_WRR; + case LoadBalancerConstants.BALANCE_ALGORITHM_LEAST_CONN: return LoadBalancerConstants.IPVS_SCHEDULER_LC; + case LoadBalancerConstants.BALANCE_ALGORITHM_LEAST_SOURCE: return LoadBalancerConstants.IPVS_SCHEDULER_SH; + default: return fallback; + } + } + public List makeCommonLbTOs(final LoadBalancerStruct struct) { return makeLbTOs(struct, null); } @@ -904,6 +950,21 @@ public LbTO call(LoadBalancerListenerInventory l) { if (vip6 != null) { to.setVip6(vip6.getIp()); } + + // Populate IPVS fields from ipvsMode systemTag + String ipvsMode = LoadBalancerSystemTags.IPVS_MODE.getTokenByResourceUuid(l.getUuid(), LoadBalancerSystemTags.IPVS_MODE_TOKEN); + if (ipvsMode != null && !ipvsMode.isEmpty()) { + to.setIpvsMode(ipvsMode); + // Map balancerAlgorithm tag to IPVS scheduler name + String balancerAlgorithm = LoadBalancerSystemTags.BALANCER_ALGORITHM.getTokenByResourceUuid(l.getUuid(), LoadBalancerSystemTags.BALANCER_ALGORITHM_TOKEN); + to.setScheduler(mapBalancerAlgorithmToIpvsScheduler(balancerAlgorithm)); + // Set connection type flag + if (LoadBalancerConstants.IPVS_MODE_DR.equals(ipvsMode)) { + to.setConnectionType(LoadBalancerConstants.IPVS_CONNECTION_TYPE_DR); + } else if (LoadBalancerConstants.IPVS_MODE_FULLNAT.equals(ipvsMode)) { + to.setConnectionType(LoadBalancerConstants.IPVS_CONNECTION_TYPE_FULLNAT); + } + } to.setSecurityPolicyType(l.getSecurityPolicyType()); if (l.getCertificateRefs() != null && !l.getCertificateRefs().isEmpty()) { to.setCertificateUuid(l.getCertificateRefs().get(0).getCertificateUuid()); diff --git a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosKeepalivedCommands.java b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosKeepalivedCommands.java index 2dd9d4051ed..857918286df 100644 --- a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosKeepalivedCommands.java +++ b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosKeepalivedCommands.java @@ -21,6 +21,8 @@ static public class VyosHaVip{ public String category; @GrayVersion(value = "5.1.0") public Integer prefixLen; + @GrayVersion(value = "5.5.16") + public boolean bindToLo; } public static class VyosHaEnableCmd extends VirtualRouterCommands.AgentCommand {