Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions conf/globalConfig/lb.xml
Original file line number Diff line number Diff line change
Expand Up @@ -109,4 +109,20 @@
<defaultValue>40</defaultValue>
<type>java.lang.Long</type>
</config>

<config>
<category>loadBalancer</category>
<name>ipvs.defaultMode</name>
<description>Default IPVS forwarding mode for IPVS listeners. Options: dr (Direct Routing), fullnat. Empty means haproxy/gobetween path.</description>
<defaultValue></defaultValue>
<type>java.lang.String</type>
</config>

<config>
<category>loadBalancer</category>
<name>ipvs.defaultScheduler</name>
<description>Default IPVS scheduling algorithm. Options: rr (round-robin), wrr (weighted round-robin), lc (least-connections), sh (source-hash).</description>
<defaultValue>rr</defaultValue>
<type>java.lang.String</type>
</config>
</globalConfig>
Original file line number Diff line number Diff line change
Expand Up @@ -779,12 +779,35 @@ private void validate(APICreateLoadBalancerListenerMsg msg) {
validateAcl(msg.getAclUuids(),new ArrayList<>(), msg.getLoadBalancerUuid());
}

insertTagIfNotExisting(
msg, LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT,
LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT.instantiateTag(
map(e(LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT_TOKEN, LoadBalancerGlobalConfig.CONNECTION_IDLE_TIMEOUT.value(Long.class)))
)
);
// Extract ipvsMode early to gate insertion of tags that are incompatible with IPVS listeners.
// Also apply IPVS_DEFAULT_MODE global config if operator has configured one and no explicit tag is given.
String earlyIpvsMode = null;
if (msg.getSystemTags() != null) {
for (String tag : msg.getSystemTags()) {
if (LoadBalancerSystemTags.IPVS_MODE.isMatch(tag)) {
earlyIpvsMode = LoadBalancerSystemTags.IPVS_MODE.getTokenByTag(tag, LoadBalancerSystemTags.IPVS_MODE_TOKEN);
break;
}
}
}
if (earlyIpvsMode == null || earlyIpvsMode.isEmpty()) {
String defaultMode = LoadBalancerGlobalConfig.IPVS_DEFAULT_MODE.value();
if (defaultMode != null && !defaultMode.isEmpty()) {
earlyIpvsMode = defaultMode;
insertTagIfNotExisting(msg, LoadBalancerSystemTags.IPVS_MODE,
LoadBalancerSystemTags.IPVS_MODE.instantiateTag(
map(e(LoadBalancerSystemTags.IPVS_MODE_TOKEN, defaultMode))));
}
}

if (earlyIpvsMode == null || earlyIpvsMode.isEmpty()) {
insertTagIfNotExisting(
msg, LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT,
LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT.instantiateTag(
map(e(LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT_TOKEN, LoadBalancerGlobalConfig.CONNECTION_IDLE_TIMEOUT.value(Long.class)))
)
);
}

insertTagIfNotExisting(
msg, LoadBalancerSystemTags.HEALTHY_THRESHOLD,
Expand Down Expand Up @@ -821,12 +844,14 @@ private void validate(APICreateLoadBalancerListenerMsg msg) {
)
);

insertTagIfNotExisting(
msg, LoadBalancerSystemTags.MAX_CONNECTION,
LoadBalancerSystemTags.MAX_CONNECTION.instantiateTag(
map(e(LoadBalancerSystemTags.MAX_CONNECTION_TOKEN, LoadBalancerGlobalConfig.MAX_CONNECTION.value(Long.class)))
)
);
if (earlyIpvsMode == null || earlyIpvsMode.isEmpty()) {
insertTagIfNotExisting(
msg, LoadBalancerSystemTags.MAX_CONNECTION,
LoadBalancerSystemTags.MAX_CONNECTION.instantiateTag(
map(e(LoadBalancerSystemTags.MAX_CONNECTION_TOKEN, LoadBalancerGlobalConfig.MAX_CONNECTION.value(Long.class)))
)
);
}

insertTagIfNotExisting(
msg, LoadBalancerSystemTags.BALANCER_ALGORITHM,
Expand Down Expand Up @@ -868,7 +893,7 @@ private void validate(APICreateLoadBalancerListenerMsg msg) {
}
}

String algorithm = null, seessionPersistence = null, httpRedirectHttps = null, redirectPort = null, statusCode = null;
String algorithm = null, seessionPersistence = null, httpRedirectHttps = null, redirectPort = null, statusCode = null, ipvsMode = null;
for (String tag : msg.getSystemTags()) {
if (LoadBalancerSystemTags.BALANCER_ALGORITHM.isMatch(tag)) {
algorithm = LoadBalancerSystemTags.BALANCER_ALGORITHM.getTokenByTag(tag,
Expand Down Expand Up @@ -898,6 +923,10 @@ private void validate(APICreateLoadBalancerListenerMsg msg) {
"could not create the loadbalancer listener with systemTag httpCompressAlgos::disable, please remove this tag"));
}
}
if (LoadBalancerSystemTags.IPVS_MODE.isMatch(tag)) {
ipvsMode = LoadBalancerSystemTags.IPVS_MODE.getTokenByTag(tag,
LoadBalancerSystemTags.IPVS_MODE_TOKEN);
}
}

if ((redirectPort != null || statusCode != null) && (httpRedirectHttps == null || HttpRedirectHttps.disable.toString().equals(httpRedirectHttps))) {
Expand Down Expand Up @@ -1179,6 +1208,8 @@ private void validate(APICreateLoadBalancerListenerMsg msg) {
);
}
}

validateIpvsMode(msg.getLoadBalancerUuid(), ipvsMode, msg.getProtocol(), msg.getSystemTags(), algorithm, null);
}

private void validate(APIDeleteLoadBalancerListenerMsg msg) {
Expand All @@ -1192,13 +1223,106 @@ private void validate(APIDeleteLoadBalancerListenerMsg msg) {

msg.setLoadBalancerUuid(lbUuid);
}

private void validateIpvsMode(String lbUuid, String ipvsMode, String protocol, List<String> systemTags, String algorithm, String excludeListenerUuid) {
boolean isIpvs = ipvsMode != null && !ipvsMode.isEmpty();

// Always enforce consistency: all listeners on the same LB must use the same forwarding path
List<String> listenerUuids = Q.New(LoadBalancerListenerVO.class)
.select(LoadBalancerListenerVO_.uuid)
.eq(LoadBalancerListenerVO_.loadBalancerUuid, lbUuid)
.listValues();
for (String listenerUuid : listenerUuids) {
if (listenerUuid.equals(excludeListenerUuid)) {
continue;
}
String existingMode = LoadBalancerSystemTags.IPVS_MODE.getTokenByResourceUuid(listenerUuid, LoadBalancerSystemTags.IPVS_MODE_TOKEN);
boolean existingIsIpvs = existingMode != null && !existingMode.isEmpty();
if (isIpvs != existingIsIpvs) {
throw new ApiMessageInterceptionException(argerr(
"cannot mix ipvs and non-ipvs (haproxy/gobetween) listeners on the same load balancer; " +
"existing listener[uuid:%s] uses [%s]",
listenerUuid, existingIsIpvs ? existingMode : "haproxy/gobetween"));
}
if (isIpvs && !ipvsMode.equals(existingMode)) {
throw new ApiMessageInterceptionException(argerr(
"all listeners on the same load balancer must have the same ipvsMode; " +
"listener[uuid:%s] has ipvsMode[%s] but the new listener specifies [%s]",
listenerUuid, existingMode, ipvsMode));
}
}

if (!isIpvs) {
return;
}

// ipvsMode value must be recognized
if (!LoadBalancerConstants.IPVS_MODES.contains(ipvsMode)) {
throw new ApiMessageInterceptionException(argerr("invalid ipvsMode [%s], supported values: %s", ipvsMode, LoadBalancerConstants.IPVS_MODES));
}

// Only TCP/UDP listeners may use ipvs
if (!LB_PROTOCOL_TCP.equals(protocol) && !LB_PROTOCOL_UDP.equals(protocol)) {
throw new ApiMessageInterceptionException(argerr("ipvsMode is only supported for tcp/udp listeners, but protocol is [%s]", protocol));
}

// ipvs listeners forbid L7 features
for (String tag : systemTags) {
if (LoadBalancerSystemTags.HTTP_MODE.isMatch(tag)) {
throw new ApiMessageInterceptionException(argerr("ipvs listener does not support httpMode"));
}
if (LoadBalancerSystemTags.HTTP_REDIRECT_HTTPS.isMatch(tag)) {
throw new ApiMessageInterceptionException(argerr("ipvs listener does not support http redirect https"));
}
if (LoadBalancerSystemTags.SESSION_PERSISTENCE.isMatch(tag)) {
String sp = LoadBalancerSystemTags.SESSION_PERSISTENCE.getTokenByTag(tag, LoadBalancerSystemTags.SESSION_PERSISTENCE_TOKEN);
if (!LoadBalancerSessionPersistence.disable.toString().equals(sp)) {
throw new ApiMessageInterceptionException(argerr("ipvs listener does not support session persistence"));
}
}
if (LoadBalancerSystemTags.COOKIE_NAME.isMatch(tag)) {
throw new ApiMessageInterceptionException(argerr("ipvs listener does not support cookie-based session persistence"));
}
if (LoadBalancerSystemTags.TCP_PROXYPROTOCOL.isMatch(tag)) {
throw new ApiMessageInterceptionException(argerr("ipvs listener does not support tcp proxy protocol"));
}
if (LoadBalancerSystemTags.MAX_CONNECTION.isMatch(tag)) {
throw new ApiMessageInterceptionException(argerr("ipvs listener does not support maxConnection"));
}
if (LoadBalancerSystemTags.CONNECTION_IDLE_TIMEOUT.isMatch(tag)) {
throw new ApiMessageInterceptionException(argerr("ipvs listener does not support connectionIdleTimeout"));
}
}

// balancerAlgorithm must be in IPVS whitelist when specified
if (algorithm != null && !LoadBalancerConstants.IPVS_ALLOWED_BALANCE_ALGORITHMS.contains(algorithm)) {
throw new ApiMessageInterceptionException(argerr(
"balancerAlgorithm [%s] is not supported in ipvs mode; allowed values: %s",
algorithm, LoadBalancerConstants.IPVS_ALLOWED_BALANCE_ALGORITHMS));
}
}

private void validate(APIUpdateLoadBalancerListenerMsg msg) {
String loadBalancerUuid = Q.New(LoadBalancerListenerVO.class).
select(LoadBalancerListenerVO_.loadBalancerUuid).
eq(LoadBalancerListenerVO_.uuid,msg.
getLoadBalancerListenerUuid()).findValue();
msg.setLoadBalancerUuid(loadBalancerUuid);
bus.makeTargetServiceIdByResourceUuid(msg, LoadBalancerConstants.SERVICE_ID, loadBalancerUuid);
// Validate ipvsMode consistency if the update carries a new ipvsMode tag
String newIpvsMode = null;
for (String tag : msg.getSystemTags()) {
if (LoadBalancerSystemTags.IPVS_MODE.isMatch(tag)) {
newIpvsMode = LoadBalancerSystemTags.IPVS_MODE.getTokenByTag(tag, LoadBalancerSystemTags.IPVS_MODE_TOKEN);
}
}
if (newIpvsMode != null) {
String protocol = Q.New(LoadBalancerListenerVO.class)
.select(LoadBalancerListenerVO_.protocol)
.eq(LoadBalancerListenerVO_.uuid, msg.getLoadBalancerListenerUuid())
.findValue();
validateIpvsMode(loadBalancerUuid, newIpvsMode, protocol, msg.getSystemTags(), null, msg.getLoadBalancerListenerUuid());
}
Comment on lines +1312 to +1325
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

更新 listener 时的 IPVS 校验还不完整。

这里仅在请求里显式带了 IPVS_MODE tag 时才调用 validateIpvsMode(),而且传进去的是 msg.getSystemTags()。这会漏掉两类场景:1)一个已经是 IPVS 的 listener 只改 balancerAlgorithm,不会再走 IPVS 白名单校验;2)把现有非 IPVS listener 切到 IPVS 时,库里已经存在的 maxConnection / connectionIdleTimeout / session* 等历史 tag 不会被检查。后面 plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/lb/VirtualRouterLoadBalancerBackend.javamakeLbTOs() 仍会把这些旧 tag 放进 parameters 下发,最终形成非法组合。建议这里先读取当前 listener 的持久化 tags 和当前 ipvsMode,合并出一份 effective 配置后再统一做校验。

🤖 Prompt for AI Agents
Verify each finding against the current code and only fix it if needed.

In
`@plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java`
around lines 1312 - 1325, The update currently calls validateIpvsMode() only
when an IPVS_MODE tag is present in msg.getSystemTags(), which misses cases
where a listener already using IPVS changes other tags or when switching a
non-IPVS listener to IPVS because persisted listener tags (maxConnection,
connectionIdleTimeout, session*, etc.) aren’t considered; fix by loading the
existing listener’s persisted system tags and current ipvsMode (via the
LoadBalancerListenerVO query used to get protocol), merge those persisted tags
with msg.getSystemTags() to produce an effective tag set and determine the
effective ipvsMode, then call validateIpvsMode(loadBalancerUuid,
effectiveIpvsMode, protocol, mergedTags, null,
msg.getLoadBalancerListenerUuid()) so all historical and new tags are validated
together before applying the update.

}

private void validate(APIAddCertificateToLoadBalancerListenerMsg msg) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,28 @@ public static enum Param {

public static final String HEALTH_CHECK_TARGET_DEFAULT = "default";

// IPVS forwarding modes for LoadBalancerListenerVO systemTag "ipvsMode::{mode}"
public static final String IPVS_MODE_DR = "dr";
public static final String IPVS_MODE_FULLNAT = "fullnat";
public static final List<String> IPVS_MODES = asList(IPVS_MODE_DR, IPVS_MODE_FULLNAT);

// IPVS scheduler names that map 1:1 from BALANCE_ALGORITHM_* values
public static final String IPVS_SCHEDULER_RR = "rr";
public static final String IPVS_SCHEDULER_WRR = "wrr";
public static final String IPVS_SCHEDULER_LC = "lc";
public static final String IPVS_SCHEDULER_SH = "sh";
// Algorithms allowed when ipvsMode is set (enforced by interceptor)
public static final List<String> IPVS_ALLOWED_BALANCE_ALGORITHMS = asList(
BALANCE_ALGORITHM_ROUND_ROBIN,
BALANCE_ALGORITHM_WEIGHT_ROUND_ROBIN,
BALANCE_ALGORITHM_LEAST_CONN,
BALANCE_ALGORITHM_LEAST_SOURCE
);

// IPVS connection type flags passed to ipvsadm (-g = DR gate, -m = masquerade/fullnat)
public static final String IPVS_CONNECTION_TYPE_DR = "-g";
public static final String IPVS_CONNECTION_TYPE_FULLNAT = "-m";

public static final List<VmInstanceConstant.VmOperation> vmOperationForDetachListener = asList(
VmInstanceConstant.VmOperation.Destroy,
VmInstanceConstant.VmOperation.DetachNic,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,5 +36,9 @@ public class LoadBalancerGlobalConfig {
@GlobalConfigValidation
public static GlobalConfig HTTP_MODE = new GlobalConfig(CATEGORY, "httpMode");

@GlobalConfigValidation
public static GlobalConfig IPVS_DEFAULT_MODE = new GlobalConfig(CATEGORY, "ipvs.defaultMode");

@GlobalConfigValidation
public static GlobalConfig IPVS_DEFAULT_SCHEDULER = new GlobalConfig(CATEGORY, "ipvs.defaultScheduler");
Comment on lines +39 to +43
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

请在全局配置层限制 IPVS 默认值的取值范围。

这里的校验还不够强:ipvs.defaultScheduler 现在可以被写成任意字符串,而 plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/lb/VirtualRouterLoadBalancerBackend.javamapBalancerAlgorithmToIpvsScheduler() 会直接把它当 fallback 下发给 VR;ipvs.defaultMode 也会在创建 listener 时才迟到失败。建议把这两个配置收敛到明确的允许集合(LoadBalancerConstants.IPVS_MODESrr/wrr/lc/sh),避免坏配置先落库、再在运行时爆出来。

🤖 Prompt for AI Agents
Verify each finding against the current code and only fix it if needed.

In
`@plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerGlobalConfig.java`
around lines 39 - 43, 限制全局配置 IPVS 默认值的取值范围:为 GlobalConfig IPVS_DEFAULT_MODE 和
IPVS_DEFAULT_SCHEDULER 增加严格校验,确保 IPVS_DEFAULT_MODE 只接受
LoadBalancerConstants.IPVS_MODES 中定义的值,IPVS_DEFAULT_SCHEDULER 只接受集合
{"rr","wrr","lc","sh"};在其 GlobalConfigValidation 的实现或注册点(与 IPVS_DEFAULT_MODE /
IPVS_DEFAULT_SCHEDULER 声明相近的校验器)添加检查逻辑并在不符合时拒绝/报错,避免非法字符串被写入后由
VirtualRouterLoadBalancerBackend.mapBalancerAlgorithmToIpvsScheduler 或 listener
创建时才失败。

}
Original file line number Diff line number Diff line change
Expand Up @@ -87,4 +87,7 @@ public class LoadBalancerSystemTags {

public static final String HTTP_COMPRESS_ALGOS_TOKEN = "httpCompressAlgos";
public static PatternedSystemTag HTTP_COMPRESS_ALGOS= new PatternedSystemTag(String.format("httpCompressAlgos::{%s}", HTTP_COMPRESS_ALGOS_TOKEN), LoadBalancerListenerVO.class);

public static final String IPVS_MODE_TOKEN = "ipvsMode";
public static PatternedSystemTag IPVS_MODE = new PatternedSystemTag(String.format("ipvsMode::{%s}", IPVS_MODE_TOKEN), LoadBalancerListenerVO.class);
}
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,11 @@ public static class LbTO {

boolean enableStatsLog;

// IPVS fields: populated from ipvsMode systemTag; empty string = haproxy/gobetween path
String ipvsMode; // "dr" | "fullnat" | ""
String scheduler; // "rr" | "wrr" | "lc" | "sh" | ""
String connectionType; // "-g" (DR) | "-m" (fullnat) | ""

public static class ServerGroup {
private String name;
private String serverGroupUuid;
Expand Down Expand Up @@ -525,6 +530,30 @@ public String getVipL3Uuid() {
public void setVipL3Uuid(String vipL3Uuid) {
this.vipL3Uuid = vipL3Uuid;
}

public String getIpvsMode() {
return ipvsMode;
}

public void setIpvsMode(String ipvsMode) {
this.ipvsMode = ipvsMode;
}

public String getScheduler() {
return scheduler;
}

public void setScheduler(String scheduler) {
this.scheduler = scheduler;
}

public String getConnectionType() {
return connectionType;
}

public void setConnectionType(String connectionType) {
this.connectionType = connectionType;
}
}

public static class RefreshLbCmd extends AgentCommand {
Expand Down Expand Up @@ -660,6 +689,23 @@ public boolean enableStatsLog(LbTO to) {
}
}

// Maps ZStack balancerAlgorithm names to IPVS scheduler names (1:1 mapping)
private static String mapBalancerAlgorithmToIpvsScheduler(String algorithm) {
String configDefault = LoadBalancerGlobalConfig.IPVS_DEFAULT_SCHEDULER.value();
String fallback = (configDefault != null && !configDefault.isEmpty())
? configDefault : LoadBalancerConstants.IPVS_SCHEDULER_RR;
if (algorithm == null) {
return fallback;
}
switch (algorithm) {
case LoadBalancerConstants.BALANCE_ALGORITHM_ROUND_ROBIN: return LoadBalancerConstants.IPVS_SCHEDULER_RR;
case LoadBalancerConstants.BALANCE_ALGORITHM_WEIGHT_ROUND_ROBIN: return LoadBalancerConstants.IPVS_SCHEDULER_WRR;
case LoadBalancerConstants.BALANCE_ALGORITHM_LEAST_CONN: return LoadBalancerConstants.IPVS_SCHEDULER_LC;
case LoadBalancerConstants.BALANCE_ALGORITHM_LEAST_SOURCE: return LoadBalancerConstants.IPVS_SCHEDULER_SH;
default: return fallback;
}
}

public List<LbTO> makeCommonLbTOs(final LoadBalancerStruct struct) {
return makeLbTOs(struct, null);
}
Expand Down Expand Up @@ -904,6 +950,21 @@ public LbTO call(LoadBalancerListenerInventory l) {
if (vip6 != null) {
to.setVip6(vip6.getIp());
}

// Populate IPVS fields from ipvsMode systemTag
String ipvsMode = LoadBalancerSystemTags.IPVS_MODE.getTokenByResourceUuid(l.getUuid(), LoadBalancerSystemTags.IPVS_MODE_TOKEN);
if (ipvsMode != null && !ipvsMode.isEmpty()) {
to.setIpvsMode(ipvsMode);
// Map balancerAlgorithm tag to IPVS scheduler name
String balancerAlgorithm = LoadBalancerSystemTags.BALANCER_ALGORITHM.getTokenByResourceUuid(l.getUuid(), LoadBalancerSystemTags.BALANCER_ALGORITHM_TOKEN);
to.setScheduler(mapBalancerAlgorithmToIpvsScheduler(balancerAlgorithm));
// Set connection type flag
if (LoadBalancerConstants.IPVS_MODE_DR.equals(ipvsMode)) {
to.setConnectionType(LoadBalancerConstants.IPVS_CONNECTION_TYPE_DR);
} else if (LoadBalancerConstants.IPVS_MODE_FULLNAT.equals(ipvsMode)) {
to.setConnectionType(LoadBalancerConstants.IPVS_CONNECTION_TYPE_FULLNAT);
}
}
to.setSecurityPolicyType(l.getSecurityPolicyType());
if (l.getCertificateRefs() != null && !l.getCertificateRefs().isEmpty()) {
to.setCertificateUuid(l.getCertificateRefs().get(0).getCertificateUuid());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ static public class VyosHaVip{
public String category;
@GrayVersion(value = "5.1.0")
public Integer prefixLen;
@GrayVersion(value = "5.5.16")
public boolean bindToLo;
}

public static class VyosHaEnableCmd extends VirtualRouterCommands.AgentCommand {
Expand Down