From 342802f6a7cb9d02fbc500894c04b01aff1062fe Mon Sep 17 00:00:00 2001 From: sesky4 Date: Fri, 24 Apr 2026 17:41:35 +0800 Subject: [PATCH 01/11] feat: TLD-level endpoint failover with per-TLD circuit breakers Adds EndpointFailoverInterceptor that automatically retries against backup TLDs (.cn, .com.cn) when the primary tencentcloudapi.com domain fails due to DNS, TLS verification, or network reachability issues. The interceptor recovers all signing inputs from the outgoing Request and re-signs against the backup host using credentials/profile read live from the owning AbstractClient, so credential rotation is honoured on every retry. No per-request tag is required. Per-TLD CircuitBreakers (60s timeout) suppress repeated attempts against a failing TLD and probe recovery via half-open. Failover is opt-out via ClientProfile.setDomainFailover(false). --- .../common/AbstractClient.java | 3 + .../common/EndpointFailoverInterceptor.java | 643 ++++++++++++++++++ .../common/profile/ClientProfile.java | 30 + .../EndpointFailoverInterceptorTest.java | 420 ++++++++++++ 4 files changed, 1096 insertions(+) create mode 100644 src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java create mode 100644 src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java diff --git a/src/main/java/com/tencentcloudapi/common/AbstractClient.java b/src/main/java/com/tencentcloudapi/common/AbstractClient.java index f182117def..d0434c81df 100644 --- a/src/main/java/com/tencentcloudapi/common/AbstractClient.java +++ b/src/main/java/com/tencentcloudapi/common/AbstractClient.java @@ -134,6 +134,9 @@ public AbstractClient( this.profile.getHttpProfile().getWriteTimeout() ); this.httpConnection.addInterceptors(this.log); + if (this.profile.getDomainFailover()) { + this.httpConnection.addInterceptors(new EndpointFailoverInterceptor(this)); + } this.trySetProxy(this.httpConnection); this.trySetSSLSocketFactory(this.httpConnection); this.trySetRegionBreaker(); diff --git a/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java b/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java new file mode 100644 index 0000000000..9a6c99d296 --- /dev/null +++ b/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java @@ -0,0 +1,643 @@ +/* + * Copyright (c) 2018 Tencent. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.tencentcloudapi.common; + +import com.tencentcloudapi.common.exception.TencentCloudSDKException; +import com.tencentcloudapi.common.profile.ClientProfile; +import com.tencentcloudapi.common.profile.HttpProfile; +import okhttp3.Headers; +import okhttp3.HttpUrl; +import okhttp3.Interceptor; +import okhttp3.MediaType; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import okio.Buffer; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.ConnectException; +import java.net.NoRouteToHostException; +import java.net.PortUnreachableException; +import java.net.SocketTimeoutException; +import java.net.URLDecoder; +import java.net.URLEncoder; +import java.net.UnknownHostException; +import java.nio.charset.StandardCharsets; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import javax.net.ssl.SSLHandshakeException; +import javax.net.ssl.SSLPeerUnverifiedException; + +/** + * OkHttp interceptor that implements TLD-level domain failover for Tencent + * Cloud API calls. When DNS resolution of the primary + * {@code *.tencentcloudapi.com} domain fails, the interceptor automatically + * rewrites the request against the next known backup TLD ({@code .cn}, + * {@code .com.cn}) and re-signs it using the owning + * {@link AbstractClient}'s live credential and profile, then retries. + *

+ * All signing inputs (method, URL, headers, body) are recovered directly from + * the outgoing {@link Request}; no per-request tag is required. Credentials + * and profile are read live on each retry, so credential rotation is honoured. + *

+ * Failover state is JVM-global and opportunistic: per-TLD circuit breakers + * suppress repeated attempts against a failing TLD; after the configured + * breaker timeout (default {@value #DEFAULT_BREAKER_TIMEOUT_MS} ms) the + * breaker half-opens and the next request probes that TLD again. + */ +public class EndpointFailoverInterceptor implements Interceptor { + + /** All known TLDs. The failover order rotates based on the user's original endpoint. */ + static final String[] KNOWN_TLDS = new String[]{ + "tencentcloudapi.com", + "tencentcloudapi.cn", + "tencentcloudapi.com.cn" + }; + + /** + * CircuitBreaker timeout (ms) controlling how long an Open breaker stays + * Open before transitioning to HalfOpen and probing the TLD again. + * Can be changed globally before any client is constructed. + * Default: 60 s. + */ + public static long DEFAULT_BREAKER_TIMEOUT_MS = 60 * 1000; + + private final AbstractClient client; + private final long breakerTimeoutMs; + + /** + * Per-host failover state, keyed by the ORIGINAL host. Shared across all + * AbstractClient instances in the JVM. + */ + static final ConcurrentHashMap STATE = + new ConcurrentHashMap(); + + /** Creates an interceptor bound to {@code client} with the default breaker timeout (60 s). */ + public EndpointFailoverInterceptor(AbstractClient client) { + this(client, DEFAULT_BREAKER_TIMEOUT_MS); + } + + /** + * Creates an interceptor with a custom breaker timeout. + * + * @param client the owning client; credential and profile are read live on each retry. + * @param breakerTimeoutMs ms to keep a breaker Open before probing its TLD again. + */ + public EndpointFailoverInterceptor(AbstractClient client, long breakerTimeoutMs) { + this.client = client; + this.breakerTimeoutMs = breakerTimeoutMs; + } + + /** Visible for testing. Resets all failover state. */ + static void resetStateForTesting() { + STATE.clear(); + } + + @Override + public Response intercept(Chain chain) throws IOException { + Request request = chain.request(); + + if (!eligibleForFailover(request)) { + return chain.proceed(request); + } + + Failover failover = new Failover(request); + + for (String candidate : failover.candidates()) { + Response response = failover.attempt(candidate, chain); + if (response != null) { + return response; + } + } + + return failover.giveUp(chain); + } + + /** + * Stateful helper that drives a single intercept. Keeps {@link #intercept} + * readable as a narrative while hiding the per-TLD circuit breaker, re-sign + * and DNS-failure bookkeeping. + */ + private final class Failover { + private final Request request; + private final String originHost; + private final int originIdx; + private final FailoverState state; + private IOException lastFailure; + + private Failover(Request request) { + this.request = request; + this.originHost = request.url().host(); + this.originIdx = tldIndexOf(originHost); + this.state = getOrCreateState(originHost); + } + + /** Host candidates in preferred try order (last-known-working first). */ + List candidates() { + List hosts = new ArrayList(KNOWN_TLDS.length); + for (int tldIdx : buildTryOrder(state, originIdx)) { + hosts.add(hostForTld(tldIdx)); + } + return hosts; + } + + /** + * Attempts a single request against {@code host}. Returns the response + * on success, or {@code null} if the caller should keep trying (breaker + * open, or DNS / network-reachability failure this attempt). Failures + * unrelated to host reachability propagate. + */ + Response attempt(String host, Chain chain) throws IOException { + int tldIdx = tldIndexOf(host); + CircuitBreaker.Token token = state.breakers[tldIdx].allow(); + if (!token.allowed) { + return null; + } + Request attempt; + try { + attempt = rewriteFor(host); + } catch (TencentCloudSDKException e) { + throw new IOException("Failed to re-sign request for failover: " + e.getMessage(), e); + } + try { + Response response = chain.proceed(attempt); + token.report(true); + state.currentIndex = tldIdx; + return response; + } catch (IOException e) { + if (!shouldFailover(e)) { + throw e; + } + token.report(false); + lastFailure = e; + return null; + } + } + + /** + * All breakers Open. Probe the last-known-working TLD once so traffic + * can recover once the network heals; otherwise surface the last failure. + */ + Response giveUp(Chain chain) throws IOException { + if (lastFailure == null) { + String probeHost = hostForTld(probeTldIdx()); + Request attempt; + try { + attempt = rewriteFor(probeHost); + } catch (TencentCloudSDKException e) { + throw new IOException("Failed to re-sign request for failover: " + e.getMessage(), e); + } + return chain.proceed(attempt); + } + throw lastFailure; + } + + private Request rewriteFor(String host) throws TencentCloudSDKException, IOException { + return rewriteForTld(request, originHost, originIdx, tldIndexOf(host)); + } + + private String hostForTld(int tldIdx) { + return tldIdx == originIdx + ? originHost + : substituteTld(originHost, KNOWN_TLDS[originIdx], KNOWN_TLDS[tldIdx]); + } + + private int probeTldIdx() { + return state.currentIndex >= 0 ? state.currentIndex : originIdx; + } + } + + /** + * A request is eligible for TLD failover when: + *

+ */ + private boolean eligibleForFailover(Request request) { + ClientProfile profile = client.getClientProfile(); + if (profile == null || !profile.getDomainFailover()) { + return false; + } + HttpProfile httpProfile = profile.getHttpProfile(); + if (httpProfile != null && httpProfile.getApigwEndpoint() != null) { + return false; + } + String host = request.url().host(); + if (tldIndexOf(host) < 0) { + return false; + } + String signMethod = profile.getSignMethod(); + if (ClientProfile.SIGN_TC3_256.equals(signMethod)) { + String auth = request.header("Authorization"); + if (auth == null || "SKIP".equals(auth)) { + return false; + } + } else if (!ClientProfile.SIGN_SHA1.equals(signMethod) + && !ClientProfile.SIGN_SHA256.equals(signMethod)) { + return false; + } + return true; + } + + /** + * Order: last-known-working TLD first, then the rest in natural KNOWN_TLDS order. + * If no successful TLD has been recorded yet, start from the origin. + */ + private int[] buildTryOrder(FailoverState state, int originIdx) { + int startIdx = state.currentIndex >= 0 ? state.currentIndex : originIdx; + int n = KNOWN_TLDS.length; + int[] order = new int[n]; + order[0] = startIdx; + int pos = 1; + for (int i = 0; i < n; i++) { + if (i != startIdx) { + order[pos++] = i; + } + } + return order; + } + + private FailoverState getOrCreateState(String originalHost) { + FailoverState s = STATE.get(originalHost); + if (s != null) { + return s; + } + FailoverState created = new FailoverState(breakerTimeoutMs); + FailoverState prev = STATE.putIfAbsent(originalHost, created); + return prev != null ? prev : created; + } + + /** + * Build a request targeting {@code KNOWN_TLDS[tldIndex]}. If the target TLD + * equals the origin, the original (already-signed) request is returned as-is. + * Otherwise the request is re-signed for the new host by recovering the signing + * inputs from the original Request and reading credentials live from {@link #client}. + */ + private Request rewriteForTld(Request original, String originalHost, int originIdx, int tldIndex) + throws TencentCloudSDKException, IOException { + if (tldIndex == originIdx) { + return original; + } + String targetHost = substituteTld(originalHost, KNOWN_TLDS[originIdx], KNOWN_TLDS[tldIndex]); + if (targetHost.equals(originalHost)) { + return original; + } + + String signMethod = client.getClientProfile().getSignMethod(); + if (ClientProfile.SIGN_TC3_256.equals(signMethod)) { + return resignTC3(original, targetHost); + } + if (ClientProfile.SIGN_SHA1.equals(signMethod) || ClientProfile.SIGN_SHA256.equals(signMethod)) { + return resignHmac(original, targetHost); + } + throw new TencentCloudSDKException( + "Signature method " + signMethod + " is invalid or not supported yet."); + } + + // ======================================================================== + // TC3-HMAC-SHA256 resign + // ======================================================================== + + private Request resignTC3(Request original, String targetHost) + throws TencentCloudSDKException, IOException { + Credential credential = client.getCredential(); + ClientProfile profile = client.getClientProfile(); + + String httpMethod = original.method(); + String contentType = original.header("Content-Type"); + if (contentType == null) { + contentType = "application/x-www-form-urlencoded"; + } + + byte[] payload = readRequestBody(original); + + String canonicalUri = original.url().encodedPath(); + if (canonicalUri == null || canonicalUri.isEmpty()) { + canonicalUri = "/"; + } + String canonicalQueryString = canonicalQueryStringFromUrl(original.url(), httpMethod); + String canonicalHeaders = "content-type:" + contentType + "\nhost:" + targetHost + "\n"; + String signedHeaders = "content-type;host"; + + String hashedRequestPayload; + if (profile.isUnsignedPayload()) { + hashedRequestPayload = Sign.sha256Hex("UNSIGNED-PAYLOAD".getBytes(StandardCharsets.UTF_8)); + } else { + hashedRequestPayload = Sign.sha256Hex(payload); + } + String canonicalRequest = httpMethod + "\n" + + canonicalUri + "\n" + + canonicalQueryString + "\n" + + canonicalHeaders + "\n" + + signedHeaders + "\n" + + hashedRequestPayload; + + String timestamp = String.valueOf(System.currentTimeMillis() / 1000); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + sdf.setTimeZone(TimeZone.getTimeZone("UTC")); + String date = sdf.format(new Date(Long.valueOf(timestamp + "000"))); + String service = targetHost.split("\\.")[0]; + String credentialScope = date + "/" + service + "/tc3_request"; + String hashedCanonicalRequest = Sign.sha256Hex(canonicalRequest.getBytes(StandardCharsets.UTF_8)); + String stringToSign = "TC3-HMAC-SHA256\n" + timestamp + "\n" + + credentialScope + "\n" + hashedCanonicalRequest; + + String secretId = credential.getSecretId(); + String secretKey = credential.getSecretKey(); + byte[] secretDate = Sign.hmac256(("TC3" + secretKey).getBytes(StandardCharsets.UTF_8), date); + byte[] secretService = Sign.hmac256(secretDate, service); + byte[] secretSigning = Sign.hmac256(secretService, "tc3_request"); + String signature = DatatypeConverter + .printHexBinary(Sign.hmac256(secretSigning, stringToSign)) + .toLowerCase(); + String authorization = "TC3-HMAC-SHA256 " + + "Credential=" + secretId + "/" + credentialScope + ", " + + "SignedHeaders=" + signedHeaders + ", " + + "Signature=" + signature; + + // Preserve all original headers EXCEPT the ones we regenerate, and substitute them. + Headers.Builder hb = new Headers.Builder(); + Headers origHeaders = original.headers(); + for (int i = 0; i < origHeaders.size(); i++) { + String name = origHeaders.name(i); + if (name.equalsIgnoreCase("Host") + || name.equalsIgnoreCase("Authorization") + || name.equalsIgnoreCase("X-TC-Timestamp")) { + continue; + } + hb.add(name, origHeaders.value(i)); + } + hb.add("Host", targetHost); + hb.add("Authorization", authorization); + hb.add("X-TC-Timestamp", timestamp); + // Token may have rotated on the credential — re-inject if present. + String token = credential.getToken(); + if (token != null && !token.isEmpty()) { + hb.set("X-TC-Token", token); + } else { + hb.removeAll("X-TC-Token"); + } + + HttpUrl newUrl = original.url().newBuilder().host(targetHost).build(); + + Request.Builder rb = original.newBuilder() + .url(newUrl) + .headers(hb.build()); + // Body is re-attached below so the rebuilt Request.body() is non-null even though + // original.newBuilder() already preserves it; keeping this explicit defends against + // future OkHttp behavior changes. + if (HttpProfile.REQ_POST.equalsIgnoreCase(httpMethod)) { + rb.post(RequestBody.create(MediaType.parse(contentType), payload)); + } else if (HttpProfile.REQ_GET.equalsIgnoreCase(httpMethod)) { + rb.get(); + } + return rb.build(); + } + + /** + * Recovers the canonical query string from a URL for TC3 signing. TC3 requires + * sorted, URL-encoded {@code key=value} pairs; this matches what + * {@code AbstractClient.getCanonicalQueryString} produces (which now sorts via TreeMap). + */ + private static String canonicalQueryStringFromUrl(HttpUrl url, String method) + throws TencentCloudSDKException { + if (HttpProfile.REQ_POST.equalsIgnoreCase(method)) { + return ""; + } + TreeMap sorted = new TreeMap(); + int size = url.querySize(); + for (int i = 0; i < size; i++) { + String name = url.queryParameterName(i); + String value = url.queryParameterValue(i); + if (value == null) { + value = ""; + } + sorted.put(name, value); + } + StringBuilder sb = new StringBuilder(); + for (Map.Entry e : sorted.entrySet()) { + try { + String v = URLEncoder.encode(e.getValue(), "UTF8"); + if (sb.length() > 0) { + sb.append("&"); + } + sb.append(e.getKey()).append("=").append(v); + } catch (UnsupportedEncodingException ex) { + throw new TencentCloudSDKException("UTF8 is not supported.", ex); + } + } + return sb.toString(); + } + + private static byte[] readRequestBody(Request request) throws IOException { + RequestBody body = request.body(); + if (body == null) { + return new byte[0]; + } + Buffer buffer = new Buffer(); + body.writeTo(buffer); + return buffer.readByteArray(); + } + + // ======================================================================== + // HmacSHA1 / HmacSHA256 resign + // ======================================================================== + + private Request resignHmac(Request original, String targetHost) + throws TencentCloudSDKException, IOException { + Credential credential = client.getCredential(); + ClientProfile profile = client.getClientProfile(); + String reqMethod = original.method(); + + Map params; + if (HttpProfile.REQ_GET.equalsIgnoreCase(reqMethod)) { + params = decodeQueryParams(original.url()); + } else if (HttpProfile.REQ_POST.equalsIgnoreCase(reqMethod)) { + byte[] bytes = readRequestBody(original); + params = decodeFormParams(new String(bytes, StandardCharsets.UTF_8)); + } else { + throw new TencentCloudSDKException("Method only support (GET, POST) for Hmac sign"); + } + params.remove("Signature"); + + // Refresh credential-derived params in case they rotated. + if (credential.getSecretId() != null && !credential.getSecretId().isEmpty()) { + params.put("SecretId", credential.getSecretId()); + } + if (credential.getToken() != null && !credential.getToken().isEmpty()) { + params.put("Token", credential.getToken()); + } else { + params.remove("Token"); + } + + String plainText = Sign.makeSignPlainText( + new TreeMap(params), + reqMethod, + targetHost, + original.url().encodedPath()); + String signature = Sign.sign(credential.getSecretKey(), plainText, profile.getSignMethod()); + + StringBuilder body = new StringBuilder(); + try { + for (Map.Entry entry : params.entrySet()) { + body.append(URLEncoder.encode(entry.getKey(), "utf-8")) + .append("=") + .append(URLEncoder.encode(entry.getValue(), "utf-8")) + .append("&"); + } + body.append("Signature=").append(URLEncoder.encode(signature, "utf-8")); + } catch (UnsupportedEncodingException e) { + throw new TencentCloudSDKException("", e); + } + + HttpUrl newUrl = original.url().newBuilder().host(targetHost).build(); + Request.Builder rb = original.newBuilder(); + + if (HttpProfile.REQ_GET.equalsIgnoreCase(reqMethod)) { + HttpUrl urlWithQuery = newUrl.newBuilder().encodedQuery(body.toString()).build(); + rb.url(urlWithQuery).get(); + } else { + rb.url(newUrl).post(RequestBody.create( + MediaType.parse("application/x-www-form-urlencoded"), + body.toString())); + } + // Preserve Host header if present, pointing at the new host. + if (original.header("Host") != null) { + rb.header("Host", targetHost); + } + return rb.build(); + } + + private static Map decodeQueryParams(HttpUrl url) { + LinkedHashMap map = new LinkedHashMap(); + int size = url.querySize(); + for (int i = 0; i < size; i++) { + String name = url.queryParameterName(i); + String value = url.queryParameterValue(i); + if (value == null) { + value = ""; + } + map.put(name, value); + } + return map; + } + + private static Map decodeFormParams(String body) throws TencentCloudSDKException { + LinkedHashMap map = new LinkedHashMap(); + if (body == null || body.isEmpty()) { + return map; + } + for (String pair : body.split("&")) { + int eq = pair.indexOf('='); + String k = eq < 0 ? pair : pair.substring(0, eq); + String v = eq < 0 ? "" : pair.substring(eq + 1); + try { + map.put(URLDecoder.decode(k, "utf-8"), URLDecoder.decode(v, "utf-8")); + } catch (UnsupportedEncodingException e) { + throw new TencentCloudSDKException("UTF-8 not supported", e); + } + } + return map; + } + + // ======================================================================== + // Host helpers + // ======================================================================== + + /** + * Failures we treat as "host unreachable / suspicious" and worth retrying + * against another TLD: outright DNS misses, TLS verification failures + * (a strong DNS-tampering signal), and connect/timeout/route errors that + * can mean the resolved IP is a black hole. SocketTimeoutException covers + * both connect and read timeouts; CircuitBreaker absorbs false positives + * for genuinely slow servers. + */ + private static boolean shouldFailover(IOException e) { + return e instanceof UnknownHostException + || e instanceof SSLPeerUnverifiedException + || e instanceof SSLHandshakeException + || e instanceof ConnectException + || e instanceof NoRouteToHostException + || e instanceof PortUnreachableException + || e instanceof SocketTimeoutException; + } + + static int tldIndexOf(String host) { + if (host == null) { + return -1; + } + for (int i = 0; i < KNOWN_TLDS.length; i++) { + String suffix = "." + KNOWN_TLDS[i]; + if (host.endsWith(suffix)) { + String prefix = host.substring(0, host.length() - suffix.length()); + if (!prefix.isEmpty() && !prefix.startsWith(".") && !prefix.endsWith(".")) { + return i; + } + } + } + return -1; + } + + static boolean isKnownTencentCloudHost(String host) { + return tldIndexOf(host) >= 0; + } + + static String substituteTld(String host, String fromTld, String toTld) { + if (host == null) { + return null; + } + String suffix = "." + fromTld; + if (host.endsWith(suffix)) { + return host.substring(0, host.length() - fromTld.length()) + toTld; + } + if (host.equals(fromTld)) { + return toTld; + } + return host; + } + + /** + * Per-originalHost failover state. Holds one {@link CircuitBreaker} per TLD + * plus the last-known-working TLD index. + */ + static final class FailoverState { + final CircuitBreaker[] breakers; + /** Index in {@link #KNOWN_TLDS} of the last TLD that served a successful request; -1 until first success. */ + volatile int currentIndex = -1; + + FailoverState(long breakerTimeoutMs) { + int n = KNOWN_TLDS.length; + this.breakers = new CircuitBreaker[n]; + for (int i = 0; i < n; i++) { + CircuitBreaker.Setting s = new CircuitBreaker.Setting(); + s.timeoutMs = breakerTimeoutMs; + this.breakers[i] = new CircuitBreaker(s); + } + } + } +} diff --git a/src/main/java/com/tencentcloudapi/common/profile/ClientProfile.java b/src/main/java/com/tencentcloudapi/common/profile/ClientProfile.java index c26389200d..fad6025ebf 100644 --- a/src/main/java/com/tencentcloudapi/common/profile/ClientProfile.java +++ b/src/main/java/com/tencentcloudapi/common/profile/ClientProfile.java @@ -67,6 +67,15 @@ public class ClientProfile { // Backup endpoint for API requests, useful in case the primary endpoint fails. private String backupEndpoint; + /** + * Flag indicating whether domain-level failover is enabled. + * When true (default), the SDK will automatically switch to backup TLDs + * (e.g. tencentcloudapi.com.cn / tencentcloudapi.cn) on DNS / TLS / network + * reachability failures of the primary domain. Custom apigw endpoints and + * SkipSign requests are passed through unchanged. + */ + private boolean domainFailover = true; + /** * Constructor to initialize ClientProfile with a specific signing method and HTTP profile. * If the signing method is null or empty, it defaults to "TC3-HMAC-SHA256". @@ -211,4 +220,25 @@ public String getBackupEndpoint() { public void setBackupEndpoint(String backupEndpoint) { this.backupEndpoint = backupEndpoint; } + + /** + * Getter for the domain failover flag. + * + * @return true if domain failover is enabled (default), false otherwise. + */ + public boolean getDomainFailover() { + return this.domainFailover; + } + + /** + * Setter for the domain failover flag. Domain failover causes the SDK + * to automatically switch to backup TLDs on DNS / TLS / network + * reachability failures of the primary *.tencentcloudapi.com domain. + * Set to false to opt out. + * + * @param enabled true to enable (default), false to disable. + */ + public void setDomainFailover(boolean enabled) { + this.domainFailover = enabled; + } } diff --git a/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java b/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java new file mode 100644 index 0000000000..ce2aea1482 --- /dev/null +++ b/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java @@ -0,0 +1,420 @@ +/* + * Copyright (c) 2018 Tencent. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.tencentcloudapi.common; + +import com.tencentcloudapi.common.profile.ClientProfile; +import com.tencentcloudapi.common.profile.HttpProfile; +import okhttp3.Call; +import okhttp3.Connection; +import okhttp3.Interceptor; +import okhttp3.MediaType; +import okhttp3.Protocol; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** Unit tests for {@link EndpointFailoverInterceptor}. Pure in-process tests; no network. */ +public class EndpointFailoverInterceptorTest { + + @Before + public void resetState() { + EndpointFailoverInterceptor.resetStateForTesting(); + } + + // ---- Pure helper tests ---- + + @Test + public void testIsKnownTencentCloudHost() { + assertTrue(EndpointFailoverInterceptor.isKnownTencentCloudHost("cvm.tencentcloudapi.com")); + assertTrue(EndpointFailoverInterceptor.isKnownTencentCloudHost("cvm.ap-shanghai.tencentcloudapi.com")); + assertTrue(EndpointFailoverInterceptor.isKnownTencentCloudHost("hunyuan.ai.ap-guangzhou.tencentcloudapi.com")); + assertTrue(EndpointFailoverInterceptor.isKnownTencentCloudHost("hunyuan.ai.tencentcloudapi.com")); + assertTrue(EndpointFailoverInterceptor.isKnownTencentCloudHost("cvm.tencentcloudapi.cn")); + assertTrue(EndpointFailoverInterceptor.isKnownTencentCloudHost("cvm.tencentcloudapi.com.cn")); + assertFalse(EndpointFailoverInterceptor.isKnownTencentCloudHost("example.com")); + assertFalse(EndpointFailoverInterceptor.isKnownTencentCloudHost("cvm.tencentcloudapi.woa.com")); + assertFalse(EndpointFailoverInterceptor.isKnownTencentCloudHost("proxy.internal")); + assertFalse(EndpointFailoverInterceptor.isKnownTencentCloudHost(null)); + } + + @Test + public void testSubstituteTldPreservesPrefix() { + assertEquals( + "cvm.tencentcloudapi.com.cn", + EndpointFailoverInterceptor.substituteTld( + "cvm.tencentcloudapi.com", "tencentcloudapi.com", "tencentcloudapi.com.cn")); + assertEquals( + "cvm.ap-shanghai.tencentcloudapi.cn", + EndpointFailoverInterceptor.substituteTld( + "cvm.ap-shanghai.tencentcloudapi.com", "tencentcloudapi.com", "tencentcloudapi.cn")); + assertEquals( + "hunyuan.ai.ap-guangzhou.tencentcloudapi.com.cn", + EndpointFailoverInterceptor.substituteTld( + "hunyuan.ai.ap-guangzhou.tencentcloudapi.com", + "tencentcloudapi.com", + "tencentcloudapi.com.cn")); + } + + @Test + public void testSubstituteTldNoMatchReturnsInput() { + assertEquals( + "example.com", + EndpointFailoverInterceptor.substituteTld( + "example.com", "tencentcloudapi.com", "tencentcloudapi.cn")); + } + + // ---- Interceptor behavior tests ---- + + @Test + public void testPassThroughForUnknownHost() throws Exception { + TestClient client = newTC3Client(); + EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); + Request req = newTC3Request("example.com"); + RecordingChain chain = new RecordingChain(req); + chain.programSuccess(); + + Response resp = it.intercept(chain); + assertEquals(200, resp.code()); + assertEquals("example.com", chain.requests.get(0).url().host()); + } + + @Test + public void testPassThroughWhenSkipSign() throws Exception { + TestClient client = newTC3Client(); + EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); + Request req = new Request.Builder() + .url("https://cvm.tencentcloudapi.com/") + .post(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), "{}")) + .header("Host", "cvm.tencentcloudapi.com") + .header("Authorization", "SKIP") + .build(); + RecordingChain chain = new RecordingChain(req); + chain.programSuccess(); + + it.intercept(chain); + assertEquals(1, chain.requests.size()); + assertEquals("cvm.tencentcloudapi.com", chain.requests.get(0).url().host()); + } + + @Test + public void testPassThroughWhenDomainFailoverDisabled() throws Exception { + TestClient client = newTC3Client(); + client.getClientProfile().setDomainFailover(false); + EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); + Request req = newTC3Request("cvm.tencentcloudapi.com"); + RecordingChain chain = new RecordingChain(req); + chain.programDnsFailure(); + try { + it.intercept(chain); + fail("expected UnknownHostException"); + } catch (UnknownHostException expected) { + // no retry attempted when failover disabled + } + assertEquals(1, chain.requests.size()); + } + + @Test + public void testPassThroughForBackupTldUsedAsEndpoint() throws Exception { + TestClient client = newTC3Client(); + EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); + Request req = newTC3Request("cvm.tencentcloudapi.cn"); + RecordingChain chain = new RecordingChain(req); + chain.programSuccess(); + + Response resp = it.intercept(chain); + assertEquals(200, resp.code()); + assertEquals(1, chain.requests.size()); + assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(0).url().host()); + } + + @Test + public void testFailoverFromCnEndpoint() throws Exception { + TestClient client = newTC3Client(); + EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); + Request req = newTC3Request("cvm.tencentcloudapi.cn"); + + RecordingChain chain = new RecordingChain(req); + chain.programDnsFailure(); + chain.programSuccess(); + + Response resp = it.intercept(chain); + assertEquals(200, resp.code()); + assertEquals(2, chain.requests.size()); + assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(0).url().host()); + assertEquals("cvm.tencentcloudapi.com", chain.requests.get(1).url().host()); + assertEquals("cvm.tencentcloudapi.com", chain.requests.get(1).header("Host")); + } + + @Test + public void testFailoverToBackupTldOnDnsFailure() throws Exception { + TestClient client = newTC3Client(); + EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); + Request req = newTC3Request("cvm.tencentcloudapi.com"); + + RecordingChain chain = new RecordingChain(req); + chain.programDnsFailure(); + chain.programSuccess(); + + Response resp = it.intercept(chain); + assertEquals(200, resp.code()); + assertEquals(2, chain.requests.size()); + assertEquals("cvm.tencentcloudapi.com", chain.requests.get(0).url().host()); + assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(1).url().host()); + assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(1).header("Host")); + assertNotNull(chain.requests.get(1).header("Authorization")); + assertNotEquals( + chain.requests.get(0).header("Authorization"), + chain.requests.get(1).header("Authorization")); + } + + @Test + public void testAllBackupTldsFailThrowsUnknownHostException() throws Exception { + TestClient client = newTC3Client(); + EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); + Request req = newTC3Request("cvm.tencentcloudapi.com"); + + RecordingChain chain = new RecordingChain(req); + chain.programDnsFailure(); + chain.programDnsFailure(); + chain.programDnsFailure(); + + try { + it.intercept(chain); + fail("expected UnknownHostException"); + } catch (IOException e) { + assertTrue("Expected UnknownHostException, got " + e.getClass().getName(), + e instanceof UnknownHostException); + } + assertEquals(3, chain.requests.size()); + assertEquals("cvm.tencentcloudapi.com", chain.requests.get(0).url().host()); + assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(1).url().host()); + assertEquals("cvm.tencentcloudapi.com.cn", chain.requests.get(2).url().host()); + } + + @Test + public void testFollowupRequestUsesKnownWorkingTld() throws Exception { + TestClient client = newTC3Client(); + EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); + + { + Request req = newTC3Request("cvm.tencentcloudapi.com"); + RecordingChain chain = new RecordingChain(req); + chain.programDnsFailure(); + chain.programSuccess(); + Response resp = it.intercept(chain); + assertEquals(200, resp.code()); + } + + { + Request req = newTC3Request("cvm.tencentcloudapi.com"); + RecordingChain chain = new RecordingChain(req); + chain.programSuccess(); + Response resp = it.intercept(chain); + assertEquals(200, resp.code()); + assertEquals(1, chain.requests.size()); + assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(0).url().host()); + } + } + + @Test + public void testResignPicksUpRotatedCredential() throws Exception { + // Rotating the credential on the AbstractClient between initial sign and + // failover resign should be reflected in the new Authorization header, + // because the interceptor reads client.credential live. + TestClient client = newTC3Client(); + EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); + Request req = newTC3Request("cvm.tencentcloudapi.com"); + String origAuth = req.header("Authorization"); + + client.setCredential(new Credential("AKIDNEW", "SKNEW")); + + RecordingChain chain = new RecordingChain(req); + chain.programDnsFailure(); + chain.programSuccess(); + + it.intercept(chain); + String resignedAuth = chain.requests.get(1).header("Authorization"); + assertNotNull(resignedAuth); + assertNotEquals(origAuth, resignedAuth); + assertTrue("resigned auth should use rotated secretId, got: " + resignedAuth, + resignedAuth.contains("Credential=AKIDNEW/")); + } + + @Test + public void testResignHmacReplacesSignatureForNewHost() throws Exception { + TestClient client = newHmacClient(); + EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); + Request req = newHmacRequest("cvm.tencentcloudapi.com"); + String originalSig = req.url().queryParameter("Signature"); + assertNotNull(originalSig); + + RecordingChain chain = new RecordingChain(req); + chain.programDnsFailure(); + chain.programSuccess(); + + it.intercept(chain); + assertEquals(2, chain.requests.size()); + Request resigned = chain.requests.get(1); + assertEquals("cvm.tencentcloudapi.cn", resigned.url().host()); + String newSig = resigned.url().queryParameter("Signature"); + assertNotNull(newSig); + assertNotEquals(originalSig, newSig); + } + + // ---- Helpers ---- + + /** Minimal concrete AbstractClient subclass usable in tests. */ + private static final class TestClient extends AbstractClient { + TestClient(ClientProfile profile) { + super("cvm.tencentcloudapi.com", "2020-01-01", + new Credential("AKIDTEST", "SKTEST"), "ap-guangzhou", profile); + } + } + + private TestClient newTC3Client() { + ClientProfile profile = new ClientProfile(); + profile.getHttpProfile().setReqMethod(HttpProfile.REQ_POST); + profile.setDomainFailover(true); + return new TestClient(profile); + } + + private TestClient newHmacClient() { + ClientProfile profile = new ClientProfile(); + profile.setSignMethod(ClientProfile.SIGN_SHA256); + profile.getHttpProfile().setReqMethod(HttpProfile.REQ_GET); + profile.setDomainFailover(true); + return new TestClient(profile); + } + + /** + * Hand-crafted TC3-signed Request minimal enough for the interceptor to + * recognise and re-sign. Body + contentType + X-TC-* headers + Authorization + * are all the interceptor needs; the signature value itself is opaque to + * re-sign (it's recomputed from scratch for the backup host). + */ + private static Request newTC3Request(String host) { + byte[] body = "{}".getBytes(); + return new Request.Builder() + .url("https://" + host + "/") + .post(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), body)) + .header("Content-Type", "application/json; charset=utf-8") + .header("Host", host) + .header("Authorization", + "TC3-HMAC-SHA256 Credential=AKIDTEST/2024-01-01/cvm/tc3_request," + + " SignedHeaders=content-type;host, Signature=deadbeef") + .header("X-TC-Action", "TestAction") + .header("X-TC-Timestamp", "1700000000") + .header("X-TC-Version", "2020-01-01") + .header("X-TC-RequestClient", "SDK_JAVA_TEST") + .header("X-TC-Region", "ap-guangzhou") + .build(); + } + + /** Hand-crafted Hmac-signed GET Request with Signature in query string. */ + private static Request newHmacRequest(String host) { + return new Request.Builder() + .url("https://" + host + "/?Action=TestAction" + + "&Version=2020-01-01" + + "&Region=ap-guangzhou" + + "&SecretId=AKIDTEST" + + "&Timestamp=1700000000" + + "&Nonce=12345" + + "&SignatureMethod=HmacSHA256" + + "&Signature=deadbeefdeadbeef") + .get() + .build(); + } + + private static Response okResponse(Request req) { + return new Response.Builder() + .request(req) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body(ResponseBody.create(MediaType.parse("application/json"), "{}")) + .build(); + } + + private static final class RecordingChain implements Interceptor.Chain { + final List requests = new ArrayList(); + private final List programmed = new ArrayList(); + private int idx = 0; + private Request current; + + RecordingChain(Request initialRequest) { + this.current = initialRequest; + } + + void programSuccess() { + programmed.add(null); + } + + void programDnsFailure() { + programmed.add(new UnknownHostException("injected DNS failure")); + } + + @Override + public Request request() { + return current; + } + + @Override + public Response proceed(Request request) throws IOException { + current = request; + requests.add(request); + if (idx >= programmed.size()) { + throw new IllegalStateException("No more programmed responses"); + } + Object next = programmed.get(idx++); + if (next instanceof Throwable) { + if (next instanceof IOException) { + throw (IOException) next; + } + throw new RuntimeException((Throwable) next); + } + if (next == null) { + return okResponse(request); + } + return (Response) next; + } + + @Override public Connection connection() { return null; } + @Override public Call call() { return null; } + @Override public int connectTimeoutMillis() { return 0; } + @Override public Interceptor.Chain withConnectTimeout(int timeout, TimeUnit unit) { return this; } + @Override public int readTimeoutMillis() { return 0; } + @Override public Interceptor.Chain withReadTimeout(int timeout, TimeUnit unit) { return this; } + @Override public int writeTimeoutMillis() { return 0; } + @Override public Interceptor.Chain withWriteTimeout(int timeout, TimeUnit unit) { return this; } + } +} From 6ba162ca392474f1da04351c0b26eb22ccdf447f Mon Sep 17 00:00:00 2001 From: sesky4 Date: Mon, 27 Apr 2026 10:53:15 +0800 Subject: [PATCH 02/11] refactor: move domainFailover to HttpProfile, internalise interceptor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Failover toggle now lives on HttpProfile alongside other transport-layer options (apigwEndpoint, timeouts, proxy). EndpointFailoverInterceptor and its breaker timeout constant are package-private — users opt out solely via HttpProfile.setDomainFailover(false), not by touching the interceptor. --- .../common/AbstractClient.java | 2 +- .../common/EndpointFailoverInterceptor.java | 29 ++++++++---------- .../common/profile/ClientProfile.java | 30 ------------------- .../common/profile/HttpProfile.java | 25 ++++++++++++++++ .../EndpointFailoverInterceptorTest.java | 4 +-- 5 files changed, 39 insertions(+), 51 deletions(-) diff --git a/src/main/java/com/tencentcloudapi/common/AbstractClient.java b/src/main/java/com/tencentcloudapi/common/AbstractClient.java index d0434c81df..afa7d63f6e 100644 --- a/src/main/java/com/tencentcloudapi/common/AbstractClient.java +++ b/src/main/java/com/tencentcloudapi/common/AbstractClient.java @@ -134,7 +134,7 @@ public AbstractClient( this.profile.getHttpProfile().getWriteTimeout() ); this.httpConnection.addInterceptors(this.log); - if (this.profile.getDomainFailover()) { + if (this.profile.getHttpProfile().getDomainFailover()) { this.httpConnection.addInterceptors(new EndpointFailoverInterceptor(this)); } this.trySetProxy(this.httpConnection); diff --git a/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java b/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java index 9a6c99d296..5d8e187b12 100644 --- a/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java +++ b/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java @@ -66,8 +66,11 @@ * suppress repeated attempts against a failing TLD; after the configured * breaker timeout (default {@value #DEFAULT_BREAKER_TIMEOUT_MS} ms) the * breaker half-opens and the next request probes that TLD again. + *

+ * Internal: instantiated by {@link AbstractClient}; users opt out via + * {@link com.tencentcloudapi.common.profile.HttpProfile#setDomainFailover(boolean)}. */ -public class EndpointFailoverInterceptor implements Interceptor { +class EndpointFailoverInterceptor implements Interceptor { /** All known TLDs. The failover order rotates based on the user's original endpoint. */ static final String[] KNOWN_TLDS = new String[]{ @@ -79,10 +82,9 @@ public class EndpointFailoverInterceptor implements Interceptor { /** * CircuitBreaker timeout (ms) controlling how long an Open breaker stays * Open before transitioning to HalfOpen and probing the TLD again. - * Can be changed globally before any client is constructed. * Default: 60 s. */ - public static long DEFAULT_BREAKER_TIMEOUT_MS = 60 * 1000; + static final long DEFAULT_BREAKER_TIMEOUT_MS = 60 * 1000; private final AbstractClient client; private final long breakerTimeoutMs; @@ -95,19 +97,9 @@ public class EndpointFailoverInterceptor implements Interceptor { new ConcurrentHashMap(); /** Creates an interceptor bound to {@code client} with the default breaker timeout (60 s). */ - public EndpointFailoverInterceptor(AbstractClient client) { - this(client, DEFAULT_BREAKER_TIMEOUT_MS); - } - - /** - * Creates an interceptor with a custom breaker timeout. - * - * @param client the owning client; credential and profile are read live on each retry. - * @param breakerTimeoutMs ms to keep a breaker Open before probing its TLD again. - */ - public EndpointFailoverInterceptor(AbstractClient client, long breakerTimeoutMs) { + EndpointFailoverInterceptor(AbstractClient client) { this.client = client; - this.breakerTimeoutMs = breakerTimeoutMs; + this.breakerTimeoutMs = DEFAULT_BREAKER_TIMEOUT_MS; } /** Visible for testing. Resets all failover state. */ @@ -240,11 +232,14 @@ private int probeTldIdx() { */ private boolean eligibleForFailover(Request request) { ClientProfile profile = client.getClientProfile(); - if (profile == null || !profile.getDomainFailover()) { + if (profile == null) { return false; } HttpProfile httpProfile = profile.getHttpProfile(); - if (httpProfile != null && httpProfile.getApigwEndpoint() != null) { + if (httpProfile == null || !httpProfile.getDomainFailover()) { + return false; + } + if (httpProfile.getApigwEndpoint() != null) { return false; } String host = request.url().host(); diff --git a/src/main/java/com/tencentcloudapi/common/profile/ClientProfile.java b/src/main/java/com/tencentcloudapi/common/profile/ClientProfile.java index fad6025ebf..c26389200d 100644 --- a/src/main/java/com/tencentcloudapi/common/profile/ClientProfile.java +++ b/src/main/java/com/tencentcloudapi/common/profile/ClientProfile.java @@ -67,15 +67,6 @@ public class ClientProfile { // Backup endpoint for API requests, useful in case the primary endpoint fails. private String backupEndpoint; - /** - * Flag indicating whether domain-level failover is enabled. - * When true (default), the SDK will automatically switch to backup TLDs - * (e.g. tencentcloudapi.com.cn / tencentcloudapi.cn) on DNS / TLS / network - * reachability failures of the primary domain. Custom apigw endpoints and - * SkipSign requests are passed through unchanged. - */ - private boolean domainFailover = true; - /** * Constructor to initialize ClientProfile with a specific signing method and HTTP profile. * If the signing method is null or empty, it defaults to "TC3-HMAC-SHA256". @@ -220,25 +211,4 @@ public String getBackupEndpoint() { public void setBackupEndpoint(String backupEndpoint) { this.backupEndpoint = backupEndpoint; } - - /** - * Getter for the domain failover flag. - * - * @return true if domain failover is enabled (default), false otherwise. - */ - public boolean getDomainFailover() { - return this.domainFailover; - } - - /** - * Setter for the domain failover flag. Domain failover causes the SDK - * to automatically switch to backup TLDs on DNS / TLS / network - * reachability failures of the primary *.tencentcloudapi.com domain. - * Set to false to opt out. - * - * @param enabled true to enable (default), false to disable. - */ - public void setDomainFailover(boolean enabled) { - this.domainFailover = enabled; - } } diff --git a/src/main/java/com/tencentcloudapi/common/profile/HttpProfile.java b/src/main/java/com/tencentcloudapi/common/profile/HttpProfile.java index f480adc3c6..88a21cb737 100644 --- a/src/main/java/com/tencentcloudapi/common/profile/HttpProfile.java +++ b/src/main/java/com/tencentcloudapi/common/profile/HttpProfile.java @@ -107,6 +107,15 @@ public class HttpProfile { */ private Object httpClient; + /** + * Whether to enable TLD-level domain failover. When true (default), the SDK + * automatically retries against backup TLDs (e.g. tencentcloudapi.com.cn / + * tencentcloudapi.cn) on DNS / TLS / network reachability failures of the + * primary domain. Custom apigw endpoints and SkipSign requests are passed + * through unchanged. + */ + private boolean domainFailover = true; + /** * Default constructor for HttpProfile. * Initializes default values for the HTTP profile configuration. @@ -413,4 +422,20 @@ public Object getHttpClient() { public void setHttpClient(Object client) { httpClient = client; } + + /** + * @return true if TLD-level domain failover is enabled (default), false otherwise. + */ + public boolean getDomainFailover() { + return this.domainFailover; + } + + /** + * Enable or disable TLD-level domain failover. See {@link #domainFailover}. + * + * @param enabled true to enable (default), false to disable. + */ + public void setDomainFailover(boolean enabled) { + this.domainFailover = enabled; + } } diff --git a/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java b/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java index ce2aea1482..b751375ba0 100644 --- a/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java +++ b/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java @@ -129,7 +129,7 @@ public void testPassThroughWhenSkipSign() throws Exception { @Test public void testPassThroughWhenDomainFailoverDisabled() throws Exception { TestClient client = newTC3Client(); - client.getClientProfile().setDomainFailover(false); + client.getClientProfile().getHttpProfile().setDomainFailover(false); EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); Request req = newTC3Request("cvm.tencentcloudapi.com"); RecordingChain chain = new RecordingChain(req); @@ -304,7 +304,6 @@ private static final class TestClient extends AbstractClient { private TestClient newTC3Client() { ClientProfile profile = new ClientProfile(); profile.getHttpProfile().setReqMethod(HttpProfile.REQ_POST); - profile.setDomainFailover(true); return new TestClient(profile); } @@ -312,7 +311,6 @@ private TestClient newHmacClient() { ClientProfile profile = new ClientProfile(); profile.setSignMethod(ClientProfile.SIGN_SHA256); profile.getHttpProfile().setReqMethod(HttpProfile.REQ_GET); - profile.setDomainFailover(true); return new TestClient(profile); } From 8cee388c927998d3e78a9f9c7ce4c37908f419d0 Mon Sep 17 00:00:00 2001 From: sesky4 Date: Mon, 27 Apr 2026 14:48:30 +0800 Subject: [PATCH 03/11] fix: reprobe original TLD after failover cooldown Reprobe the user's original Tencent Cloud API domain after the breaker cooldown so traffic can automatically return once the primary TLD recovers. Add coverage for the cooldown-based reprobe path while keeping the existing preference for the last known working TLD before cooldown expires. --- .../common/EndpointFailoverInterceptor.java | 44 ++++++++++++++++--- .../EndpointFailoverInterceptorTest.java | 30 +++++++++++++ 2 files changed, 67 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java b/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java index 5d8e187b12..e19c17be66 100644 --- a/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java +++ b/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java @@ -146,7 +146,7 @@ private Failover(Request request) { this.state = getOrCreateState(originHost); } - /** Host candidates in preferred try order (last-known-working first). */ + /** Host candidates in preferred try order, with the original TLD reprobed once its cooldown expires. */ List candidates() { List hosts = new ArrayList(KNOWN_TLDS.length); for (int tldIdx : buildTryOrder(state, originIdx)) { @@ -177,12 +177,18 @@ Response attempt(String host, Chain chain) throws IOException { Response response = chain.proceed(attempt); token.report(true); state.currentIndex = tldIdx; + if (tldIdx == originIdx) { + state.clearOriginProbe(); + } return response; } catch (IOException e) { if (!shouldFailover(e)) { throw e; } token.report(false); + if (tldIdx == originIdx) { + state.scheduleOriginProbe(breakerTimeoutMs); + } lastFailure = e; return null; } @@ -260,17 +266,27 @@ private boolean eligibleForFailover(Request request) { } /** - * Order: last-known-working TLD first, then the rest in natural KNOWN_TLDS order. - * If no successful TLD has been recorded yet, start from the origin. + * Order: last-known-working TLD first, unless the original TLD is due for a + * recovery probe. Once that cooldown expires, the original TLD moves to the + * front so traffic can automatically return to the user's configured domain. */ private int[] buildTryOrder(FailoverState state, int originIdx) { - int startIdx = state.currentIndex >= 0 ? state.currentIndex : originIdx; + int preferredIdx = state.currentIndex >= 0 ? state.currentIndex : originIdx; int n = KNOWN_TLDS.length; int[] order = new int[n]; - order[0] = startIdx; - int pos = 1; + boolean[] added = new boolean[n]; + int pos = 0; + + if (originIdx != preferredIdx && state.shouldProbeOrigin()) { + order[pos++] = originIdx; + added[originIdx] = true; + } + if (!added[preferredIdx]) { + order[pos++] = preferredIdx; + added[preferredIdx] = true; + } for (int i = 0; i < n; i++) { - if (i != startIdx) { + if (!added[i]) { order[pos++] = i; } } @@ -624,6 +640,8 @@ static final class FailoverState { final CircuitBreaker[] breakers; /** Index in {@link #KNOWN_TLDS} of the last TLD that served a successful request; -1 until first success. */ volatile int currentIndex = -1; + /** Timestamp after which the original TLD should be reprobed; -1 when no reprobe is pending. */ + volatile long originProbeAfterMs = -1; FailoverState(long breakerTimeoutMs) { int n = KNOWN_TLDS.length; @@ -634,5 +652,17 @@ static final class FailoverState { this.breakers[i] = new CircuitBreaker(s); } } + + void scheduleOriginProbe(long delayMs) { + this.originProbeAfterMs = System.currentTimeMillis() + delayMs; + } + + void clearOriginProbe() { + this.originProbeAfterMs = -1; + } + + boolean shouldProbeOrigin() { + return this.originProbeAfterMs >= 0 && System.currentTimeMillis() >= this.originProbeAfterMs; + } } } diff --git a/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java b/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java index b751375ba0..5985541312 100644 --- a/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java +++ b/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java @@ -246,6 +246,36 @@ public void testFollowupRequestUsesKnownWorkingTld() throws Exception { } } + @Test + public void testFollowupRequestReprobesOriginalTldAfterCooldown() throws Exception { + TestClient client = newTC3Client(); + EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); + + { + Request req = newTC3Request("cvm.tencentcloudapi.com"); + RecordingChain chain = new RecordingChain(req); + chain.programDnsFailure(); + chain.programSuccess(); + Response resp = it.intercept(chain); + assertEquals(200, resp.code()); + } + + EndpointFailoverInterceptor.FailoverState state = + EndpointFailoverInterceptor.STATE.get("cvm.tencentcloudapi.com"); + assertNotNull(state); + state.originProbeAfterMs = 0; + + { + Request req = newTC3Request("cvm.tencentcloudapi.com"); + RecordingChain chain = new RecordingChain(req); + chain.programSuccess(); + Response resp = it.intercept(chain); + assertEquals(200, resp.code()); + assertEquals(1, chain.requests.size()); + assertEquals("cvm.tencentcloudapi.com", chain.requests.get(0).url().host()); + } + } + @Test public void testResignPicksUpRotatedCredential() throws Exception { // Rotating the credential on the AbstractClient between initial sign and From 0c2c309cbc73846783a486a298dbb88e112c6b9d Mon Sep 17 00:00:00 2001 From: sesky4 Date: Mon, 27 Apr 2026 16:23:49 +0800 Subject: [PATCH 04/11] fix: allow SKIP V3 requests to fail over Unsigned V3 endpoints still need TLD rewrite when primary domain fails, so treat `Authorization: SKIP` as rewrite-only instead of excluding it from failover eligibility. --- .../common/EndpointFailoverInterceptor.java | 74 ++++++++++--------- .../EndpointFailoverInterceptorTest.java | 57 ++++++++++---- 2 files changed, 83 insertions(+), 48 deletions(-) diff --git a/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java b/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java index e19c17be66..931134a7b2 100644 --- a/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java +++ b/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java @@ -228,41 +228,11 @@ private int probeTldIdx() { } /** - * A request is eligible for TLD failover when: - *

    - *
  • the owning client has domain failover enabled
    • - *
  • the profile has no apigw-endpoint override (we never rewrite custom proxies)
    • - *
  • the target host is a recognised Tencent Cloud API domain
    • - *
  • for TC3-signed requests, Authorization header is present and not "SKIP"
    • - *
+ * A request is eligible for TLD failover when the target host is a + * recognised Tencent Cloud API domain. */ private boolean eligibleForFailover(Request request) { - ClientProfile profile = client.getClientProfile(); - if (profile == null) { - return false; - } - HttpProfile httpProfile = profile.getHttpProfile(); - if (httpProfile == null || !httpProfile.getDomainFailover()) { - return false; - } - if (httpProfile.getApigwEndpoint() != null) { - return false; - } - String host = request.url().host(); - if (tldIndexOf(host) < 0) { - return false; - } - String signMethod = profile.getSignMethod(); - if (ClientProfile.SIGN_TC3_256.equals(signMethod)) { - String auth = request.header("Authorization"); - if (auth == null || "SKIP".equals(auth)) { - return false; - } - } else if (!ClientProfile.SIGN_SHA1.equals(signMethod) - && !ClientProfile.SIGN_SHA256.equals(signMethod)) { - return false; - } - return true; + return tldIndexOf(request.url().host()) >= 0; } /** @@ -320,6 +290,9 @@ private Request rewriteForTld(Request original, String originalHost, int originI } String signMethod = client.getClientProfile().getSignMethod(); + if (isSkipSignV3Request(original, signMethod)) { + return rewriteSkipSignV3(original, targetHost); + } if (ClientProfile.SIGN_TC3_256.equals(signMethod)) { return resignTC3(original, targetHost); } @@ -331,9 +304,42 @@ private Request rewriteForTld(Request original, String originalHost, int originI } // ======================================================================== - // TC3-HMAC-SHA256 resign + // TC3-HMAC-SHA256 resign / SKIP rewrite // ======================================================================== + private static boolean isSkipSignV3Request(Request original, String signMethod) { + return ClientProfile.SIGN_TC3_256.equals(signMethod) + && "SKIP".equals(original.header("Authorization")); + } + + private Request rewriteSkipSignV3(Request original, String targetHost) throws IOException { + String httpMethod = original.method(); + String contentType = original.header("Content-Type"); + byte[] payload = readRequestBody(original); + + Headers.Builder hb = new Headers.Builder(); + Headers origHeaders = original.headers(); + for (int i = 0; i < origHeaders.size(); i++) { + String name = origHeaders.name(i); + if (name.equalsIgnoreCase("Host")) { + continue; + } + hb.add(name, origHeaders.value(i)); + } + hb.add("Host", targetHost); + + HttpUrl newUrl = original.url().newBuilder().host(targetHost).build(); + Request.Builder rb = original.newBuilder() + .url(newUrl) + .headers(hb.build()); + if (HttpProfile.REQ_POST.equalsIgnoreCase(httpMethod)) { + rb.post(RequestBody.create(MediaType.parse(contentType), payload)); + } else if (HttpProfile.REQ_GET.equalsIgnoreCase(httpMethod)) { + rb.get(); + } + return rb.build(); + } + private Request resignTC3(Request original, String targetHost) throws TencentCloudSDKException, IOException { Credential credential = client.getCredential(); diff --git a/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java b/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java index 5985541312..83cf69c4f2 100644 --- a/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java +++ b/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java @@ -112,35 +112,49 @@ public void testPassThroughForUnknownHost() throws Exception { public void testPassThroughWhenSkipSign() throws Exception { TestClient client = newTC3Client(); EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); - Request req = new Request.Builder() - .url("https://cvm.tencentcloudapi.com/") - .post(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), "{}")) - .header("Host", "cvm.tencentcloudapi.com") - .header("Authorization", "SKIP") - .build(); + Request req = newSkipSignV3Request("cvm.tencentcloudapi.com"); RecordingChain chain = new RecordingChain(req); chain.programSuccess(); it.intercept(chain); assertEquals(1, chain.requests.size()); assertEquals("cvm.tencentcloudapi.com", chain.requests.get(0).url().host()); + assertEquals("SKIP", chain.requests.get(0).header("Authorization")); + } + + @Test + public void testFailoverRewritesSkipSignV3Request() throws Exception { + TestClient client = newTC3Client(); + EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); + Request req = newSkipSignV3Request("cvm.tencentcloudapi.com"); + RecordingChain chain = new RecordingChain(req); + chain.programDnsFailure(); + chain.programSuccess(); + + Response resp = it.intercept(chain); + assertEquals(200, resp.code()); + assertEquals(2, chain.requests.size()); + assertEquals("cvm.tencentcloudapi.com", chain.requests.get(0).url().host()); + assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(1).url().host()); + assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(1).header("Host")); + assertEquals("SKIP", chain.requests.get(1).header("Authorization")); } @Test - public void testPassThroughWhenDomainFailoverDisabled() throws Exception { + public void testKnownDomainStillFailsOverAfterRuntimeDisable() throws Exception { TestClient client = newTC3Client(); client.getClientProfile().getHttpProfile().setDomainFailover(false); EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); Request req = newTC3Request("cvm.tencentcloudapi.com"); RecordingChain chain = new RecordingChain(req); chain.programDnsFailure(); - try { - it.intercept(chain); - fail("expected UnknownHostException"); - } catch (UnknownHostException expected) { - // no retry attempted when failover disabled - } - assertEquals(1, chain.requests.size()); + chain.programSuccess(); + + Response resp = it.intercept(chain); + assertEquals(200, resp.code()); + assertEquals(2, chain.requests.size()); + assertEquals("cvm.tencentcloudapi.com", chain.requests.get(0).url().host()); + assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(1).url().host()); } @Test @@ -368,6 +382,21 @@ private static Request newTC3Request(String host) { .build(); } + private static Request newSkipSignV3Request(String host) { + return new Request.Builder() + .url("https://" + host + "/") + .post(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), "{}")) + .header("Content-Type", "application/json; charset=utf-8") + .header("Host", host) + .header("Authorization", "SKIP") + .header("X-TC-Action", "TestAction") + .header("X-TC-Timestamp", "1700000000") + .header("X-TC-Version", "2020-01-01") + .header("X-TC-RequestClient", "SDK_JAVA_TEST") + .header("X-TC-Region", "ap-guangzhou") + .build(); + } + /** Hand-crafted Hmac-signed GET Request with Signature in query string. */ private static Request newHmacRequest(String host) { return new Request.Builder() From 69c1bc92b615fffbadd93db2614d47b8811884a0 Mon Sep 17 00:00:00 2001 From: sesky4 Date: Mon, 27 Apr 2026 17:26:55 +0800 Subject: [PATCH 05/11] fix: drop regional labels on failover hosts --- .../common/EndpointFailoverInterceptor.java | 27 ++++++++++++++++++- .../EndpointFailoverInterceptorTest.java | 24 ++++++++++++++--- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java b/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java index 931134a7b2..2b19dcd5c6 100644 --- a/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java +++ b/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java @@ -630,7 +630,11 @@ static String substituteTld(String host, String fromTld, String toTld) { } String suffix = "." + fromTld; if (host.endsWith(suffix)) { - return host.substring(0, host.length() - fromTld.length()) + toTld; + String prefix = host.substring(0, host.length() - suffix.length()); + if (prefix.isEmpty()) { + return toTld; + } + return stripRegionalLabel(prefix) + "." + toTld; } if (host.equals(fromTld)) { return toTld; @@ -638,6 +642,27 @@ static String substituteTld(String host, String fromTld, String toTld) { return host; } + private static String stripRegionalLabel(String prefix) { + int lastDot = prefix.lastIndexOf('.'); + if (lastDot < 0) { + return prefix; + } + String lastLabel = prefix.substring(lastDot + 1); + if (!looksLikeRegionLabel(lastLabel)) { + return prefix; + } + return prefix.substring(0, lastDot); + } + + private static boolean looksLikeRegionLabel(String label) { + return label != null && (label.startsWith("ap-") + || label.startsWith("na-") + || label.startsWith("eu-") + || label.startsWith("sa-") + || label.startsWith("af-") + || label.startsWith("me-")); + } + /** * Per-originalHost failover state. Holds one {@link CircuitBreaker} per TLD * plus the last-known-working TLD index. diff --git a/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java b/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java index 83cf69c4f2..c9205920c3 100644 --- a/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java +++ b/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java @@ -68,17 +68,17 @@ public void testIsKnownTencentCloudHost() { } @Test - public void testSubstituteTldPreservesPrefix() { + public void testSubstituteTldDropsRegionalLabel() { assertEquals( "cvm.tencentcloudapi.com.cn", EndpointFailoverInterceptor.substituteTld( "cvm.tencentcloudapi.com", "tencentcloudapi.com", "tencentcloudapi.com.cn")); assertEquals( - "cvm.ap-shanghai.tencentcloudapi.cn", + "cvm.tencentcloudapi.cn", EndpointFailoverInterceptor.substituteTld( "cvm.ap-shanghai.tencentcloudapi.com", "tencentcloudapi.com", "tencentcloudapi.cn")); assertEquals( - "hunyuan.ai.ap-guangzhou.tencentcloudapi.com.cn", + "hunyuan.ai.tencentcloudapi.com.cn", EndpointFailoverInterceptor.substituteTld( "hunyuan.ai.ap-guangzhou.tencentcloudapi.com", "tencentcloudapi.com", @@ -211,6 +211,24 @@ public void testFailoverToBackupTldOnDnsFailure() throws Exception { chain.requests.get(1).header("Authorization")); } + @Test + public void testFailoverDropsRegionalLabelFromHost() throws Exception { + TestClient client = newTC3Client(); + EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); + Request req = newTC3Request("cvm.ap-guangzhou.tencentcloudapi.com"); + + RecordingChain chain = new RecordingChain(req); + chain.programDnsFailure(); + chain.programSuccess(); + + Response resp = it.intercept(chain); + assertEquals(200, resp.code()); + assertEquals(2, chain.requests.size()); + assertEquals("cvm.ap-guangzhou.tencentcloudapi.com", chain.requests.get(0).url().host()); + assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(1).url().host()); + assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(1).header("Host")); + } + @Test public void testAllBackupTldsFailThrowsUnknownHostException() throws Exception { TestClient client = newTC3Client(); From b026845876b555d3477d1468993ece10b5d721cb Mon Sep 17 00:00:00 2001 From: sesky4 Date: Tue, 28 Apr 2026 17:52:04 +0800 Subject: [PATCH 06/11] Update EndpointFailoverInterceptor.java --- .../common/EndpointFailoverInterceptor.java | 72 +++++++------------ 1 file changed, 27 insertions(+), 45 deletions(-) diff --git a/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java b/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java index 2b19dcd5c6..f8aa2fdb6a 100644 --- a/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java +++ b/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java @@ -19,36 +19,18 @@ import com.tencentcloudapi.common.exception.TencentCloudSDKException; import com.tencentcloudapi.common.profile.ClientProfile; import com.tencentcloudapi.common.profile.HttpProfile; -import okhttp3.Headers; -import okhttp3.HttpUrl; -import okhttp3.Interceptor; -import okhttp3.MediaType; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; +import okhttp3.*; import okio.Buffer; +import javax.net.ssl.SSLHandshakeException; +import javax.net.ssl.SSLPeerUnverifiedException; import java.io.IOException; import java.io.UnsupportedEncodingException; -import java.net.ConnectException; -import java.net.NoRouteToHostException; -import java.net.PortUnreachableException; -import java.net.SocketTimeoutException; -import java.net.URLDecoder; -import java.net.URLEncoder; -import java.net.UnknownHostException; +import java.net.*; import java.nio.charset.StandardCharsets; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.TimeZone; -import java.util.TreeMap; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; -import javax.net.ssl.SSLHandshakeException; -import javax.net.ssl.SSLPeerUnverifiedException; /** * OkHttp interceptor that implements TLD-level domain failover for Tencent @@ -72,7 +54,9 @@ */ class EndpointFailoverInterceptor implements Interceptor { - /** All known TLDs. The failover order rotates based on the user's original endpoint. */ + /** + * All known TLDs. The failover order rotates based on the user's original endpoint. + */ static final String[] KNOWN_TLDS = new String[]{ "tencentcloudapi.com", "tencentcloudapi.cn", @@ -96,13 +80,17 @@ class EndpointFailoverInterceptor implements Interceptor { static final ConcurrentHashMap STATE = new ConcurrentHashMap(); - /** Creates an interceptor bound to {@code client} with the default breaker timeout (60 s). */ + /** + * Creates an interceptor bound to {@code client} with the default breaker timeout (60 s). + */ EndpointFailoverInterceptor(AbstractClient client) { this.client = client; this.breakerTimeoutMs = DEFAULT_BREAKER_TIMEOUT_MS; } - /** Visible for testing. Resets all failover state. */ + /** + * Visible for testing. Resets all failover state. + */ static void resetStateForTesting() { STATE.clear(); } @@ -146,7 +134,9 @@ private Failover(Request request) { this.state = getOrCreateState(originHost); } - /** Host candidates in preferred try order, with the original TLD reprobed once its cooldown expires. */ + /** + * Host candidates in preferred try order, with the original TLD reprobed once its cooldown expires. + */ List candidates() { List hosts = new ArrayList(KNOWN_TLDS.length); for (int tldIdx : buildTryOrder(state, originIdx)) { @@ -294,19 +284,15 @@ private Request rewriteForTld(Request original, String originalHost, int originI return rewriteSkipSignV3(original, targetHost); } if (ClientProfile.SIGN_TC3_256.equals(signMethod)) { - return resignTC3(original, targetHost); + return resignV3(original, targetHost); } if (ClientProfile.SIGN_SHA1.equals(signMethod) || ClientProfile.SIGN_SHA256.equals(signMethod)) { - return resignHmac(original, targetHost); + return resignV1(original, targetHost); } throw new TencentCloudSDKException( "Signature method " + signMethod + " is invalid or not supported yet."); } - // ======================================================================== - // TC3-HMAC-SHA256 resign / SKIP rewrite - // ======================================================================== - private static boolean isSkipSignV3Request(Request original, String signMethod) { return ClientProfile.SIGN_TC3_256.equals(signMethod) && "SKIP".equals(original.header("Authorization")); @@ -340,7 +326,7 @@ private Request rewriteSkipSignV3(Request original, String targetHost) throws IO return rb.build(); } - private Request resignTC3(Request original, String targetHost) + private Request resignV3(Request original, String targetHost) throws TencentCloudSDKException, IOException { Credential credential = client.getCredential(); ClientProfile profile = client.getClientProfile(); @@ -481,11 +467,7 @@ private static byte[] readRequestBody(Request request) throws IOException { return buffer.readByteArray(); } - // ======================================================================== - // HmacSHA1 / HmacSHA256 resign - // ======================================================================== - - private Request resignHmac(Request original, String targetHost) + private Request resignV1(Request original, String targetHost) throws TencentCloudSDKException, IOException { Credential credential = client.getCredential(); ClientProfile profile = client.getClientProfile(); @@ -582,10 +564,6 @@ private static Map decodeFormParams(String body) throws TencentC return map; } - // ======================================================================== - // Host helpers - // ======================================================================== - /** * Failures we treat as "host unreachable / suspicious" and worth retrying * against another TLD: outright DNS misses, TLS verification failures @@ -669,9 +647,13 @@ private static boolean looksLikeRegionLabel(String label) { */ static final class FailoverState { final CircuitBreaker[] breakers; - /** Index in {@link #KNOWN_TLDS} of the last TLD that served a successful request; -1 until first success. */ + /** + * Index in {@link #KNOWN_TLDS} of the last TLD that served a successful request; -1 until first success. + */ volatile int currentIndex = -1; - /** Timestamp after which the original TLD should be reprobed; -1 when no reprobe is pending. */ + /** + * Timestamp after which the original TLD should be reprobed; -1 when no reprobe is pending. + */ volatile long originProbeAfterMs = -1; FailoverState(long breakerTimeoutMs) { From ae8cb523be5b04484b451c8dc06ff9e70f53b35b Mon Sep 17 00:00:00 2001 From: sesky4 Date: Tue, 28 Apr 2026 18:08:00 +0800 Subject: [PATCH 07/11] Update EndpointFailoverInterceptorTest.java --- .../EndpointFailoverInterceptorTest.java | 475 +++++++++++++++++- 1 file changed, 470 insertions(+), 5 deletions(-) diff --git a/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java b/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java index c9205920c3..99a5244539 100644 --- a/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java +++ b/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java @@ -22,6 +22,7 @@ import okhttp3.Connection; import okhttp3.Interceptor; import okhttp3.MediaType; +import okhttp3.OkHttpClient; import okhttp3.Protocol; import okhttp3.Request; import okhttp3.RequestBody; @@ -30,20 +31,37 @@ import org.junit.Before; import org.junit.Test; +import javax.net.ssl.SSLHandshakeException; +import javax.net.ssl.SSLPeerUnverifiedException; import java.io.IOException; +import java.net.ConnectException; +import java.net.NoRouteToHostException; +import java.net.SocketTimeoutException; import java.net.UnknownHostException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.LinkedList; import java.util.List; +import java.util.Queue; import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -/** Unit tests for {@link EndpointFailoverInterceptor}. Pure in-process tests; no network. */ +/** + * Tests for {@link EndpointFailoverInterceptor}. Two flavours, no network: + *
    + *
  • Hand-rolled {@link RecordingChain}: cheap unit tests over a fake Chain.
    • + *
  • {@link TransportStub} plumbed into a real {@link OkHttpClient}: end-to-end + * tests through the actual OkHttp interceptor pipeline. Programs DNS misses, + * TLS failures, timeouts and 5xx outcomes per attempt.
    • + *
+ */ public class EndpointFailoverInterceptorTest { @Before @@ -236,9 +254,9 @@ public void testAllBackupTldsFailThrowsUnknownHostException() throws Exception { Request req = newTC3Request("cvm.tencentcloudapi.com"); RecordingChain chain = new RecordingChain(req); - chain.programDnsFailure(); - chain.programDnsFailure(); - chain.programDnsFailure(); + chain.programDnsFailure("first dns miss"); + chain.programDnsFailure("second dns miss"); + chain.programDnsFailure("third dns miss"); try { it.intercept(chain); @@ -246,11 +264,15 @@ public void testAllBackupTldsFailThrowsUnknownHostException() throws Exception { } catch (IOException e) { assertTrue("Expected UnknownHostException, got " + e.getClass().getName(), e instanceof UnknownHostException); + // Last attempt's exception must surface verbatim — confirms the + // interceptor preserves root cause rather than wrapping/swallowing. + assertEquals("third dns miss", e.getMessage()); } assertEquals(3, chain.requests.size()); assertEquals("cvm.tencentcloudapi.com", chain.requests.get(0).url().host()); assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(1).url().host()); assertEquals("cvm.tencentcloudapi.com.cn", chain.requests.get(2).url().host()); + chain.assertAllProgrammedConsumed(); } @Test @@ -265,16 +287,24 @@ public void testFollowupRequestUsesKnownWorkingTld() throws Exception { chain.programSuccess(); Response resp = it.intercept(chain); assertEquals(200, resp.code()); + chain.assertAllProgrammedConsumed(); } { Request req = newTC3Request("cvm.tencentcloudapi.com"); RecordingChain chain = new RecordingChain(req); + // Two outcomes programmed: if the interceptor wrongly probes + // .com first, it would consume the failure and need the second + // success — leaving zero leftovers. With correct behavior only + // .cn is tried, leaving one outcome unconsumed (asserted below). + chain.programSuccess(); chain.programSuccess(); Response resp = it.intercept(chain); assertEquals(200, resp.code()); assertEquals(1, chain.requests.size()); assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(0).url().host()); + assertEquals("must take exactly one outcome (no .com probe)", + 1, chain.programmedRemaining()); } } @@ -290,6 +320,7 @@ public void testFollowupRequestReprobesOriginalTldAfterCooldown() throws Excepti chain.programSuccess(); Response resp = it.intercept(chain); assertEquals(200, resp.code()); + chain.assertAllProgrammedConsumed(); } EndpointFailoverInterceptor.FailoverState state = @@ -300,11 +331,20 @@ public void testFollowupRequestReprobesOriginalTldAfterCooldown() throws Excepti { Request req = newTC3Request("cvm.tencentcloudapi.com"); RecordingChain chain = new RecordingChain(req); + // Belt-and-braces: extra success queued so a wrong fallback path + // would still complete the test rather than blow up with + // "no programmed outcomes" — which would mask the real bug. + chain.programSuccess(); chain.programSuccess(); Response resp = it.intercept(chain); assertEquals(200, resp.code()); assertEquals(1, chain.requests.size()); assertEquals("cvm.tencentcloudapi.com", chain.requests.get(0).url().host()); + assertEquals("must take exactly one outcome (origin probe succeeded first)", + 1, chain.programmedRemaining()); + // After the successful origin probe the cooldown must clear, + // otherwise the next request would reprobe forever. + assertEquals(-1, state.originProbeAfterMs); } } @@ -455,7 +495,23 @@ void programSuccess() { } void programDnsFailure() { - programmed.add(new UnknownHostException("injected DNS failure")); + programDnsFailure("injected DNS failure"); + } + + void programDnsFailure(String message) { + programmed.add(new UnknownHostException(message)); + } + + int programmedRemaining() { + return programmed.size() - idx; + } + + void assertAllProgrammedConsumed() { + if (programmedRemaining() != 0) { + throw new AssertionError( + "Expected all programmed outcomes to be consumed but " + + programmedRemaining() + " remain"); + } } @Override @@ -492,4 +548,413 @@ public Response proceed(Request request) throws IOException { @Override public int writeTimeoutMillis() { return 0; } @Override public Interceptor.Chain withWriteTimeout(int timeout, TimeUnit unit) { return this; } } + + // ==================================================================== + // End-to-end tests: real OkHttpClient pipeline + TransportStub. + // Exercises the interceptor through OkHttp's actual chain, not a hand- + // rolled fake. Catches integration bugs RecordingChain can't. + // ==================================================================== + + // ---- shouldFailover branch coverage ---- + + @Test + public void e2eFailoverOnSslHandshakeException() throws Exception { + runE2EFailoverScenario(new SSLHandshakeException("tls handshake failed")); + } + + @Test + public void e2eFailoverOnSslPeerUnverifiedException() throws Exception { + runE2EFailoverScenario(new SSLPeerUnverifiedException("cert mismatch")); + } + + @Test + public void e2eFailoverOnConnectException() throws Exception { + runE2EFailoverScenario(new ConnectException("connection refused")); + } + + @Test + public void e2eFailoverOnNoRouteToHostException() throws Exception { + runE2EFailoverScenario(new NoRouteToHostException("no route")); + } + + @Test + public void e2eFailoverOnSocketTimeoutException() throws Exception { + runE2EFailoverScenario(new SocketTimeoutException("read timed out")); + } + + private void runE2EFailoverScenario(IOException firstFailure) throws Exception { + AbstractClient client = newTC3Client(); + TransportStub transport = new TransportStub(); + OkHttpClient http = newE2EClient(client, transport); + + transport.programFailure(firstFailure); + transport.programSuccess(200, "{\"Response\":{}}"); + + Request req = newTC3Request("cvm.tencentcloudapi.com"); + Response resp = http.newCall(req).execute(); + assertEquals(200, resp.code()); + assertEquals("{\"Response\":{}}", resp.body().string()); + + assertEquals(2, transport.received.size()); + assertEquals("cvm.tencentcloudapi.com", transport.received.get(0).url().host()); + assertEquals("cvm.tencentcloudapi.cn", transport.received.get(1).url().host()); + } + + // ---- Non-failover IOException must propagate without retry ---- + + @Test + public void e2eGenericIOExceptionPropagatesWithoutFailover() throws Exception { + AbstractClient client = newTC3Client(); + TransportStub transport = new TransportStub(); + OkHttpClient http = newE2EClient(client, transport); + + IOException unrelated = new IOException("some unrelated I/O error"); + transport.programFailure(unrelated); + + Request req = newTC3Request("cvm.tencentcloudapi.com"); + try { + http.newCall(req).execute(); + fail("expected IOException to propagate"); + } catch (IOException e) { + assertEquals("some unrelated I/O error", e.getMessage()); + } + assertEquals("must not retry on non-failover IOException", 1, transport.received.size()); + } + + // ---- Response body / status / headers reach caller intact ---- + + @Test + public void e2eResponseBodyAndStatusPreservedAfterFailover() throws Exception { + AbstractClient client = newTC3Client(); + TransportStub transport = new TransportStub(); + OkHttpClient http = newE2EClient(client, transport); + + transport.programFailure(new UnknownHostException("dns miss")); + transport.programSuccess(202, "{\"Response\":{\"X\":42}}"); + + Request req = newTC3Request("cvm.tencentcloudapi.com"); + Response resp = http.newCall(req).execute(); + assertEquals(202, resp.code()); + assertEquals("{\"Response\":{\"X\":42}}", resp.body().string()); + } + + // ---- 5xx server response is not a failover trigger ---- + + @Test + public void e2e5xxResponseDoesNotTriggerFailover() throws Exception { + AbstractClient client = newTC3Client(); + TransportStub transport = new TransportStub(); + OkHttpClient http = newE2EClient(client, transport); + + // Server reachable, returns 503 — interceptor must surface this, not retry. + transport.programSuccess(503, "service unavailable"); + + Request req = newTC3Request("cvm.tencentcloudapi.com"); + Response resp = http.newCall(req).execute(); + assertEquals(503, resp.code()); + assertEquals(1, transport.received.size()); + } + + // ---- TC3 resign preserves body, content-type, signing scope ---- + + @Test + public void e2eTC3ResignPreservesBodyAndContentType() throws Exception { + AbstractClient client = newTC3Client(); + TransportStub transport = new TransportStub(); + OkHttpClient http = newE2EClient(client, transport); + + transport.programFailure(new UnknownHostException("dns miss")); + transport.programSuccess(200, "{}"); + + String payload = "{\"Limit\":10,\"Offset\":0,\"Filters\":[\"a\",\"b\"]}"; + Request req = newTC3RequestWithBody("cvm.tencentcloudapi.com", payload); + http.newCall(req).execute(); + + Request resigned = transport.received.get(1); + assertEquals("cvm.tencentcloudapi.cn", resigned.url().host()); + assertEquals("application/json; charset=utf-8", resigned.header("Content-Type")); + // Authorization must be regenerated with the new host bound into the signed scope. + String origAuth = transport.received.get(0).header("Authorization"); + String newAuth = resigned.header("Authorization"); + assertNotNull(newAuth); + assertNotEquals(origAuth, newAuth); + assertTrue("resigned auth must be TC3", newAuth.startsWith("TC3-HMAC-SHA256 ")); + assertTrue("scope must include new service host segment", + newAuth.contains("/cvm/tc3_request")); + // Body must round-trip byte-identical through resign. + assertEquals(payload, bodyAsString(resigned)); + } + + // ---- X-TC-Token rotation visible to resigned request ---- + + @Test + public void e2eResignReflectsRotatedToken() throws Exception { + AbstractClient client = newTC3Client(); + client.setCredential(new Credential("AKIDTEST", "SKTEST", "tok-v1")); + TransportStub transport = new TransportStub(); + OkHttpClient http = newE2EClient(client, transport); + + transport.programFailure(new UnknownHostException("dns miss")); + transport.programSuccess(200, "{}"); + + // Build a TC3 request that already carries the v1 token. + Request req = newTC3Request("cvm.tencentcloudapi.com") + .newBuilder() + .header("X-TC-Token", "tok-v1") + .build(); + + // Rotate the token between original sign and resign. + client.setCredential(new Credential("AKIDTEST", "SKTEST", "tok-v2")); + + http.newCall(req).execute(); + assertEquals("tok-v2", transport.received.get(1).header("X-TC-Token")); + } + + @Test + public void e2eResignDropsTokenWhenCleared() throws Exception { + AbstractClient client = newTC3Client(); + client.setCredential(new Credential("AKIDTEST", "SKTEST", "tok-v1")); + TransportStub transport = new TransportStub(); + OkHttpClient http = newE2EClient(client, transport); + + transport.programFailure(new UnknownHostException("dns miss")); + transport.programSuccess(200, "{}"); + + Request req = newTC3Request("cvm.tencentcloudapi.com") + .newBuilder() + .header("X-TC-Token", "tok-v1") + .build(); + + client.setCredential(new Credential("AKIDTEST", "SKTEST")); // no token + + http.newCall(req).execute(); + assertNull("token must be removed from resigned request when credential drops it", + transport.received.get(1).header("X-TC-Token")); + } + + // ---- Hmac (V1) resign preserves all query params; Signature replaced exactly once ---- + + @Test + public void e2eHmacResignPreservesQueryParams() throws Exception { + AbstractClient client = newHmacClient(); + TransportStub transport = new TransportStub(); + OkHttpClient http = newE2EClient(client, transport); + + transport.programFailure(new UnknownHostException("dns miss")); + transport.programSuccess(200, "{}"); + + Request req = newHmacRequest("cvm.tencentcloudapi.com"); + http.newCall(req).execute(); + + Request resigned = transport.received.get(1); + assertEquals("cvm.tencentcloudapi.cn", resigned.url().host()); + assertEquals("TestAction", resigned.url().queryParameter("Action")); + assertEquals("2020-01-01", resigned.url().queryParameter("Version")); + assertEquals("ap-guangzhou", resigned.url().queryParameter("Region")); + assertEquals("AKIDTEST", resigned.url().queryParameter("SecretId")); + assertEquals("12345", resigned.url().queryParameter("Nonce")); + assertEquals("HmacSHA256", resigned.url().queryParameter("SignatureMethod")); + // Must still have a Signature, must differ from the original. + String newSig = resigned.url().queryParameter("Signature"); + assertNotNull(newSig); + assertNotEquals("deadbeefdeadbeef", newSig); + // Old "Signature=deadbeefdeadbeef" must NOT survive as a duplicate query + // param (resigner must drop it before signing, not append alongside). + List sigValues = resigned.url().queryParameterValues("Signature"); + assertEquals("must have exactly one Signature param", 1, sigValues.size()); + } + + // ---- giveUp probe path: every breaker open ---- + + @Test + public void e2eGiveUpProbesLastKnownGoodTldWhenAllBreakersOpen() throws Exception { + AbstractClient client = newTC3Client(); + TransportStub transport = new TransportStub(); + OkHttpClient http = newE2EClient(client, transport); + + // Step 1: prime currentIndex to .cn by failing .com once then succeeding on .cn. + transport.programFailure(new UnknownHostException("dns miss")); + transport.programSuccess(200, "{}"); + http.newCall(newTC3Request("cvm.tencentcloudapi.com")).execute(); + transport.received.clear(); + + // Step 2: force every breaker into Open state by directly opening them. + EndpointFailoverInterceptor.FailoverState state = + EndpointFailoverInterceptor.STATE.get("cvm.tencentcloudapi.com"); + assertNotNull(state); + for (CircuitBreaker breaker : state.breakers) { + // Trip past maxFailNum=5 with fresh tokens. + for (int i = 0; i < 6; i++) { + CircuitBreaker.Token t = breaker.allow(); + if (t.allowed) { + t.report(false); + } + } + } + + // Step 3: a fresh request should still get a chance to probe the + // last-known-good TLD (.cn) — lastFailure is null on this attempt. + transport.programSuccess(200, "{\"Response\":{}}"); + Response resp = http.newCall(newTC3Request("cvm.tencentcloudapi.com")).execute(); + assertEquals(200, resp.code()); + assertEquals("must probe last-known-good TLD when every breaker is open", + 1, transport.received.size()); + assertEquals("cvm.tencentcloudapi.cn", transport.received.get(0).url().host()); + } + + // ---- Sustained failure trips the breaker (subsequent attempts skip the host) ---- + + @Test + public void e2eRepeatedDnsFailureTripsBreakerOnOriginTld() throws Exception { + AbstractClient client = newTC3Client(); + TransportStub transport = new TransportStub(); + OkHttpClient http = newE2EClient(client, transport); + + // Drive 5 requests where .com always fails DNS and .cn always succeeds. + // The .cn success path does not touch the .com breaker, so .com + // accumulates 5 failures with 100% fail-rate (failures>=maxFailNum=5 + // && failPercentage>=0.75) and trips Open. Subsequent requests must + // skip .com on the first attempt. Force origin reprobe each loop so + // every iteration actually hits .com first (otherwise `.cn`-prefer + // ordering kicks in after the first success). + EndpointFailoverInterceptor.FailoverState state = null; + for (int i = 0; i < 5; i++) { + transport.programFailure(new UnknownHostException("dns miss")); + transport.programSuccess(200, "{}"); + http.newCall(newTC3Request("cvm.tencentcloudapi.com")).execute(); + if (state == null) { + state = EndpointFailoverInterceptor.STATE.get("cvm.tencentcloudapi.com"); + assertNotNull(state); + } + state.originProbeAfterMs = 0; + } + + // Next request: .com breaker is Open, interceptor must skip .com on + // the first attempt and go straight to .cn. + transport.received.clear(); + transport.programSuccess(200, "{}"); + Response resp = http.newCall(newTC3Request("cvm.tencentcloudapi.com")).execute(); + assertEquals(200, resp.code()); + assertEquals("breaker should short-circuit .com without sending it to transport", + 1, transport.received.size()); + assertEquals("cvm.tencentcloudapi.cn", transport.received.get(0).url().host()); + } + + // ---- Original signature/host must NOT be the one that hit the wire on resign ---- + + @Test + public void e2eOriginalRequestNotSentTwice() throws Exception { + AbstractClient client = newTC3Client(); + TransportStub transport = new TransportStub(); + OkHttpClient http = newE2EClient(client, transport); + + transport.programFailure(new UnknownHostException("dns miss")); + transport.programSuccess(200, "{}"); + + Request req = newTC3Request("cvm.tencentcloudapi.com"); + http.newCall(req).execute(); + + Request first = transport.received.get(0); + Request second = transport.received.get(1); + assertNotEquals("hosts must differ between attempts", + first.url().host(), second.url().host()); + assertNotEquals("Authorization must be resigned for new host", + first.header("Authorization"), second.header("Authorization")); + assertEquals("Host header must track the URL host on resign", + second.url().host(), second.header("Host")); + } + + // ---- E2E helpers ---- + + private static Request newTC3RequestWithBody(String host, String body) { + return new Request.Builder() + .url("https://" + host + "/") + .post(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), + body.getBytes(StandardCharsets.UTF_8))) + .header("Content-Type", "application/json; charset=utf-8") + .header("Host", host) + .header("Authorization", + "TC3-HMAC-SHA256 Credential=AKIDTEST/2024-01-01/cvm/tc3_request," + + " SignedHeaders=content-type;host, Signature=deadbeef") + .header("X-TC-Action", "TestAction") + .header("X-TC-Timestamp", "1700000000") + .header("X-TC-Version", "2020-01-01") + .header("X-TC-RequestClient", "SDK_JAVA_TEST") + .header("X-TC-Region", "ap-guangzhou") + .build(); + } + + /** + * Builds an {@link OkHttpClient} whose interceptor chain is: + * EndpointFailoverInterceptor → TransportStub. The transport stub plays + * the role of the network so each test runs offline yet exercises the + * real OkHttp pipeline (unlike {@link RecordingChain}). + */ + private static OkHttpClient newE2EClient(AbstractClient client, TransportStub transport) { + return new OkHttpClient.Builder() + .addInterceptor(new EndpointFailoverInterceptor(client)) + .addInterceptor(transport) + .build(); + } + + private static String bodyAsString(Request req) throws IOException { + if (req.body() == null) { + return ""; + } + okio.Buffer buf = new okio.Buffer(); + req.body().writeTo(buf); + return buf.readUtf8(); + } + + /** + * Terminal interceptor that replaces the network. Tests script a queue of + * {@link IOException} (failure) / {@link Response} (success) outcomes; each + * proceed call consumes one entry. Records every request that reaches it + * so tests can assert host / header / body content per attempt. + */ + private static final class TransportStub implements Interceptor { + final List received = new ArrayList(); + private final Queue programmed = new LinkedList(); + + void programFailure(IOException e) { + programmed.add(e); + } + + void programSuccess(int code, String body) { + programmed.add(new ProgrammedResponse(code, body)); + } + + @Override + public Response intercept(Chain chain) throws IOException { + Request request = chain.request(); + received.add(request); + Object next = programmed.poll(); + if (next == null) { + throw new IllegalStateException( + "TransportStub got an unexpected request to " + + request.url() + " — no programmed outcome left"); + } + if (next instanceof IOException) { + throw (IOException) next; + } + ProgrammedResponse pr = (ProgrammedResponse) next; + return new Response.Builder() + .request(request) + .protocol(Protocol.HTTP_1_1) + .code(pr.code) + .message(pr.code == 200 ? "OK" : "Error") + .body(ResponseBody.create(MediaType.parse("application/json"), pr.body)) + .build(); + } + + private static final class ProgrammedResponse { + final int code; + final String body; + + ProgrammedResponse(int code, String body) { + this.code = code; + this.body = body; + } + } + } } From a382322878eb40ed9f913d1df836a3686beb0d5a Mon Sep 17 00:00:00 2001 From: sesky4 Date: Tue, 28 Apr 2026 18:09:54 +0800 Subject: [PATCH 08/11] Update EndpointFailoverInterceptorTest.java --- .../EndpointFailoverInterceptorTest.java | 106 +++++++++++++++++- 1 file changed, 104 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java b/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java index 99a5244539..e23d1a25ce 100644 --- a/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java +++ b/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java @@ -352,7 +352,11 @@ public void testFollowupRequestReprobesOriginalTldAfterCooldown() throws Excepti public void testResignPicksUpRotatedCredential() throws Exception { // Rotating the credential on the AbstractClient between initial sign and // failover resign should be reflected in the new Authorization header, - // because the interceptor reads client.credential live. + // because the interceptor reads client.credential live. Verify the + // resigned signature is actually computed with the NEW SecretKey, not + // just the new SecretId — independently reproduce the TC3 signature + // using SKNEW + the timestamp echoed in the resigned headers, then + // assert byte-for-byte equality. TestClient client = newTC3Client(); EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); Request req = newTC3Request("cvm.tencentcloudapi.com"); @@ -365,11 +369,34 @@ public void testResignPicksUpRotatedCredential() throws Exception { chain.programSuccess(); it.intercept(chain); - String resignedAuth = chain.requests.get(1).header("Authorization"); + Request resigned = chain.requests.get(1); + String resignedAuth = resigned.header("Authorization"); assertNotNull(resignedAuth); assertNotEquals(origAuth, resignedAuth); assertTrue("resigned auth should use rotated secretId, got: " + resignedAuth, resignedAuth.contains("Credential=AKIDNEW/")); + + String resignedTimestamp = resigned.header("X-TC-Timestamp"); + assertNotNull(resignedTimestamp); + String expectedAuth = reproduceTc3SignaturePost( + "AKIDNEW", "SKNEW", + "cvm.tencentcloudapi.cn", "cvm", + "application/json; charset=utf-8", + "{}".getBytes(StandardCharsets.UTF_8), + Long.parseLong(resignedTimestamp)); + assertEquals("resigned signature must be computable with the rotated SK", + expectedAuth, resignedAuth); + + // Negative control: same inputs but with the OLD secret produce a + // different signature — proves the assertion above is meaningful and + // not a tautology. + String wrongAuth = reproduceTc3SignaturePost( + "AKIDTEST", "SKTEST", + "cvm.tencentcloudapi.cn", "cvm", + "application/json; charset=utf-8", + "{}".getBytes(StandardCharsets.UTF_8), + Long.parseLong(resignedTimestamp)); + assertNotEquals(wrongAuth, resignedAuth); } @Test @@ -391,10 +418,85 @@ public void testResignHmacReplacesSignatureForNewHost() throws Exception { String newSig = resigned.url().queryParameter("Signature"); assertNotNull(newSig); assertNotEquals(originalSig, newSig); + + // Stronger check: independently reproduce the expected V1 signature + // for the new host using the same SK and parameters, and compare. + // Sig changing alone is too weak — host change forces signature + // change for any half-correct implementation. This catches subtle + // bugs like signing with the wrong host, double-signing the old + // Signature param, or losing query params during resign. + java.util.TreeMap params = new java.util.TreeMap(); + params.put("Action", "TestAction"); + params.put("Version", "2020-01-01"); + params.put("Region", "ap-guangzhou"); + params.put("SecretId", "AKIDTEST"); + params.put("Timestamp", "1700000000"); + params.put("Nonce", "12345"); + params.put("SignatureMethod", "HmacSHA256"); + String plain = Sign.makeSignPlainText( + params, HttpProfile.REQ_GET, "cvm.tencentcloudapi.cn", "/"); + String expectedSig = Sign.sign("SKTEST", plain, ClientProfile.SIGN_SHA256); + assertEquals(expectedSig, newSig); + + // Negative control: signing for the ORIGINAL host produces a different + // signature, proving the resign actually rebound to the new host. + String wrongHostPlain = Sign.makeSignPlainText( + params, HttpProfile.REQ_GET, "cvm.tencentcloudapi.com", "/"); + String wrongHostSig = Sign.sign("SKTEST", wrongHostPlain, ClientProfile.SIGN_SHA256); + assertNotEquals(wrongHostSig, newSig); + + // Confirm every original query param is preserved (none lost on resign). + assertEquals("TestAction", resigned.url().queryParameter("Action")); + assertEquals("2020-01-01", resigned.url().queryParameter("Version")); + assertEquals("ap-guangzhou", resigned.url().queryParameter("Region")); + assertEquals("AKIDTEST", resigned.url().queryParameter("SecretId")); + assertEquals("1700000000", resigned.url().queryParameter("Timestamp")); + assertEquals("12345", resigned.url().queryParameter("Nonce")); + assertEquals("HmacSHA256", resigned.url().queryParameter("SignatureMethod")); + // Old Signature must be replaced, not duplicated. + assertEquals("Signature must appear exactly once", + 1, resigned.url().queryParameterValues("Signature").size()); } // ---- Helpers ---- + /** + * Independently reproduces a TC3-HMAC-SHA256 Authorization header for a + * POST request with content-type;host as the signed headers. Used by + * {@link #testResignPicksUpRotatedCredential} to verify the interceptor's + * resigned signature byte-for-byte. Mirrors {@code resignV3} in the + * interceptor; if either drifts, this assertion catches it. + */ + private static String reproduceTc3SignaturePost(String secretId, String secretKey, + String host, String service, + String contentType, byte[] payload, + long timestampSec) throws Exception { + String canonicalHeaders = "content-type:" + contentType + "\nhost:" + host + "\n"; + String signedHeaders = "content-type;host"; + String hashedPayload = Sign.sha256Hex(payload); + String canonicalRequest = "POST\n/\n\n" + canonicalHeaders + "\n" + + signedHeaders + "\n" + hashedPayload; + + java.text.SimpleDateFormat sdf = new java.text.SimpleDateFormat("yyyy-MM-dd"); + sdf.setTimeZone(java.util.TimeZone.getTimeZone("UTC")); + String date = sdf.format(new java.util.Date(timestampSec * 1000L)); + String credentialScope = date + "/" + service + "/tc3_request"; + String hashedCanonical = Sign.sha256Hex(canonicalRequest.getBytes(StandardCharsets.UTF_8)); + String stringToSign = "TC3-HMAC-SHA256\n" + timestampSec + "\n" + + credentialScope + "\n" + hashedCanonical; + + byte[] secretDate = Sign.hmac256(("TC3" + secretKey).getBytes(StandardCharsets.UTF_8), date); + byte[] secretService = Sign.hmac256(secretDate, service); + byte[] secretSigning = Sign.hmac256(secretService, "tc3_request"); + String signature = DatatypeConverter + .printHexBinary(Sign.hmac256(secretSigning, stringToSign)) + .toLowerCase(); + return "TC3-HMAC-SHA256 " + + "Credential=" + secretId + "/" + credentialScope + ", " + + "SignedHeaders=" + signedHeaders + ", " + + "Signature=" + signature; + } + /** Minimal concrete AbstractClient subclass usable in tests. */ private static final class TestClient extends AbstractClient { TestClient(ClientProfile profile) { From d4838aa1e1ed2a58819f54ac32872037161ea011 Mon Sep 17 00:00:00 2001 From: sesky4 Date: Tue, 28 Apr 2026 18:17:37 +0800 Subject: [PATCH 09/11] dev --- .../common/EndpointFailoverInterceptor.java | 48 ++-- .../EndpointFailoverInterceptorTest.java | 270 ++++++++++++++++-- 2 files changed, 271 insertions(+), 47 deletions(-) diff --git a/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java b/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java index f8aa2fdb6a..fa4b8c423b 100644 --- a/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java +++ b/src/main/java/com/tencentcloudapi/common/EndpointFailoverInterceptor.java @@ -112,7 +112,7 @@ public Response intercept(Chain chain) throws IOException { } } - return failover.giveUp(chain); + throw failover.exhausted(); } /** @@ -125,7 +125,13 @@ private final class Failover { private final String originHost; private final int originIdx; private final FailoverState state; - private IOException lastFailure; + /** + * One entry per attempted (or breaker-skipped) candidate, in attempt + * order. Used by {@link #exhausted()} to surface every TLD's failure + * via Throwable.addSuppressed, so the caller's log shows all 3 root + * causes instead of just the last one. + */ + private final List attemptFailures = new ArrayList(); private Failover(Request request) { this.request = request; @@ -155,6 +161,8 @@ Response attempt(String host, Chain chain) throws IOException { int tldIdx = tldIndexOf(host); CircuitBreaker.Token token = state.breakers[tldIdx].allow(); if (!token.allowed) { + attemptFailures.add(new IOException( + "skipped " + host + ": circuit breaker open")); return null; } Request attempt; @@ -179,27 +187,31 @@ Response attempt(String host, Chain chain) throws IOException { if (tldIdx == originIdx) { state.scheduleOriginProbe(breakerTimeoutMs); } - lastFailure = e; + attemptFailures.add(new IOException( + "attempt against " + host + " failed: " + e.getClass().getSimpleName() + + ": " + e.getMessage(), e)); return null; } } /** - * All breakers Open. Probe the last-known-working TLD once so traffic - * can recover once the network heals; otherwise surface the last failure. + * Every candidate either failed or was short-circuited. Returns the + * last attempt's exception as the primary cause and attaches every + * other attempt's exception as a suppressed throwable, so all three + * TLDs' root causes are visible in a single stack trace dump. */ - Response giveUp(Chain chain) throws IOException { - if (lastFailure == null) { - String probeHost = hostForTld(probeTldIdx()); - Request attempt; - try { - attempt = rewriteFor(probeHost); - } catch (TencentCloudSDKException e) { - throw new IOException("Failed to re-sign request for failover: " + e.getMessage(), e); - } - return chain.proceed(attempt); + IOException exhausted() { + if (attemptFailures.isEmpty()) { + // Defensive: candidates() never returns empty for a known TLD, + // but guard against future refactors leaving this branch live. + return new IOException("Endpoint failover produced no attempts for " + originHost); } - throw lastFailure; + int last = attemptFailures.size() - 1; + IOException primary = attemptFailures.get(last); + for (int i = 0; i < last; i++) { + primary.addSuppressed(attemptFailures.get(i)); + } + return primary; } private Request rewriteFor(String host) throws TencentCloudSDKException, IOException { @@ -211,10 +223,6 @@ private String hostForTld(int tldIdx) { ? originHost : substituteTld(originHost, KNOWN_TLDS[originIdx], KNOWN_TLDS[tldIdx]); } - - private int probeTldIdx() { - return state.currentIndex >= 0 ? state.currentIndex : originIdx; - } } /** diff --git a/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java b/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java index e23d1a25ce..30ea4bdb51 100644 --- a/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java +++ b/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java @@ -248,7 +248,7 @@ public void testFailoverDropsRegionalLabelFromHost() throws Exception { } @Test - public void testAllBackupTldsFailThrowsUnknownHostException() throws Exception { + public void testAllBackupTldsFailAggregatesEveryAttemptFailure() throws Exception { TestClient client = newTC3Client(); EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); Request req = newTC3Request("cvm.tencentcloudapi.com"); @@ -258,16 +258,35 @@ public void testAllBackupTldsFailThrowsUnknownHostException() throws Exception { chain.programDnsFailure("second dns miss"); chain.programDnsFailure("third dns miss"); + IOException thrown = null; try { it.intercept(chain); - fail("expected UnknownHostException"); + fail("expected IOException"); } catch (IOException e) { - assertTrue("Expected UnknownHostException, got " + e.getClass().getName(), - e instanceof UnknownHostException); - // Last attempt's exception must surface verbatim — confirms the - // interceptor preserves root cause rather than wrapping/swallowing. - assertEquals("third dns miss", e.getMessage()); + thrown = e; } + // Primary cause = last attempt. Wrapper IOException carries the + // original UnknownHostException as its cause so the root type is + // recoverable for callers that want to switch on it. + assertTrue("primary message should mention last host, got: " + thrown.getMessage(), + thrown.getMessage().contains("cvm.tencentcloudapi.com.cn")); + assertTrue("primary message should mention last attempt's failure, got: " + thrown.getMessage(), + thrown.getMessage().contains("third dns miss")); + assertNotNull(thrown.getCause()); + assertTrue(thrown.getCause() instanceof UnknownHostException); + assertEquals("third dns miss", thrown.getCause().getMessage()); + + // Every other attempt is attached as a suppressed exception so a + // single stack-trace dump exposes all 3 root causes. + Throwable[] suppressed = thrown.getSuppressed(); + assertEquals(2, suppressed.length); + assertTrue(suppressed[0].getMessage().contains("cvm.tencentcloudapi.com")); + assertTrue(suppressed[0].getMessage().contains("first dns miss")); + assertTrue(suppressed[0].getCause() instanceof UnknownHostException); + assertTrue(suppressed[1].getMessage().contains("cvm.tencentcloudapi.cn")); + assertTrue(suppressed[1].getMessage().contains("second dns miss")); + assertTrue(suppressed[1].getCause() instanceof UnknownHostException); + assertEquals(3, chain.requests.size()); assertEquals("cvm.tencentcloudapi.com", chain.requests.get(0).url().host()); assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(1).url().host()); @@ -275,6 +294,196 @@ public void testAllBackupTldsFailThrowsUnknownHostException() throws Exception { chain.assertAllProgrammedConsumed(); } + @Test + public void testAggregatedFailurePreservesPerAttemptCauseTypes() throws Exception { + // Different IOException subtypes per attempt — each must round-trip + // intact through the aggregation so callers can switch on root type + // (e.g. distinguish TLS vs DNS vs connect failure for diagnostics). + TestClient client = newTC3Client(); + EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); + Request req = newTC3Request("cvm.tencentcloudapi.com"); + + RecordingChain chain = new RecordingChain(req); + chain.programFailure(new UnknownHostException("dns miss .com")); + chain.programFailure(new SSLHandshakeException("tls fail .cn")); + chain.programFailure(new ConnectException("connect fail .com.cn")); + + IOException thrown = null; + try { + it.intercept(chain); + fail("expected IOException"); + } catch (IOException e) { + thrown = e; + } + // Primary = last attempt's wrapper; cause = ConnectException. + assertTrue(thrown.getCause() instanceof ConnectException); + assertEquals("connect fail .com.cn", thrown.getCause().getMessage()); + + Throwable[] suppressed = thrown.getSuppressed(); + assertEquals(2, suppressed.length); + assertTrue(suppressed[0].getCause() instanceof UnknownHostException); + assertEquals("dns miss .com", suppressed[0].getCause().getMessage()); + assertTrue(suppressed[1].getCause() instanceof SSLHandshakeException); + assertEquals("tls fail .cn", suppressed[1].getCause().getMessage()); + chain.assertAllProgrammedConsumed(); + } + + @Test + public void testAggregatedFailureMixesBreakerSkipsWithRealFailures() throws Exception { + // Pre-open the .com breaker so the first candidate is short-circuited + // (no real attempt, placeholder IOException with no cause). The next + // two TLDs hit transport and fail. Aggregation must include all three + // entries in attempt order. + TestClient client = newTC3Client(); + EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); + + // Open the .com breaker via a 5-failure burst on a throwaway chain. + for (int i = 0; i < 5; i++) { + RecordingChain warmup = new RecordingChain(newTC3Request("cvm.tencentcloudapi.com")); + warmup.programDnsFailure("warmup fail " + i); + warmup.programSuccess(); // .cn succeeds, doesn't touch .com breaker + it.intercept(warmup); + EndpointFailoverInterceptor.FailoverState state = + EndpointFailoverInterceptor.STATE.get("cvm.tencentcloudapi.com"); + state.originProbeAfterMs = 0; // force .com to be retried each loop + } + + // .com breaker should now be Open. + EndpointFailoverInterceptor.FailoverState state = + EndpointFailoverInterceptor.STATE.get("cvm.tencentcloudapi.com"); + // Sanity: confirm Open. + assertFalse("expected .com breaker Open after 5 failures", + state.breakers[0].allow().allowed); + + // Real run: .com short-circuits, .cn fails, .com.cn fails. + // candidates() order with currentIndex=.cn and originProbeAfterMs=0: + // first .com (origin reprobe due), then .cn (preferred), then .com.cn. + state.originProbeAfterMs = 0; + Request req = newTC3Request("cvm.tencentcloudapi.com"); + RecordingChain chain = new RecordingChain(req); + chain.programFailure(new SSLHandshakeException("cn tls fail")); + chain.programFailure(new ConnectException("com.cn connect fail")); + + IOException thrown = null; + try { + it.intercept(chain); + fail("expected IOException"); + } catch (IOException e) { + thrown = e; + } + + // Only 2 chain.proceed calls — .com was skipped by breaker. + assertEquals(2, chain.requests.size()); + assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(0).url().host()); + assertEquals("cvm.tencentcloudapi.com.cn", chain.requests.get(1).url().host()); + + // Primary = last attempt (com.cn ConnectException). + assertTrue(thrown.getCause() instanceof ConnectException); + + // Suppressed: [.com breaker skip placeholder, .cn SSLHandshake wrapper]. + Throwable[] suppressed = thrown.getSuppressed(); + assertEquals(2, suppressed.length); + // Order matters — must reflect attempt order, not failure-type grouping. + assertTrue("first suppressed must be .com breaker skip, got: " + suppressed[0].getMessage(), + suppressed[0].getMessage().contains("cvm.tencentcloudapi.com") + && suppressed[0].getMessage().contains("circuit breaker open")); + assertNull("breaker-skip placeholder has no underlying cause", + suppressed[0].getCause()); + assertTrue(suppressed[1].getCause() instanceof SSLHandshakeException); + } + + @Test + public void testAggregatedFailureWhenPrimaryIsBreakerSkip() throws Exception { + // Pre-fail .cn and .com.cn breakers so they're Open while .com is still + // Closed. Then drive a request where .com fails (the only attempt + // that reaches transport). Both breaker-skips arrive AFTER .com's + // failure in attempt order, so the *primary* (last entry) is a + // breaker-skip placeholder with no cause. + TestClient client = newTC3Client(); + EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); + + EndpointFailoverInterceptor.FailoverState state = + new EndpointFailoverInterceptor.FailoverState(60_000); + EndpointFailoverInterceptor.STATE.put("cvm.tencentcloudapi.com", state); + // Open .cn (idx=1) and .com.cn (idx=2); leave .com (idx=0) Closed. + for (int idx : new int[]{1, 2}) { + for (int i = 0; i < 6; i++) { + CircuitBreaker.Token t = state.breakers[idx].allow(); + if (t.allowed) { + t.report(false); + } + } + } + + Request req = newTC3Request("cvm.tencentcloudapi.com"); + RecordingChain chain = new RecordingChain(req); + chain.programFailure(new UnknownHostException("com dns fail")); + + IOException thrown = null; + try { + it.intercept(chain); + fail("expected IOException"); + } catch (IOException e) { + thrown = e; + } + // Only .com hit transport. + assertEquals(1, chain.requests.size()); + assertEquals("cvm.tencentcloudapi.com", chain.requests.get(0).url().host()); + + // Primary = last attempt (.com.cn breaker skip), no cause. + assertNull("breaker-skip primary has no cause", thrown.getCause()); + assertTrue(thrown.getMessage().contains("cvm.tencentcloudapi.com.cn")); + assertTrue(thrown.getMessage().contains("circuit breaker open")); + + // Suppressed: [.com real failure, .cn breaker skip]. + Throwable[] suppressed = thrown.getSuppressed(); + assertEquals(2, suppressed.length); + assertTrue(suppressed[0].getCause() instanceof UnknownHostException); + assertEquals("com dns fail", suppressed[0].getCause().getMessage()); + assertNull(suppressed[1].getCause()); + assertTrue(suppressed[1].getMessage().contains("cvm.tencentcloudapi.cn")); + } + + @Test + public void testFailoverDoesNotPollutateNextRequestAttemptFailures() throws Exception { + // attemptFailures lives on Failover (per-intercept), not on the + // interceptor — guard against accidental promotion to instance state + // that would carry stale failures into the next request. + TestClient client = newTC3Client(); + EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); + + // Run 1: 1 failure + 1 success → no exception, no aggregation. + { + RecordingChain chain = new RecordingChain(newTC3Request("cvm.tencentcloudapi.com")); + chain.programDnsFailure("run1 fail"); + chain.programSuccess(); + Response resp = it.intercept(chain); + assertEquals(200, resp.code()); + } + + // Run 2: all-fail. Suppressed must contain ONLY run-2 failures, not + // any leftover from run 1. + { + RecordingChain chain = new RecordingChain(newTC3Request("cvm.tencentcloudapi.com")); + chain.programDnsFailure("run2 com fail"); + chain.programDnsFailure("run2 cn fail"); + chain.programDnsFailure("run2 com.cn fail"); + IOException thrown = null; + try { + it.intercept(chain); + fail("expected IOException"); + } catch (IOException e) { + thrown = e; + } + assertEquals(2, thrown.getSuppressed().length); + for (Throwable s : thrown.getSuppressed()) { + assertFalse("must not leak run-1 failure into run-2 aggregation: " + s.getMessage(), + s.getMessage().contains("run1")); + } + assertTrue(thrown.getMessage().contains("run2")); + } + } + @Test public void testFollowupRequestUsesKnownWorkingTld() throws Exception { TestClient client = newTC3Client(); @@ -604,6 +813,10 @@ void programDnsFailure(String message) { programmed.add(new UnknownHostException(message)); } + void programFailure(IOException e) { + programmed.add(e); + } + int programmedRemaining() { return programmed.size() - idx; } @@ -866,26 +1079,19 @@ public void e2eHmacResignPreservesQueryParams() throws Exception { assertEquals("must have exactly one Signature param", 1, sigValues.size()); } - // ---- giveUp probe path: every breaker open ---- + // ---- All breakers Open: surface aggregated IOException, do not probe ---- @Test - public void e2eGiveUpProbesLastKnownGoodTldWhenAllBreakersOpen() throws Exception { + public void e2eAllBreakersOpenThrowsAggregatedWithoutProbing() throws Exception { AbstractClient client = newTC3Client(); TransportStub transport = new TransportStub(); OkHttpClient http = newE2EClient(client, transport); - // Step 1: prime currentIndex to .cn by failing .com once then succeeding on .cn. - transport.programFailure(new UnknownHostException("dns miss")); - transport.programSuccess(200, "{}"); - http.newCall(newTC3Request("cvm.tencentcloudapi.com")).execute(); - transport.received.clear(); - - // Step 2: force every breaker into Open state by directly opening them. + // Force every breaker into Open state directly. EndpointFailoverInterceptor.FailoverState state = - EndpointFailoverInterceptor.STATE.get("cvm.tencentcloudapi.com"); - assertNotNull(state); + new EndpointFailoverInterceptor.FailoverState(60_000); + EndpointFailoverInterceptor.STATE.put("cvm.tencentcloudapi.com", state); for (CircuitBreaker breaker : state.breakers) { - // Trip past maxFailNum=5 with fresh tokens. for (int i = 0; i < 6; i++) { CircuitBreaker.Token t = breaker.allow(); if (t.allowed) { @@ -894,14 +1100,24 @@ public void e2eGiveUpProbesLastKnownGoodTldWhenAllBreakersOpen() throws Exceptio } } - // Step 3: a fresh request should still get a chance to probe the - // last-known-good TLD (.cn) — lastFailure is null on this attempt. - transport.programSuccess(200, "{\"Response\":{}}"); - Response resp = http.newCall(newTC3Request("cvm.tencentcloudapi.com")).execute(); - assertEquals(200, resp.code()); - assertEquals("must probe last-known-good TLD when every breaker is open", - 1, transport.received.size()); - assertEquals("cvm.tencentcloudapi.cn", transport.received.get(0).url().host()); + IOException thrown = null; + try { + http.newCall(newTC3Request("cvm.tencentcloudapi.com")).execute(); + fail("expected IOException when every breaker is open"); + } catch (IOException e) { + thrown = e; + } + // Primary message names the last skipped host. + assertTrue("primary message must mention breaker skip, got: " + thrown.getMessage(), + thrown.getMessage().contains("circuit breaker open")); + // Two suppressed entries — one per other TLD. + assertEquals(2, thrown.getSuppressed().length); + for (Throwable s : thrown.getSuppressed()) { + assertTrue("suppressed must mention breaker skip, got: " + s.getMessage(), + s.getMessage().contains("circuit breaker open")); + } + assertEquals("must not send any request when every breaker is open", + 0, transport.received.size()); } // ---- Sustained failure trips the breaker (subsequent attempts skip the host) ---- From 5d4b613a9a296dd252b06ab1a0cd22dd1c8bd96d Mon Sep 17 00:00:00 2001 From: sesky4 Date: Tue, 28 Apr 2026 18:26:00 +0800 Subject: [PATCH 10/11] Update EndpointFailoverInterceptorTest.java --- .../EndpointFailoverInterceptorTest.java | 1525 ++++++----------- 1 file changed, 551 insertions(+), 974 deletions(-) diff --git a/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java b/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java index 30ea4bdb51..31eef4e23d 100644 --- a/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java +++ b/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java @@ -16,16 +16,18 @@ */ package com.tencentcloudapi.common; +import com.tencentcloudapi.common.exception.TencentCloudSDKException; +import com.tencentcloudapi.common.http.HttpConnection; import com.tencentcloudapi.common.profile.ClientProfile; import com.tencentcloudapi.common.profile.HttpProfile; -import okhttp3.Call; -import okhttp3.Connection; +import com.tencentcloudapi.cvm.v20170312.CvmClient; +import com.tencentcloudapi.cvm.v20170312.models.DescribeInstancesRequest; +import com.tencentcloudapi.cvm.v20170312.models.DescribeInstancesResponse; import okhttp3.Interceptor; import okhttp3.MediaType; import okhttp3.OkHttpClient; import okhttp3.Protocol; import okhttp3.Request; -import okhttp3.RequestBody; import okhttp3.Response; import okhttp3.ResponseBody; import org.junit.Before; @@ -34,17 +36,17 @@ import javax.net.ssl.SSLHandshakeException; import javax.net.ssl.SSLPeerUnverifiedException; import java.io.IOException; +import java.lang.reflect.Field; import java.net.ConnectException; import java.net.NoRouteToHostException; import java.net.SocketTimeoutException; import java.net.UnknownHostException; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Queue; -import java.util.concurrent.TimeUnit; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -54,13 +56,18 @@ import static org.junit.Assert.fail; /** - * Tests for {@link EndpointFailoverInterceptor}. Two flavours, no network: - *
    - *
  • Hand-rolled {@link RecordingChain}: cheap unit tests over a fake Chain.
    • - *
  • {@link TransportStub} plumbed into a real {@link OkHttpClient}: end-to-end - * tests through the actual OkHttp interceptor pipeline. Programs DNS misses, - * TLS failures, timeouts and 5xx outcomes per attempt.
    • - *
+ * Tests for {@link EndpointFailoverInterceptor}. + * + *

All behavior tests drive a real {@link CvmClient} with the standard + * profile/credential flow exactly the way users construct one — so the + * full pipeline (sign → log interceptor → failover interceptor → HTTP) + * runs end-to-end. Network is short-circuited by injecting a + * {@link TransportStub} interceptor at the tail of the OkHttpClient inside + * {@link HttpConnection}; the stub plays back scripted DNS misses, TLS + * failures, timeouts, and JSON success bodies per attempt. + * + *

The pure-helper tests at the top exercise package-private static + * methods directly — no client / no pipeline needed. */ public class EndpointFailoverInterceptorTest { @@ -69,7 +76,9 @@ public void resetState() { EndpointFailoverInterceptor.resetStateForTesting(); } - // ---- Pure helper tests ---- + // ================================================================= + // Pure helper tests + // ================================================================= @Test public void testIsKnownTencentCloudHost() { @@ -111,1124 +120,683 @@ public void testSubstituteTldNoMatchReturnsInput() { "example.com", "tencentcloudapi.com", "tencentcloudapi.cn")); } - // ---- Interceptor behavior tests ---- + // ================================================================= + // Behavior tests via real CvmClient + injected transport stub + // ================================================================= + + // ---- Pass-through paths ---- @Test public void testPassThroughForUnknownHost() throws Exception { - TestClient client = newTC3Client(); - EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); - Request req = newTC3Request("example.com"); - RecordingChain chain = new RecordingChain(req); - chain.programSuccess(); - - Response resp = it.intercept(chain); - assertEquals(200, resp.code()); - assertEquals("example.com", chain.requests.get(0).url().host()); + // Override endpoint to a non-Tencent host — interceptor must be inert. + ClientProfile profile = new ClientProfile(); + profile.getHttpProfile().setEndpoint("example.com"); + CvmClient client = newCvm(profile); + TransportStub transport = installStub(client); + transport.programOk(); + + client.DescribeInstances(new DescribeInstancesRequest()); + assertEquals(1, transport.received.size()); + assertEquals("example.com", transport.received.get(0).url().host()); } @Test - public void testPassThroughWhenSkipSign() throws Exception { - TestClient client = newTC3Client(); - EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); - Request req = newSkipSignV3Request("cvm.tencentcloudapi.com"); - RecordingChain chain = new RecordingChain(req); - chain.programSuccess(); - - it.intercept(chain); - assertEquals(1, chain.requests.size()); - assertEquals("cvm.tencentcloudapi.com", chain.requests.get(0).url().host()); - assertEquals("SKIP", chain.requests.get(0).header("Authorization")); + public void testFailoverFromComEndpoint() throws Exception { + CvmClient client = newCvm(); + TransportStub transport = installStub(client); + transport.programFailure(new UnknownHostException("dns miss")); + transport.programOk(); + + DescribeInstancesResponse resp = client.DescribeInstances(new DescribeInstancesRequest()); + assertNotNull(resp); + assertEquals(2, transport.received.size()); + assertEquals("cvm.tencentcloudapi.com", transport.received.get(0).url().host()); + assertEquals("cvm.tencentcloudapi.cn", transport.received.get(1).url().host()); + // Resigned request must carry Host header tracking new URL host. + assertEquals("cvm.tencentcloudapi.cn", transport.received.get(1).header("Host")); + // Authorization recomputed for new host. + assertNotEquals( + transport.received.get(0).header("Authorization"), + transport.received.get(1).header("Authorization")); } @Test - public void testFailoverRewritesSkipSignV3Request() throws Exception { - TestClient client = newTC3Client(); - EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); - Request req = newSkipSignV3Request("cvm.tencentcloudapi.com"); - RecordingChain chain = new RecordingChain(req); - chain.programDnsFailure(); - chain.programSuccess(); - - Response resp = it.intercept(chain); - assertEquals(200, resp.code()); - assertEquals(2, chain.requests.size()); - assertEquals("cvm.tencentcloudapi.com", chain.requests.get(0).url().host()); - assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(1).url().host()); - assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(1).header("Host")); - assertEquals("SKIP", chain.requests.get(1).header("Authorization")); + public void testFailoverFromCnEndpoint() throws Exception { + ClientProfile profile = new ClientProfile(); + profile.getHttpProfile().setEndpoint("cvm.tencentcloudapi.cn"); + CvmClient client = newCvm(profile); + TransportStub transport = installStub(client); + transport.programFailure(new UnknownHostException("dns miss")); + transport.programOk(); + + client.DescribeInstances(new DescribeInstancesRequest()); + assertEquals(2, transport.received.size()); + assertEquals("cvm.tencentcloudapi.cn", transport.received.get(0).url().host()); + assertEquals("cvm.tencentcloudapi.com", transport.received.get(1).url().host()); } @Test - public void testKnownDomainStillFailsOverAfterRuntimeDisable() throws Exception { - TestClient client = newTC3Client(); - client.getClientProfile().getHttpProfile().setDomainFailover(false); - EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); - Request req = newTC3Request("cvm.tencentcloudapi.com"); - RecordingChain chain = new RecordingChain(req); - chain.programDnsFailure(); - chain.programSuccess(); - - Response resp = it.intercept(chain); - assertEquals(200, resp.code()); - assertEquals(2, chain.requests.size()); - assertEquals("cvm.tencentcloudapi.com", chain.requests.get(0).url().host()); - assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(1).url().host()); + public void testFailoverDropsRegionalLabelFromHost() throws Exception { + ClientProfile profile = new ClientProfile(); + profile.getHttpProfile().setEndpoint("cvm.ap-guangzhou.tencentcloudapi.com"); + CvmClient client = newCvm(profile); + TransportStub transport = installStub(client); + transport.programFailure(new UnknownHostException("dns miss")); + transport.programOk(); + + client.DescribeInstances(new DescribeInstancesRequest()); + assertEquals(2, transport.received.size()); + assertEquals("cvm.ap-guangzhou.tencentcloudapi.com", + transport.received.get(0).url().host()); + assertEquals("cvm.tencentcloudapi.cn", transport.received.get(1).url().host()); } @Test - public void testPassThroughForBackupTldUsedAsEndpoint() throws Exception { - TestClient client = newTC3Client(); - EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); - Request req = newTC3Request("cvm.tencentcloudapi.cn"); - RecordingChain chain = new RecordingChain(req); - chain.programSuccess(); - - Response resp = it.intercept(chain); - assertEquals(200, resp.code()); - assertEquals(1, chain.requests.size()); - assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(0).url().host()); + public void testKnownDomainFailsOverEvenWhenFailoverDisabledAtRuntime() throws Exception { + // setDomainFailover(false) AFTER ctor — the interceptor was already + // installed at ctor time, so flipping the flag later cannot remove it. + // Documents the actual (slightly surprising) behavior. + CvmClient client = newCvm(); + client.getClientProfile().getHttpProfile().setDomainFailover(false); + TransportStub transport = installStub(client); + transport.programFailure(new UnknownHostException("dns miss")); + transport.programOk(); + + client.DescribeInstances(new DescribeInstancesRequest()); + assertEquals(2, transport.received.size()); } + // ---- shouldFailover branch coverage ---- + @Test - public void testFailoverFromCnEndpoint() throws Exception { - TestClient client = newTC3Client(); - EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); - Request req = newTC3Request("cvm.tencentcloudapi.cn"); - - RecordingChain chain = new RecordingChain(req); - chain.programDnsFailure(); - chain.programSuccess(); - - Response resp = it.intercept(chain); - assertEquals(200, resp.code()); - assertEquals(2, chain.requests.size()); - assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(0).url().host()); - assertEquals("cvm.tencentcloudapi.com", chain.requests.get(1).url().host()); - assertEquals("cvm.tencentcloudapi.com", chain.requests.get(1).header("Host")); + public void testFailoverOnSslHandshakeException() throws Exception { + runSingleFailureScenario(new SSLHandshakeException("tls handshake failed")); } @Test - public void testFailoverToBackupTldOnDnsFailure() throws Exception { - TestClient client = newTC3Client(); - EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); - Request req = newTC3Request("cvm.tencentcloudapi.com"); - - RecordingChain chain = new RecordingChain(req); - chain.programDnsFailure(); - chain.programSuccess(); - - Response resp = it.intercept(chain); - assertEquals(200, resp.code()); - assertEquals(2, chain.requests.size()); - assertEquals("cvm.tencentcloudapi.com", chain.requests.get(0).url().host()); - assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(1).url().host()); - assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(1).header("Host")); - assertNotNull(chain.requests.get(1).header("Authorization")); - assertNotEquals( - chain.requests.get(0).header("Authorization"), - chain.requests.get(1).header("Authorization")); + public void testFailoverOnSslPeerUnverifiedException() throws Exception { + runSingleFailureScenario(new SSLPeerUnverifiedException("cert mismatch")); } @Test - public void testFailoverDropsRegionalLabelFromHost() throws Exception { - TestClient client = newTC3Client(); - EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); - Request req = newTC3Request("cvm.ap-guangzhou.tencentcloudapi.com"); - - RecordingChain chain = new RecordingChain(req); - chain.programDnsFailure(); - chain.programSuccess(); - - Response resp = it.intercept(chain); - assertEquals(200, resp.code()); - assertEquals(2, chain.requests.size()); - assertEquals("cvm.ap-guangzhou.tencentcloudapi.com", chain.requests.get(0).url().host()); - assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(1).url().host()); - assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(1).header("Host")); + public void testFailoverOnConnectException() throws Exception { + runSingleFailureScenario(new ConnectException("connection refused")); } @Test - public void testAllBackupTldsFailAggregatesEveryAttemptFailure() throws Exception { - TestClient client = newTC3Client(); - EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); - Request req = newTC3Request("cvm.tencentcloudapi.com"); - - RecordingChain chain = new RecordingChain(req); - chain.programDnsFailure("first dns miss"); - chain.programDnsFailure("second dns miss"); - chain.programDnsFailure("third dns miss"); - - IOException thrown = null; - try { - it.intercept(chain); - fail("expected IOException"); - } catch (IOException e) { - thrown = e; - } - // Primary cause = last attempt. Wrapper IOException carries the - // original UnknownHostException as its cause so the root type is - // recoverable for callers that want to switch on it. - assertTrue("primary message should mention last host, got: " + thrown.getMessage(), - thrown.getMessage().contains("cvm.tencentcloudapi.com.cn")); - assertTrue("primary message should mention last attempt's failure, got: " + thrown.getMessage(), - thrown.getMessage().contains("third dns miss")); - assertNotNull(thrown.getCause()); - assertTrue(thrown.getCause() instanceof UnknownHostException); - assertEquals("third dns miss", thrown.getCause().getMessage()); - - // Every other attempt is attached as a suppressed exception so a - // single stack-trace dump exposes all 3 root causes. - Throwable[] suppressed = thrown.getSuppressed(); - assertEquals(2, suppressed.length); - assertTrue(suppressed[0].getMessage().contains("cvm.tencentcloudapi.com")); - assertTrue(suppressed[0].getMessage().contains("first dns miss")); - assertTrue(suppressed[0].getCause() instanceof UnknownHostException); - assertTrue(suppressed[1].getMessage().contains("cvm.tencentcloudapi.cn")); - assertTrue(suppressed[1].getMessage().contains("second dns miss")); - assertTrue(suppressed[1].getCause() instanceof UnknownHostException); - - assertEquals(3, chain.requests.size()); - assertEquals("cvm.tencentcloudapi.com", chain.requests.get(0).url().host()); - assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(1).url().host()); - assertEquals("cvm.tencentcloudapi.com.cn", chain.requests.get(2).url().host()); - chain.assertAllProgrammedConsumed(); + public void testFailoverOnNoRouteToHostException() throws Exception { + runSingleFailureScenario(new NoRouteToHostException("no route")); } @Test - public void testAggregatedFailurePreservesPerAttemptCauseTypes() throws Exception { - // Different IOException subtypes per attempt — each must round-trip - // intact through the aggregation so callers can switch on root type - // (e.g. distinguish TLS vs DNS vs connect failure for diagnostics). - TestClient client = newTC3Client(); - EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); - Request req = newTC3Request("cvm.tencentcloudapi.com"); - - RecordingChain chain = new RecordingChain(req); - chain.programFailure(new UnknownHostException("dns miss .com")); - chain.programFailure(new SSLHandshakeException("tls fail .cn")); - chain.programFailure(new ConnectException("connect fail .com.cn")); - - IOException thrown = null; - try { - it.intercept(chain); - fail("expected IOException"); - } catch (IOException e) { - thrown = e; - } - // Primary = last attempt's wrapper; cause = ConnectException. - assertTrue(thrown.getCause() instanceof ConnectException); - assertEquals("connect fail .com.cn", thrown.getCause().getMessage()); + public void testFailoverOnSocketTimeoutException() throws Exception { + runSingleFailureScenario(new SocketTimeoutException("read timed out")); + } - Throwable[] suppressed = thrown.getSuppressed(); - assertEquals(2, suppressed.length); - assertTrue(suppressed[0].getCause() instanceof UnknownHostException); - assertEquals("dns miss .com", suppressed[0].getCause().getMessage()); - assertTrue(suppressed[1].getCause() instanceof SSLHandshakeException); - assertEquals("tls fail .cn", suppressed[1].getCause().getMessage()); - chain.assertAllProgrammedConsumed(); + private void runSingleFailureScenario(IOException firstFailure) throws Exception { + CvmClient client = newCvm(); + TransportStub transport = installStub(client); + transport.programFailure(firstFailure); + transport.programOk(); + + client.DescribeInstances(new DescribeInstancesRequest()); + assertEquals(2, transport.received.size()); + assertEquals("cvm.tencentcloudapi.com", transport.received.get(0).url().host()); + assertEquals("cvm.tencentcloudapi.cn", transport.received.get(1).url().host()); } + // ---- Non-failover IOException must propagate without retry ---- + @Test - public void testAggregatedFailureMixesBreakerSkipsWithRealFailures() throws Exception { - // Pre-open the .com breaker so the first candidate is short-circuited - // (no real attempt, placeholder IOException with no cause). The next - // two TLDs hit transport and fail. Aggregation must include all three - // entries in attempt order. - TestClient client = newTC3Client(); - EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); - - // Open the .com breaker via a 5-failure burst on a throwaway chain. - for (int i = 0; i < 5; i++) { - RecordingChain warmup = new RecordingChain(newTC3Request("cvm.tencentcloudapi.com")); - warmup.programDnsFailure("warmup fail " + i); - warmup.programSuccess(); // .cn succeeds, doesn't touch .com breaker - it.intercept(warmup); - EndpointFailoverInterceptor.FailoverState state = - EndpointFailoverInterceptor.STATE.get("cvm.tencentcloudapi.com"); - state.originProbeAfterMs = 0; // force .com to be retried each loop - } + public void testGenericIOExceptionPropagatesWithoutFailover() throws Exception { + CvmClient client = newCvm(); + TransportStub transport = installStub(client); + transport.programFailure(new IOException("some unrelated I/O error")); - // .com breaker should now be Open. - EndpointFailoverInterceptor.FailoverState state = - EndpointFailoverInterceptor.STATE.get("cvm.tencentcloudapi.com"); - // Sanity: confirm Open. - assertFalse("expected .com breaker Open after 5 failures", - state.breakers[0].allow().allowed); - - // Real run: .com short-circuits, .cn fails, .com.cn fails. - // candidates() order with currentIndex=.cn and originProbeAfterMs=0: - // first .com (origin reprobe due), then .cn (preferred), then .com.cn. - state.originProbeAfterMs = 0; - Request req = newTC3Request("cvm.tencentcloudapi.com"); - RecordingChain chain = new RecordingChain(req); - chain.programFailure(new SSLHandshakeException("cn tls fail")); - chain.programFailure(new ConnectException("com.cn connect fail")); - - IOException thrown = null; try { - it.intercept(chain); - fail("expected IOException"); - } catch (IOException e) { - thrown = e; + client.DescribeInstances(new DescribeInstancesRequest()); + fail("expected SDK exception"); + } catch (TencentCloudSDKException e) { + // SDK wraps the IOException as cause. + Throwable cause = unwrapToIOException(e); + assertEquals("some unrelated I/O error", cause.getMessage()); } + assertEquals("must not retry on non-failover IOException", 1, transport.received.size()); + } - // Only 2 chain.proceed calls — .com was skipped by breaker. - assertEquals(2, chain.requests.size()); - assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(0).url().host()); - assertEquals("cvm.tencentcloudapi.com.cn", chain.requests.get(1).url().host()); + // ---- HTTP body / status reaches caller intact after failover ---- - // Primary = last attempt (com.cn ConnectException). - assertTrue(thrown.getCause() instanceof ConnectException); + @Test + public void testApiResponseDeliveredAfterFailover() throws Exception { + CvmClient client = newCvm(); + TransportStub transport = installStub(client); + transport.programFailure(new UnknownHostException("dns miss")); + transport.programJsonOk("{\"Response\":{\"TotalCount\":42,\"InstanceSet\":[],\"RequestId\":\"req-xyz\"}}"); - // Suppressed: [.com breaker skip placeholder, .cn SSLHandshake wrapper]. - Throwable[] suppressed = thrown.getSuppressed(); - assertEquals(2, suppressed.length); - // Order matters — must reflect attempt order, not failure-type grouping. - assertTrue("first suppressed must be .com breaker skip, got: " + suppressed[0].getMessage(), - suppressed[0].getMessage().contains("cvm.tencentcloudapi.com") - && suppressed[0].getMessage().contains("circuit breaker open")); - assertNull("breaker-skip placeholder has no underlying cause", - suppressed[0].getCause()); - assertTrue(suppressed[1].getCause() instanceof SSLHandshakeException); + DescribeInstancesResponse resp = client.DescribeInstances(new DescribeInstancesRequest()); + assertEquals(Long.valueOf(42), resp.getTotalCount()); + assertEquals("req-xyz", resp.getRequestId()); } - @Test - public void testAggregatedFailureWhenPrimaryIsBreakerSkip() throws Exception { - // Pre-fail .cn and .com.cn breakers so they're Open while .com is still - // Closed. Then drive a request where .com fails (the only attempt - // that reaches transport). Both breaker-skips arrive AFTER .com's - // failure in attempt order, so the *primary* (last entry) is a - // breaker-skip placeholder with no cause. - TestClient client = newTC3Client(); - EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); - - EndpointFailoverInterceptor.FailoverState state = - new EndpointFailoverInterceptor.FailoverState(60_000); - EndpointFailoverInterceptor.STATE.put("cvm.tencentcloudapi.com", state); - // Open .cn (idx=1) and .com.cn (idx=2); leave .com (idx=0) Closed. - for (int idx : new int[]{1, 2}) { - for (int i = 0; i < 6; i++) { - CircuitBreaker.Token t = state.breakers[idx].allow(); - if (t.allowed) { - t.report(false); - } - } - } + // ---- 5xx server response is not a failover trigger ---- - Request req = newTC3Request("cvm.tencentcloudapi.com"); - RecordingChain chain = new RecordingChain(req); - chain.programFailure(new UnknownHostException("com dns fail")); + @Test + public void test5xxResponseDoesNotTriggerFailover() throws Exception { + CvmClient client = newCvm(); + TransportStub transport = installStub(client); + transport.programResponse(503, "service unavailable"); - IOException thrown = null; try { - it.intercept(chain); - fail("expected IOException"); - } catch (IOException e) { - thrown = e; + client.DescribeInstances(new DescribeInstancesRequest()); + fail("expected SDK exception for 503"); + } catch (TencentCloudSDKException e) { + assertTrue("message should mention non-200 code, got: " + e.getMessage(), + e.getMessage().contains("503")); } - // Only .com hit transport. - assertEquals(1, chain.requests.size()); - assertEquals("cvm.tencentcloudapi.com", chain.requests.get(0).url().host()); - - // Primary = last attempt (.com.cn breaker skip), no cause. - assertNull("breaker-skip primary has no cause", thrown.getCause()); - assertTrue(thrown.getMessage().contains("cvm.tencentcloudapi.com.cn")); - assertTrue(thrown.getMessage().contains("circuit breaker open")); - - // Suppressed: [.com real failure, .cn breaker skip]. - Throwable[] suppressed = thrown.getSuppressed(); - assertEquals(2, suppressed.length); - assertTrue(suppressed[0].getCause() instanceof UnknownHostException); - assertEquals("com dns fail", suppressed[0].getCause().getMessage()); - assertNull(suppressed[1].getCause()); - assertTrue(suppressed[1].getMessage().contains("cvm.tencentcloudapi.cn")); + assertEquals(1, transport.received.size()); } + // ---- TC3 resign preserves body / content-type / signing scope ---- + @Test - public void testFailoverDoesNotPollutateNextRequestAttemptFailures() throws Exception { - // attemptFailures lives on Failover (per-intercept), not on the - // interceptor — guard against accidental promotion to instance state - // that would carry stale failures into the next request. - TestClient client = newTC3Client(); - EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); - - // Run 1: 1 failure + 1 success → no exception, no aggregation. - { - RecordingChain chain = new RecordingChain(newTC3Request("cvm.tencentcloudapi.com")); - chain.programDnsFailure("run1 fail"); - chain.programSuccess(); - Response resp = it.intercept(chain); - assertEquals(200, resp.code()); - } + public void testTC3ResignPreservesBodyAndContentType() throws Exception { + CvmClient client = newCvm(); + TransportStub transport = installStub(client); + transport.programFailure(new UnknownHostException("dns miss")); + transport.programOk(); - // Run 2: all-fail. Suppressed must contain ONLY run-2 failures, not - // any leftover from run 1. - { - RecordingChain chain = new RecordingChain(newTC3Request("cvm.tencentcloudapi.com")); - chain.programDnsFailure("run2 com fail"); - chain.programDnsFailure("run2 cn fail"); - chain.programDnsFailure("run2 com.cn fail"); - IOException thrown = null; - try { - it.intercept(chain); - fail("expected IOException"); - } catch (IOException e) { - thrown = e; - } - assertEquals(2, thrown.getSuppressed().length); - for (Throwable s : thrown.getSuppressed()) { - assertFalse("must not leak run-1 failure into run-2 aggregation: " + s.getMessage(), - s.getMessage().contains("run1")); - } - assertTrue(thrown.getMessage().contains("run2")); - } - } + DescribeInstancesRequest req = new DescribeInstancesRequest(); + req.setLimit(10L); + req.setOffset(0L); + req.setInstanceIds(new String[]{"ins-aaa", "ins-bbb"}); + client.DescribeInstances(req); - @Test - public void testFollowupRequestUsesKnownWorkingTld() throws Exception { - TestClient client = newTC3Client(); - EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); - - { - Request req = newTC3Request("cvm.tencentcloudapi.com"); - RecordingChain chain = new RecordingChain(req); - chain.programDnsFailure(); - chain.programSuccess(); - Response resp = it.intercept(chain); - assertEquals(200, resp.code()); - chain.assertAllProgrammedConsumed(); - } + Request first = transport.received.get(0); + Request resigned = transport.received.get(1); - { - Request req = newTC3Request("cvm.tencentcloudapi.com"); - RecordingChain chain = new RecordingChain(req); - // Two outcomes programmed: if the interceptor wrongly probes - // .com first, it would consume the failure and need the second - // success — leaving zero leftovers. With correct behavior only - // .cn is tried, leaving one outcome unconsumed (asserted below). - chain.programSuccess(); - chain.programSuccess(); - Response resp = it.intercept(chain); - assertEquals(200, resp.code()); - assertEquals(1, chain.requests.size()); - assertEquals("cvm.tencentcloudapi.cn", chain.requests.get(0).url().host()); - assertEquals("must take exactly one outcome (no .com probe)", - 1, chain.programmedRemaining()); - } + // Same body bytes round-trip through resign. + assertArrayEquals(bodyBytes(first), bodyBytes(resigned)); + assertEquals(first.header("Content-Type"), resigned.header("Content-Type")); + + // Authorization rebound for new host scope. + assertNotEquals(first.header("Authorization"), resigned.header("Authorization")); + assertTrue(resigned.header("Authorization").startsWith("TC3-HMAC-SHA256 ")); + assertTrue(resigned.header("Authorization").contains("/cvm/tc3_request")); } + // ---- X-TC-Token rotation visible to resigned request ---- + @Test - public void testFollowupRequestReprobesOriginalTldAfterCooldown() throws Exception { - TestClient client = newTC3Client(); - EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); - - { - Request req = newTC3Request("cvm.tencentcloudapi.com"); - RecordingChain chain = new RecordingChain(req); - chain.programDnsFailure(); - chain.programSuccess(); - Response resp = it.intercept(chain); - assertEquals(200, resp.code()); - chain.assertAllProgrammedConsumed(); - } + public void testResignReflectsRotatedToken() throws Exception { + CvmClient client = newCvm(); + client.setCredential(new Credential("AKIDTEST", "SKTEST", "tok-v1")); + TransportStub transport = installStub(client); + transport.programFailure(new UnknownHostException("dns miss")); + transport.programOk(); - EndpointFailoverInterceptor.FailoverState state = - EndpointFailoverInterceptor.STATE.get("cvm.tencentcloudapi.com"); - assertNotNull(state); - state.originProbeAfterMs = 0; - - { - Request req = newTC3Request("cvm.tencentcloudapi.com"); - RecordingChain chain = new RecordingChain(req); - // Belt-and-braces: extra success queued so a wrong fallback path - // would still complete the test rather than blow up with - // "no programmed outcomes" — which would mask the real bug. - chain.programSuccess(); - chain.programSuccess(); - Response resp = it.intercept(chain); - assertEquals(200, resp.code()); - assertEquals(1, chain.requests.size()); - assertEquals("cvm.tencentcloudapi.com", chain.requests.get(0).url().host()); - assertEquals("must take exactly one outcome (origin probe succeeded first)", - 1, chain.programmedRemaining()); - // After the successful origin probe the cooldown must clear, - // otherwise the next request would reprobe forever. - assertEquals(-1, state.originProbeAfterMs); - } - } + // Mid-flight credential rotation: hook a one-shot interceptor that + // swaps the token after the first request goes out, before the + // failover interceptor resigns for the backup TLD. + AtomicTokenSwapper swapper = new AtomicTokenSwapper(client, "tok-v2"); + installInterceptorBefore(client, swapper); - @Test - public void testResignPicksUpRotatedCredential() throws Exception { - // Rotating the credential on the AbstractClient between initial sign and - // failover resign should be reflected in the new Authorization header, - // because the interceptor reads client.credential live. Verify the - // resigned signature is actually computed with the NEW SecretKey, not - // just the new SecretId — independently reproduce the TC3 signature - // using SKNEW + the timestamp echoed in the resigned headers, then - // assert byte-for-byte equality. - TestClient client = newTC3Client(); - EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); - Request req = newTC3Request("cvm.tencentcloudapi.com"); - String origAuth = req.header("Authorization"); - - client.setCredential(new Credential("AKIDNEW", "SKNEW")); - - RecordingChain chain = new RecordingChain(req); - chain.programDnsFailure(); - chain.programSuccess(); - - it.intercept(chain); - Request resigned = chain.requests.get(1); - String resignedAuth = resigned.header("Authorization"); - assertNotNull(resignedAuth); - assertNotEquals(origAuth, resignedAuth); - assertTrue("resigned auth should use rotated secretId, got: " + resignedAuth, - resignedAuth.contains("Credential=AKIDNEW/")); - - String resignedTimestamp = resigned.header("X-TC-Timestamp"); - assertNotNull(resignedTimestamp); - String expectedAuth = reproduceTc3SignaturePost( - "AKIDNEW", "SKNEW", - "cvm.tencentcloudapi.cn", "cvm", - "application/json; charset=utf-8", - "{}".getBytes(StandardCharsets.UTF_8), - Long.parseLong(resignedTimestamp)); - assertEquals("resigned signature must be computable with the rotated SK", - expectedAuth, resignedAuth); - - // Negative control: same inputs but with the OLD secret produce a - // different signature — proves the assertion above is meaningful and - // not a tautology. - String wrongAuth = reproduceTc3SignaturePost( - "AKIDTEST", "SKTEST", - "cvm.tencentcloudapi.cn", "cvm", - "application/json; charset=utf-8", - "{}".getBytes(StandardCharsets.UTF_8), - Long.parseLong(resignedTimestamp)); - assertNotEquals(wrongAuth, resignedAuth); + client.DescribeInstances(new DescribeInstancesRequest()); + assertEquals("tok-v1", transport.received.get(0).header("X-TC-Token")); + assertEquals("tok-v2", transport.received.get(1).header("X-TC-Token")); } @Test - public void testResignHmacReplacesSignatureForNewHost() throws Exception { - TestClient client = newHmacClient(); - EndpointFailoverInterceptor it = new EndpointFailoverInterceptor(client); - Request req = newHmacRequest("cvm.tencentcloudapi.com"); - String originalSig = req.url().queryParameter("Signature"); - assertNotNull(originalSig); - - RecordingChain chain = new RecordingChain(req); - chain.programDnsFailure(); - chain.programSuccess(); - - it.intercept(chain); - assertEquals(2, chain.requests.size()); - Request resigned = chain.requests.get(1); - assertEquals("cvm.tencentcloudapi.cn", resigned.url().host()); - String newSig = resigned.url().queryParameter("Signature"); - assertNotNull(newSig); - assertNotEquals(originalSig, newSig); - - // Stronger check: independently reproduce the expected V1 signature - // for the new host using the same SK and parameters, and compare. - // Sig changing alone is too weak — host change forces signature - // change for any half-correct implementation. This catches subtle - // bugs like signing with the wrong host, double-signing the old - // Signature param, or losing query params during resign. - java.util.TreeMap params = new java.util.TreeMap(); - params.put("Action", "TestAction"); - params.put("Version", "2020-01-01"); - params.put("Region", "ap-guangzhou"); - params.put("SecretId", "AKIDTEST"); - params.put("Timestamp", "1700000000"); - params.put("Nonce", "12345"); - params.put("SignatureMethod", "HmacSHA256"); - String plain = Sign.makeSignPlainText( - params, HttpProfile.REQ_GET, "cvm.tencentcloudapi.cn", "/"); - String expectedSig = Sign.sign("SKTEST", plain, ClientProfile.SIGN_SHA256); - assertEquals(expectedSig, newSig); - - // Negative control: signing for the ORIGINAL host produces a different - // signature, proving the resign actually rebound to the new host. - String wrongHostPlain = Sign.makeSignPlainText( - params, HttpProfile.REQ_GET, "cvm.tencentcloudapi.com", "/"); - String wrongHostSig = Sign.sign("SKTEST", wrongHostPlain, ClientProfile.SIGN_SHA256); - assertNotEquals(wrongHostSig, newSig); - - // Confirm every original query param is preserved (none lost on resign). - assertEquals("TestAction", resigned.url().queryParameter("Action")); - assertEquals("2020-01-01", resigned.url().queryParameter("Version")); - assertEquals("ap-guangzhou", resigned.url().queryParameter("Region")); - assertEquals("AKIDTEST", resigned.url().queryParameter("SecretId")); - assertEquals("1700000000", resigned.url().queryParameter("Timestamp")); - assertEquals("12345", resigned.url().queryParameter("Nonce")); - assertEquals("HmacSHA256", resigned.url().queryParameter("SignatureMethod")); - // Old Signature must be replaced, not duplicated. - assertEquals("Signature must appear exactly once", - 1, resigned.url().queryParameterValues("Signature").size()); - } - - // ---- Helpers ---- + public void testResignDropsTokenWhenCleared() throws Exception { + CvmClient client = newCvm(); + client.setCredential(new Credential("AKIDTEST", "SKTEST", "tok-v1")); + TransportStub transport = installStub(client); + transport.programFailure(new UnknownHostException("dns miss")); + transport.programOk(); - /** - * Independently reproduces a TC3-HMAC-SHA256 Authorization header for a - * POST request with content-type;host as the signed headers. Used by - * {@link #testResignPicksUpRotatedCredential} to verify the interceptor's - * resigned signature byte-for-byte. Mirrors {@code resignV3} in the - * interceptor; if either drifts, this assertion catches it. - */ - private static String reproduceTc3SignaturePost(String secretId, String secretKey, - String host, String service, - String contentType, byte[] payload, - long timestampSec) throws Exception { - String canonicalHeaders = "content-type:" + contentType + "\nhost:" + host + "\n"; - String signedHeaders = "content-type;host"; - String hashedPayload = Sign.sha256Hex(payload); - String canonicalRequest = "POST\n/\n\n" + canonicalHeaders + "\n" - + signedHeaders + "\n" + hashedPayload; - - java.text.SimpleDateFormat sdf = new java.text.SimpleDateFormat("yyyy-MM-dd"); - sdf.setTimeZone(java.util.TimeZone.getTimeZone("UTC")); - String date = sdf.format(new java.util.Date(timestampSec * 1000L)); - String credentialScope = date + "/" + service + "/tc3_request"; - String hashedCanonical = Sign.sha256Hex(canonicalRequest.getBytes(StandardCharsets.UTF_8)); - String stringToSign = "TC3-HMAC-SHA256\n" + timestampSec + "\n" - + credentialScope + "\n" + hashedCanonical; - - byte[] secretDate = Sign.hmac256(("TC3" + secretKey).getBytes(StandardCharsets.UTF_8), date); - byte[] secretService = Sign.hmac256(secretDate, service); - byte[] secretSigning = Sign.hmac256(secretService, "tc3_request"); - String signature = DatatypeConverter - .printHexBinary(Sign.hmac256(secretSigning, stringToSign)) - .toLowerCase(); - return "TC3-HMAC-SHA256 " - + "Credential=" + secretId + "/" + credentialScope + ", " - + "SignedHeaders=" + signedHeaders + ", " - + "Signature=" + signature; - } + // Rotate creds to one without a token between attempts. + AtomicTokenSwapper clearer = new AtomicTokenSwapper(client, null); + installInterceptorBefore(client, clearer); - /** Minimal concrete AbstractClient subclass usable in tests. */ - private static final class TestClient extends AbstractClient { - TestClient(ClientProfile profile) { - super("cvm.tencentcloudapi.com", "2020-01-01", - new Credential("AKIDTEST", "SKTEST"), "ap-guangzhou", profile); - } + client.DescribeInstances(new DescribeInstancesRequest()); + assertEquals("tok-v1", transport.received.get(0).header("X-TC-Token")); + assertNull("token must be removed on resign when credential drops it", + transport.received.get(1).header("X-TC-Token")); } - private TestClient newTC3Client() { - ClientProfile profile = new ClientProfile(); - profile.getHttpProfile().setReqMethod(HttpProfile.REQ_POST); - return new TestClient(profile); - } + // ---- Hmac (V1) resign preserves all params; signature rebuilt for new host ---- - private TestClient newHmacClient() { + @Test + public void testHmacResignPreservesQueryParams() throws Exception { ClientProfile profile = new ClientProfile(); profile.setSignMethod(ClientProfile.SIGN_SHA256); profile.getHttpProfile().setReqMethod(HttpProfile.REQ_GET); - return new TestClient(profile); - } - - /** - * Hand-crafted TC3-signed Request minimal enough for the interceptor to - * recognise and re-sign. Body + contentType + X-TC-* headers + Authorization - * are all the interceptor needs; the signature value itself is opaque to - * re-sign (it's recomputed from scratch for the backup host). - */ - private static Request newTC3Request(String host) { - byte[] body = "{}".getBytes(); - return new Request.Builder() - .url("https://" + host + "/") - .post(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), body)) - .header("Content-Type", "application/json; charset=utf-8") - .header("Host", host) - .header("Authorization", - "TC3-HMAC-SHA256 Credential=AKIDTEST/2024-01-01/cvm/tc3_request," - + " SignedHeaders=content-type;host, Signature=deadbeef") - .header("X-TC-Action", "TestAction") - .header("X-TC-Timestamp", "1700000000") - .header("X-TC-Version", "2020-01-01") - .header("X-TC-RequestClient", "SDK_JAVA_TEST") - .header("X-TC-Region", "ap-guangzhou") - .build(); - } - - private static Request newSkipSignV3Request(String host) { - return new Request.Builder() - .url("https://" + host + "/") - .post(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), "{}")) - .header("Content-Type", "application/json; charset=utf-8") - .header("Host", host) - .header("Authorization", "SKIP") - .header("X-TC-Action", "TestAction") - .header("X-TC-Timestamp", "1700000000") - .header("X-TC-Version", "2020-01-01") - .header("X-TC-RequestClient", "SDK_JAVA_TEST") - .header("X-TC-Region", "ap-guangzhou") - .build(); - } + CvmClient client = newCvm(profile); + TransportStub transport = installStub(client); + transport.programFailure(new UnknownHostException("dns miss")); + transport.programOk(); - /** Hand-crafted Hmac-signed GET Request with Signature in query string. */ - private static Request newHmacRequest(String host) { - return new Request.Builder() - .url("https://" + host + "/?Action=TestAction" - + "&Version=2020-01-01" - + "&Region=ap-guangzhou" - + "&SecretId=AKIDTEST" - + "&Timestamp=1700000000" - + "&Nonce=12345" - + "&SignatureMethod=HmacSHA256" - + "&Signature=deadbeefdeadbeef") - .get() - .build(); - } + client.DescribeInstances(new DescribeInstancesRequest()); - private static Response okResponse(Request req) { - return new Response.Builder() - .request(req) - .protocol(Protocol.HTTP_1_1) - .code(200) - .message("OK") - .body(ResponseBody.create(MediaType.parse("application/json"), "{}")) - .build(); + Request resigned = transport.received.get(1); + assertEquals("cvm.tencentcloudapi.cn", resigned.url().host()); + assertEquals("DescribeInstances", resigned.url().queryParameter("Action")); + assertEquals("2017-03-12", resigned.url().queryParameter("Version")); + assertEquals("ap-guangzhou", resigned.url().queryParameter("Region")); + assertEquals("AKIDTEST", resigned.url().queryParameter("SecretId")); + assertEquals("HmacSHA256", resigned.url().queryParameter("SignatureMethod")); + // Signature replaced, not appended. + List sigs = resigned.url().queryParameterValues("Signature"); + assertEquals("must have exactly one Signature param", 1, sigs.size()); + assertNotEquals(transport.received.get(0).url().queryParameter("Signature"), + resigned.url().queryParameter("Signature")); } - private static final class RecordingChain implements Interceptor.Chain { - final List requests = new ArrayList(); - private final List programmed = new ArrayList(); - private int idx = 0; - private Request current; - - RecordingChain(Request initialRequest) { - this.current = initialRequest; - } - - void programSuccess() { - programmed.add(null); - } + // ---- Aggregation: every TLD failure surfaces in a single exception ---- - void programDnsFailure() { - programDnsFailure("injected DNS failure"); - } + @Test + public void testAllBackupTldsFailAggregatesEveryAttemptFailure() throws Exception { + CvmClient client = newCvm(); + TransportStub transport = installStub(client); + transport.programFailure(new UnknownHostException("first dns miss")); + transport.programFailure(new UnknownHostException("second dns miss")); + transport.programFailure(new UnknownHostException("third dns miss")); - void programDnsFailure(String message) { - programmed.add(new UnknownHostException(message)); + TencentCloudSDKException sdkEx = null; + try { + client.DescribeInstances(new DescribeInstancesRequest()); + fail("expected SDK exception"); + } catch (TencentCloudSDKException e) { + sdkEx = e; } - void programFailure(IOException e) { - programmed.add(e); - } + IOException primary = unwrapToIOException(sdkEx); + assertTrue(primary.getMessage().contains("cvm.tencentcloudapi.com.cn")); + assertTrue(primary.getMessage().contains("third dns miss")); + assertTrue(primary.getCause() instanceof UnknownHostException); + assertEquals("third dns miss", primary.getCause().getMessage()); - int programmedRemaining() { - return programmed.size() - idx; - } + Throwable[] suppressed = primary.getSuppressed(); + assertEquals(2, suppressed.length); + assertTrue(suppressed[0].getMessage().contains("cvm.tencentcloudapi.com")); + assertTrue(suppressed[0].getCause() instanceof UnknownHostException); + assertEquals("first dns miss", suppressed[0].getCause().getMessage()); + assertTrue(suppressed[1].getMessage().contains("cvm.tencentcloudapi.cn")); + assertTrue(suppressed[1].getCause() instanceof UnknownHostException); + assertEquals("second dns miss", suppressed[1].getCause().getMessage()); - void assertAllProgrammedConsumed() { - if (programmedRemaining() != 0) { - throw new AssertionError( - "Expected all programmed outcomes to be consumed but " - + programmedRemaining() + " remain"); - } - } + assertEquals(3, transport.received.size()); + } - @Override - public Request request() { - return current; - } + @Test + public void testAggregatedFailurePreservesPerAttemptCauseTypes() throws Exception { + CvmClient client = newCvm(); + TransportStub transport = installStub(client); + transport.programFailure(new UnknownHostException("dns miss .com")); + transport.programFailure(new SSLHandshakeException("tls fail .cn")); + transport.programFailure(new ConnectException("connect fail .com.cn")); - @Override - public Response proceed(Request request) throws IOException { - current = request; - requests.add(request); - if (idx >= programmed.size()) { - throw new IllegalStateException("No more programmed responses"); - } - Object next = programmed.get(idx++); - if (next instanceof Throwable) { - if (next instanceof IOException) { - throw (IOException) next; - } - throw new RuntimeException((Throwable) next); - } - if (next == null) { - return okResponse(request); - } - return (Response) next; + TencentCloudSDKException sdkEx = null; + try { + client.DescribeInstances(new DescribeInstancesRequest()); + fail("expected SDK exception"); + } catch (TencentCloudSDKException e) { + sdkEx = e; } - @Override public Connection connection() { return null; } - @Override public Call call() { return null; } - @Override public int connectTimeoutMillis() { return 0; } - @Override public Interceptor.Chain withConnectTimeout(int timeout, TimeUnit unit) { return this; } - @Override public int readTimeoutMillis() { return 0; } - @Override public Interceptor.Chain withReadTimeout(int timeout, TimeUnit unit) { return this; } - @Override public int writeTimeoutMillis() { return 0; } - @Override public Interceptor.Chain withWriteTimeout(int timeout, TimeUnit unit) { return this; } + IOException primary = unwrapToIOException(sdkEx); + assertTrue(primary.getCause() instanceof ConnectException); + Throwable[] suppressed = primary.getSuppressed(); + assertEquals(2, suppressed.length); + assertTrue(suppressed[0].getCause() instanceof UnknownHostException); + assertTrue(suppressed[1].getCause() instanceof SSLHandshakeException); } - // ==================================================================== - // End-to-end tests: real OkHttpClient pipeline + TransportStub. - // Exercises the interceptor through OkHttp's actual chain, not a hand- - // rolled fake. Catches integration bugs RecordingChain can't. - // ==================================================================== + @Test + public void testAggregatedFailureMixesBreakerSkipsWithRealFailures() throws Exception { + // Pre-open .com breaker, then drive a request where .cn and .com.cn + // both fail at transport. .com is short-circuited (placeholder, no + // cause); the other two contribute real cause chains. + CvmClient client = newCvm(); + EndpointFailoverInterceptor.FailoverState state = + new EndpointFailoverInterceptor.FailoverState(60_000); + EndpointFailoverInterceptor.STATE.put("cvm.tencentcloudapi.com", state); + tripBreaker(state.breakers[0]); // .com Open - // ---- shouldFailover branch coverage ---- + TransportStub transport = installStub(client); + // candidates() with currentIndex=-1 (.com origin still preferred but + // breaker open) → order: .com (skipped), .cn, .com.cn. + transport.programFailure(new SSLHandshakeException("cn tls fail")); + transport.programFailure(new ConnectException("com.cn connect fail")); - @Test - public void e2eFailoverOnSslHandshakeException() throws Exception { - runE2EFailoverScenario(new SSLHandshakeException("tls handshake failed")); - } + TencentCloudSDKException sdkEx = null; + try { + client.DescribeInstances(new DescribeInstancesRequest()); + fail("expected SDK exception"); + } catch (TencentCloudSDKException e) { + sdkEx = e; + } - @Test - public void e2eFailoverOnSslPeerUnverifiedException() throws Exception { - runE2EFailoverScenario(new SSLPeerUnverifiedException("cert mismatch")); - } + // .com never reached transport. + assertEquals(2, transport.received.size()); + assertEquals("cvm.tencentcloudapi.cn", transport.received.get(0).url().host()); + assertEquals("cvm.tencentcloudapi.com.cn", transport.received.get(1).url().host()); - @Test - public void e2eFailoverOnConnectException() throws Exception { - runE2EFailoverScenario(new ConnectException("connection refused")); - } + IOException primary = unwrapToIOException(sdkEx); + // Last attempt = ConnectException on .com.cn. + assertTrue(primary.getCause() instanceof ConnectException); - @Test - public void e2eFailoverOnNoRouteToHostException() throws Exception { - runE2EFailoverScenario(new NoRouteToHostException("no route")); + // Suppressed[0] = .com breaker placeholder; Suppressed[1] = .cn TLS failure. + Throwable[] suppressed = primary.getSuppressed(); + assertEquals(2, suppressed.length); + assertTrue(suppressed[0].getMessage().contains("cvm.tencentcloudapi.com")); + assertTrue(suppressed[0].getMessage().contains("circuit breaker open")); + assertNull("breaker-skip placeholder has no cause", suppressed[0].getCause()); + assertTrue(suppressed[1].getCause() instanceof SSLHandshakeException); } @Test - public void e2eFailoverOnSocketTimeoutException() throws Exception { - runE2EFailoverScenario(new SocketTimeoutException("read timed out")); - } - - private void runE2EFailoverScenario(IOException firstFailure) throws Exception { - AbstractClient client = newTC3Client(); - TransportStub transport = new TransportStub(); - OkHttpClient http = newE2EClient(client, transport); + public void testAggregatedFailureWhenPrimaryIsBreakerSkip() throws Exception { + // Pre-open .cn and .com.cn — only .com reaches transport. Suppressed + // and primary both contain the breaker-skip placeholders. + CvmClient client = newCvm(); + EndpointFailoverInterceptor.FailoverState state = + new EndpointFailoverInterceptor.FailoverState(60_000); + EndpointFailoverInterceptor.STATE.put("cvm.tencentcloudapi.com", state); + tripBreaker(state.breakers[1]); + tripBreaker(state.breakers[2]); - transport.programFailure(firstFailure); - transport.programSuccess(200, "{\"Response\":{}}"); + TransportStub transport = installStub(client); + transport.programFailure(new UnknownHostException("com dns fail")); - Request req = newTC3Request("cvm.tencentcloudapi.com"); - Response resp = http.newCall(req).execute(); - assertEquals(200, resp.code()); - assertEquals("{\"Response\":{}}", resp.body().string()); + TencentCloudSDKException sdkEx = null; + try { + client.DescribeInstances(new DescribeInstancesRequest()); + fail("expected SDK exception"); + } catch (TencentCloudSDKException e) { + sdkEx = e; + } - assertEquals(2, transport.received.size()); + assertEquals(1, transport.received.size()); assertEquals("cvm.tencentcloudapi.com", transport.received.get(0).url().host()); - assertEquals("cvm.tencentcloudapi.cn", transport.received.get(1).url().host()); - } - // ---- Non-failover IOException must propagate without retry ---- + IOException primary = unwrapToIOException(sdkEx); + // Last attempt = .com.cn breaker skip → no cause. + assertNull(primary.getCause()); + assertTrue(primary.getMessage().contains("cvm.tencentcloudapi.com.cn")); + assertTrue(primary.getMessage().contains("circuit breaker open")); + + Throwable[] suppressed = primary.getSuppressed(); + assertEquals(2, suppressed.length); + assertTrue(suppressed[0].getCause() instanceof UnknownHostException); + assertEquals("com dns fail", suppressed[0].getCause().getMessage()); + assertNull(suppressed[1].getCause()); + assertTrue(suppressed[1].getMessage().contains("cvm.tencentcloudapi.cn")); + } @Test - public void e2eGenericIOExceptionPropagatesWithoutFailover() throws Exception { - AbstractClient client = newTC3Client(); - TransportStub transport = new TransportStub(); - OkHttpClient http = newE2EClient(client, transport); + public void testFailoverDoesNotPolluteNextRequestAttemptFailures() throws Exception { + CvmClient client = newCvm(); + TransportStub transport = installStub(client); - IOException unrelated = new IOException("some unrelated I/O error"); - transport.programFailure(unrelated); + // Run 1: 1 fail + 1 success. + transport.programFailure(new UnknownHostException("run1 fail")); + transport.programOk(); + client.DescribeInstances(new DescribeInstancesRequest()); + transport.received.clear(); - Request req = newTC3Request("cvm.tencentcloudapi.com"); + // Run 2: all fail. Suppressed must contain ONLY run-2 failures. + transport.programFailure(new UnknownHostException("run2 com fail")); + transport.programFailure(new UnknownHostException("run2 cn fail")); + transport.programFailure(new UnknownHostException("run2 com.cn fail")); + TencentCloudSDKException sdkEx = null; try { - http.newCall(req).execute(); - fail("expected IOException to propagate"); - } catch (IOException e) { - assertEquals("some unrelated I/O error", e.getMessage()); + client.DescribeInstances(new DescribeInstancesRequest()); + fail("expected SDK exception"); + } catch (TencentCloudSDKException e) { + sdkEx = e; } - assertEquals("must not retry on non-failover IOException", 1, transport.received.size()); + IOException primary = unwrapToIOException(sdkEx); + assertEquals(2, primary.getSuppressed().length); + for (Throwable s : primary.getSuppressed()) { + assertFalse("must not leak run-1 failure into run-2 aggregation: " + s.getMessage(), + s.getMessage().contains("run1")); + } + assertTrue(primary.getMessage().contains("run2")); } - // ---- Response body / status / headers reach caller intact ---- + // ---- All breakers open: aggregated, zero transport hits ---- @Test - public void e2eResponseBodyAndStatusPreservedAfterFailover() throws Exception { - AbstractClient client = newTC3Client(); - TransportStub transport = new TransportStub(); - OkHttpClient http = newE2EClient(client, transport); + public void testAllBreakersOpenThrowsAggregatedWithoutProbing() throws Exception { + CvmClient client = newCvm(); + EndpointFailoverInterceptor.FailoverState state = + new EndpointFailoverInterceptor.FailoverState(60_000); + EndpointFailoverInterceptor.STATE.put("cvm.tencentcloudapi.com", state); + for (CircuitBreaker breaker : state.breakers) { + tripBreaker(breaker); + } - transport.programFailure(new UnknownHostException("dns miss")); - transport.programSuccess(202, "{\"Response\":{\"X\":42}}"); + TransportStub transport = installStub(client); + TencentCloudSDKException sdkEx = null; + try { + client.DescribeInstances(new DescribeInstancesRequest()); + fail("expected SDK exception when every breaker is open"); + } catch (TencentCloudSDKException e) { + sdkEx = e; + } - Request req = newTC3Request("cvm.tencentcloudapi.com"); - Response resp = http.newCall(req).execute(); - assertEquals(202, resp.code()); - assertEquals("{\"Response\":{\"X\":42}}", resp.body().string()); + IOException primary = unwrapToIOException(sdkEx); + assertTrue(primary.getMessage().contains("circuit breaker open")); + assertEquals(2, primary.getSuppressed().length); + for (Throwable s : primary.getSuppressed()) { + assertTrue(s.getMessage().contains("circuit breaker open")); + } + assertEquals("must not send any request when every breaker is open", + 0, transport.received.size()); } - // ---- 5xx server response is not a failover trigger ---- + // ---- Followup ordering: known-working TLD preferred; origin reprobed after cooldown ---- @Test - public void e2e5xxResponseDoesNotTriggerFailover() throws Exception { - AbstractClient client = newTC3Client(); - TransportStub transport = new TransportStub(); - OkHttpClient http = newE2EClient(client, transport); + public void testFollowupRequestUsesKnownWorkingTld() throws Exception { + CvmClient client = newCvm(); + TransportStub transport = installStub(client); - // Server reachable, returns 503 — interceptor must surface this, not retry. - transport.programSuccess(503, "service unavailable"); + transport.programFailure(new UnknownHostException("first dns miss")); + transport.programOk(); + client.DescribeInstances(new DescribeInstancesRequest()); + assertEquals(2, transport.received.size()); - Request req = newTC3Request("cvm.tencentcloudapi.com"); - Response resp = http.newCall(req).execute(); - assertEquals(503, resp.code()); + transport.received.clear(); + // Ample programmed outcomes — if the interceptor wrongly reprobes + // .com it will consume more than one and the assertion catches it. + transport.programOk(); + transport.programOk(); + client.DescribeInstances(new DescribeInstancesRequest()); assertEquals(1, transport.received.size()); + assertEquals("cvm.tencentcloudapi.cn", transport.received.get(0).url().host()); } - // ---- TC3 resign preserves body, content-type, signing scope ---- - @Test - public void e2eTC3ResignPreservesBodyAndContentType() throws Exception { - AbstractClient client = newTC3Client(); - TransportStub transport = new TransportStub(); - OkHttpClient http = newE2EClient(client, transport); + public void testFollowupRequestReprobesOriginalTldAfterCooldown() throws Exception { + CvmClient client = newCvm(); + TransportStub transport = installStub(client); - transport.programFailure(new UnknownHostException("dns miss")); - transport.programSuccess(200, "{}"); + transport.programFailure(new UnknownHostException("first dns miss")); + transport.programOk(); + client.DescribeInstances(new DescribeInstancesRequest()); - String payload = "{\"Limit\":10,\"Offset\":0,\"Filters\":[\"a\",\"b\"]}"; - Request req = newTC3RequestWithBody("cvm.tencentcloudapi.com", payload); - http.newCall(req).execute(); + EndpointFailoverInterceptor.FailoverState state = + EndpointFailoverInterceptor.STATE.get("cvm.tencentcloudapi.com"); + assertNotNull(state); + state.originProbeAfterMs = 0; // simulate cooldown elapsed - Request resigned = transport.received.get(1); - assertEquals("cvm.tencentcloudapi.cn", resigned.url().host()); - assertEquals("application/json; charset=utf-8", resigned.header("Content-Type")); - // Authorization must be regenerated with the new host bound into the signed scope. - String origAuth = transport.received.get(0).header("Authorization"); - String newAuth = resigned.header("Authorization"); - assertNotNull(newAuth); - assertNotEquals(origAuth, newAuth); - assertTrue("resigned auth must be TC3", newAuth.startsWith("TC3-HMAC-SHA256 ")); - assertTrue("scope must include new service host segment", - newAuth.contains("/cvm/tc3_request")); - // Body must round-trip byte-identical through resign. - assertEquals(payload, bodyAsString(resigned)); + transport.received.clear(); + transport.programOk(); + transport.programOk(); + client.DescribeInstances(new DescribeInstancesRequest()); + assertEquals(1, transport.received.size()); + assertEquals("cvm.tencentcloudapi.com", transport.received.get(0).url().host()); + assertEquals("origin probe must clear cooldown after a successful reprobe", + -1, state.originProbeAfterMs); } - // ---- X-TC-Token rotation visible to resigned request ---- + // ---- Resigned request must use rotated SecretId/Key ---- @Test - public void e2eResignReflectsRotatedToken() throws Exception { - AbstractClient client = newTC3Client(); - client.setCredential(new Credential("AKIDTEST", "SKTEST", "tok-v1")); - TransportStub transport = new TransportStub(); - OkHttpClient http = newE2EClient(client, transport); - + public void testResignPicksUpRotatedCredential() throws Exception { + final CvmClient client = newCvm(); + TransportStub transport = installStub(client); transport.programFailure(new UnknownHostException("dns miss")); - transport.programSuccess(200, "{}"); - - // Build a TC3 request that already carries the v1 token. - Request req = newTC3Request("cvm.tencentcloudapi.com") - .newBuilder() - .header("X-TC-Token", "tok-v1") - .build(); - - // Rotate the token between original sign and resign. - client.setCredential(new Credential("AKIDTEST", "SKTEST", "tok-v2")); + transport.programOk(); + + // Swap creds mid-flight (between original sign and resign). + installInterceptorBefore(client, new Interceptor() { + private boolean swapped = false; + @Override public Response intercept(Chain chain) throws IOException { + if (!swapped) { + swapped = true; + client.setCredential(new Credential("AKIDNEW", "SKNEW")); + } + return chain.proceed(chain.request()); + } + }); - http.newCall(req).execute(); - assertEquals("tok-v2", transport.received.get(1).header("X-TC-Token")); + client.DescribeInstances(new DescribeInstancesRequest()); + assertEquals(2, transport.received.size()); + // First request signed with old creds, resign with new ones. + assertTrue(transport.received.get(0).header("Authorization").contains("Credential=AKIDTEST/")); + assertTrue(transport.received.get(1).header("Authorization").contains("Credential=AKIDNEW/")); } - @Test - public void e2eResignDropsTokenWhenCleared() throws Exception { - AbstractClient client = newTC3Client(); - client.setCredential(new Credential("AKIDTEST", "SKTEST", "tok-v1")); - TransportStub transport = new TransportStub(); - OkHttpClient http = newE2EClient(client, transport); - - transport.programFailure(new UnknownHostException("dns miss")); - transport.programSuccess(200, "{}"); - - Request req = newTC3Request("cvm.tencentcloudapi.com") - .newBuilder() - .header("X-TC-Token", "tok-v1") - .build(); + // ================================================================= + // Helpers + // ================================================================= - client.setCredential(new Credential("AKIDTEST", "SKTEST")); // no token - - http.newCall(req).execute(); - assertNull("token must be removed from resigned request when credential drops it", - transport.received.get(1).header("X-TC-Token")); + private static CvmClient newCvm() { + return newCvm(new ClientProfile()); } - // ---- Hmac (V1) resign preserves all query params; Signature replaced exactly once ---- - - @Test - public void e2eHmacResignPreservesQueryParams() throws Exception { - AbstractClient client = newHmacClient(); - TransportStub transport = new TransportStub(); - OkHttpClient http = newE2EClient(client, transport); - - transport.programFailure(new UnknownHostException("dns miss")); - transport.programSuccess(200, "{}"); - - Request req = newHmacRequest("cvm.tencentcloudapi.com"); - http.newCall(req).execute(); - - Request resigned = transport.received.get(1); - assertEquals("cvm.tencentcloudapi.cn", resigned.url().host()); - assertEquals("TestAction", resigned.url().queryParameter("Action")); - assertEquals("2020-01-01", resigned.url().queryParameter("Version")); - assertEquals("ap-guangzhou", resigned.url().queryParameter("Region")); - assertEquals("AKIDTEST", resigned.url().queryParameter("SecretId")); - assertEquals("12345", resigned.url().queryParameter("Nonce")); - assertEquals("HmacSHA256", resigned.url().queryParameter("SignatureMethod")); - // Must still have a Signature, must differ from the original. - String newSig = resigned.url().queryParameter("Signature"); - assertNotNull(newSig); - assertNotEquals("deadbeefdeadbeef", newSig); - // Old "Signature=deadbeefdeadbeef" must NOT survive as a duplicate query - // param (resigner must drop it before signing, not append alongside). - List sigValues = resigned.url().queryParameterValues("Signature"); - assertEquals("must have exactly one Signature param", 1, sigValues.size()); + private static CvmClient newCvm(ClientProfile profile) { + return new CvmClient( + new Credential("AKIDTEST", "SKTEST"), + "ap-guangzhou", + profile); } - // ---- All breakers Open: surface aggregated IOException, do not probe ---- - - @Test - public void e2eAllBreakersOpenThrowsAggregatedWithoutProbing() throws Exception { - AbstractClient client = newTC3Client(); - TransportStub transport = new TransportStub(); - OkHttpClient http = newE2EClient(client, transport); + /** + * Reaches into the CvmClient's HttpConnection and rebuilds its OkHttpClient + * with {@code stub} appended as the terminal interceptor, so all in-flight + * traffic is short-circuited to the stub instead of hitting the network. + * Returns the stub for scripting. + */ + private static TransportStub installStub(AbstractClient client) { + TransportStub stub = new TransportStub(); + OkHttpClient orig = grabOkHttpClient(client); + setOkHttpClient(client, orig.newBuilder().addInterceptor(stub).build()); + return stub; + } - // Force every breaker into Open state directly. - EndpointFailoverInterceptor.FailoverState state = - new EndpointFailoverInterceptor.FailoverState(60_000); - EndpointFailoverInterceptor.STATE.put("cvm.tencentcloudapi.com", state); - for (CircuitBreaker breaker : state.breakers) { - for (int i = 0; i < 6; i++) { - CircuitBreaker.Token t = breaker.allow(); - if (t.allowed) { - t.report(false); - } - } + /** + * Adds an interceptor BEFORE the existing chain so it sees the request + * before the failover interceptor. Used for mid-flight credential rotation + * scenarios where the test needs to mutate state between attempts. + */ + private static void installInterceptorBefore(AbstractClient client, Interceptor it) { + OkHttpClient orig = grabOkHttpClient(client); + OkHttpClient.Builder b = new OkHttpClient.Builder() + .connectTimeout(orig.connectTimeoutMillis(), java.util.concurrent.TimeUnit.MILLISECONDS) + .readTimeout(orig.readTimeoutMillis(), java.util.concurrent.TimeUnit.MILLISECONDS) + .writeTimeout(orig.writeTimeoutMillis(), java.util.concurrent.TimeUnit.MILLISECONDS) + .addInterceptor(it); + for (Interceptor existing : orig.interceptors()) { + b.addInterceptor(existing); } + setOkHttpClient(client, b.build()); + } - IOException thrown = null; + private static OkHttpClient grabOkHttpClient(AbstractClient client) { try { - http.newCall(newTC3Request("cvm.tencentcloudapi.com")).execute(); - fail("expected IOException when every breaker is open"); - } catch (IOException e) { - thrown = e; - } - // Primary message names the last skipped host. - assertTrue("primary message must mention breaker skip, got: " + thrown.getMessage(), - thrown.getMessage().contains("circuit breaker open")); - // Two suppressed entries — one per other TLD. - assertEquals(2, thrown.getSuppressed().length); - for (Throwable s : thrown.getSuppressed()) { - assertTrue("suppressed must mention breaker skip, got: " + s.getMessage(), - s.getMessage().contains("circuit breaker open")); + Field f = AbstractClient.class.getDeclaredField("httpConnection"); + f.setAccessible(true); + HttpConnection conn = (HttpConnection) f.get(client); + return (OkHttpClient) conn.getHttpClient(); + } catch (ReflectiveOperationException e) { + throw new RuntimeException(e); } - assertEquals("must not send any request when every breaker is open", - 0, transport.received.size()); } - // ---- Sustained failure trips the breaker (subsequent attempts skip the host) ---- - - @Test - public void e2eRepeatedDnsFailureTripsBreakerOnOriginTld() throws Exception { - AbstractClient client = newTC3Client(); - TransportStub transport = new TransportStub(); - OkHttpClient http = newE2EClient(client, transport); - - // Drive 5 requests where .com always fails DNS and .cn always succeeds. - // The .cn success path does not touch the .com breaker, so .com - // accumulates 5 failures with 100% fail-rate (failures>=maxFailNum=5 - // && failPercentage>=0.75) and trips Open. Subsequent requests must - // skip .com on the first attempt. Force origin reprobe each loop so - // every iteration actually hits .com first (otherwise `.cn`-prefer - // ordering kicks in after the first success). - EndpointFailoverInterceptor.FailoverState state = null; - for (int i = 0; i < 5; i++) { - transport.programFailure(new UnknownHostException("dns miss")); - transport.programSuccess(200, "{}"); - http.newCall(newTC3Request("cvm.tencentcloudapi.com")).execute(); - if (state == null) { - state = EndpointFailoverInterceptor.STATE.get("cvm.tencentcloudapi.com"); - assertNotNull(state); + private static void setOkHttpClient(AbstractClient client, OkHttpClient http) { + try { + Field f = AbstractClient.class.getDeclaredField("httpConnection"); + f.setAccessible(true); + HttpConnection conn = (HttpConnection) f.get(client); + conn.setHttpClient(http); + } catch (ReflectiveOperationException e) { + throw new RuntimeException(e); + } + } + + private static void tripBreaker(CircuitBreaker breaker) { + // CircuitBreaker.Setting.maxFailNum=5, maxFailPercentage=0.75 by + // default — 6 consecutive failures guarantee Open state. + for (int i = 0; i < 6; i++) { + CircuitBreaker.Token t = breaker.allow(); + if (t.allowed) { + t.report(false); } - state.originProbeAfterMs = 0; } - - // Next request: .com breaker is Open, interceptor must skip .com on - // the first attempt and go straight to .cn. - transport.received.clear(); - transport.programSuccess(200, "{}"); - Response resp = http.newCall(newTC3Request("cvm.tencentcloudapi.com")).execute(); - assertEquals(200, resp.code()); - assertEquals("breaker should short-circuit .com without sending it to transport", - 1, transport.received.size()); - assertEquals("cvm.tencentcloudapi.cn", transport.received.get(0).url().host()); - } - - // ---- Original signature/host must NOT be the one that hit the wire on resign ---- - - @Test - public void e2eOriginalRequestNotSentTwice() throws Exception { - AbstractClient client = newTC3Client(); - TransportStub transport = new TransportStub(); - OkHttpClient http = newE2EClient(client, transport); - - transport.programFailure(new UnknownHostException("dns miss")); - transport.programSuccess(200, "{}"); - - Request req = newTC3Request("cvm.tencentcloudapi.com"); - http.newCall(req).execute(); - - Request first = transport.received.get(0); - Request second = transport.received.get(1); - assertNotEquals("hosts must differ between attempts", - first.url().host(), second.url().host()); - assertNotEquals("Authorization must be resigned for new host", - first.header("Authorization"), second.header("Authorization")); - assertEquals("Host header must track the URL host on resign", - second.url().host(), second.header("Host")); - } - - // ---- E2E helpers ---- - - private static Request newTC3RequestWithBody(String host, String body) { - return new Request.Builder() - .url("https://" + host + "/") - .post(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), - body.getBytes(StandardCharsets.UTF_8))) - .header("Content-Type", "application/json; charset=utf-8") - .header("Host", host) - .header("Authorization", - "TC3-HMAC-SHA256 Credential=AKIDTEST/2024-01-01/cvm/tc3_request," - + " SignedHeaders=content-type;host, Signature=deadbeef") - .header("X-TC-Action", "TestAction") - .header("X-TC-Timestamp", "1700000000") - .header("X-TC-Version", "2020-01-01") - .header("X-TC-RequestClient", "SDK_JAVA_TEST") - .header("X-TC-Region", "ap-guangzhou") - .build(); } /** - * Builds an {@link OkHttpClient} whose interceptor chain is: - * EndpointFailoverInterceptor → TransportStub. The transport stub plays - * the role of the network so each test runs offline yet exercises the - * real OkHttp pipeline (unlike {@link RecordingChain}). + * SDK wraps the failover IOException as the cause of a TencentCloudSDKException. + * Walk one level down to get the IOException that carries primary message + * and suppressed entries. */ - private static OkHttpClient newE2EClient(AbstractClient client, TransportStub transport) { - return new OkHttpClient.Builder() - .addInterceptor(new EndpointFailoverInterceptor(client)) - .addInterceptor(transport) - .build(); + private static IOException unwrapToIOException(TencentCloudSDKException e) { + Throwable cause = e.getCause(); + assertNotNull("SDK exception must wrap an IOException, got null cause", cause); + assertTrue("expected IOException cause, got " + cause.getClass().getName(), + cause instanceof IOException); + return (IOException) cause; } - private static String bodyAsString(Request req) throws IOException { + private static byte[] bodyBytes(Request req) throws IOException { if (req.body() == null) { - return ""; + return new byte[0]; } okio.Buffer buf = new okio.Buffer(); req.body().writeTo(buf); - return buf.readUtf8(); + return buf.readByteArray(); + } + + /** Mid-flight token rotator used by token-rotation tests. Swaps once. */ + private static final class AtomicTokenSwapper implements Interceptor { + private final AbstractClient client; + private final String newToken; + private boolean swapped = false; + + AtomicTokenSwapper(AbstractClient client, String newToken) { + this.client = client; + this.newToken = newToken; + } + + @Override + public Response intercept(Chain chain) throws IOException { + if (!swapped) { + swapped = true; + Credential cur = client.getCredential(); + client.setCredential(newToken == null + ? new Credential(cur.getSecretId(), cur.getSecretKey()) + : new Credential(cur.getSecretId(), cur.getSecretKey(), newToken)); + } + return chain.proceed(chain.request()); + } } /** * Terminal interceptor that replaces the network. Tests script a queue of - * {@link IOException} (failure) / {@link Response} (success) outcomes; each - * proceed call consumes one entry. Records every request that reaches it - * so tests can assert host / header / body content per attempt. + * outcomes (IOException / Response). Records every request that reaches it. */ private static final class TransportStub implements Interceptor { final List received = new ArrayList(); @@ -1238,7 +806,16 @@ void programFailure(IOException e) { programmed.add(e); } - void programSuccess(int code, String body) { + /** Returns a minimal valid Tencent Cloud JSON envelope. */ + void programOk() { + programJsonOk("{\"Response\":{\"RequestId\":\"req-ok\"}}"); + } + + void programJsonOk(String json) { + programmed.add(new ProgrammedResponse(200, json)); + } + + void programResponse(int code, String body) { programmed.add(new ProgrammedResponse(code, body)); } From a82ee27cb810cca90d5e0da2c10ffb78de4b2734 Mon Sep 17 00:00:00 2001 From: sesky4 Date: Tue, 28 Apr 2026 18:55:07 +0800 Subject: [PATCH 11/11] Update EndpointFailoverInterceptorTest.java --- .../EndpointFailoverInterceptorTest.java | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) diff --git a/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java b/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java index 31eef4e23d..8919ceb419 100644 --- a/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java +++ b/src/test/java/com/tencentcloudapi/common/EndpointFailoverInterceptorTest.java @@ -595,6 +595,158 @@ public void testAllBreakersOpenThrowsAggregatedWithoutProbing() throws Exception 0, transport.received.size()); } + // ---- Breaker lifecycle: real traffic drives Closed → Open → HalfOpen → Closed ---- + + @Test + public void testBreakerOpensAfterSustainedRealFailure() throws Exception { + // Drive the .com breaker entirely through the public API: 5 attempts + // where .com always fails DNS and .cn always succeeds. .cn never + // touches .com's breaker, so .com accumulates 5/5 failures (≥maxFailNum=5, + // 100%≥maxFailPercentage=0.75) and trips Open. After that, the next + // request must skip .com without sending it to transport. + CvmClient client = newCvm(); + TransportStub transport = installStub(client); + + for (int i = 0; i < 5; i++) { + transport.programFailure(new UnknownHostException("real fail " + i)); + transport.programOk(); + client.DescribeInstances(new DescribeInstancesRequest()); + // Force origin reprobe so .com is hit again next loop. + EndpointFailoverInterceptor.FailoverState s = + EndpointFailoverInterceptor.STATE.get("cvm.tencentcloudapi.com"); + s.originProbeAfterMs = 0; + } + assertEquals(10, transport.received.size()); + + // Sanity: state exists, breaker[0] (.com) is Open. + EndpointFailoverInterceptor.FailoverState state = + EndpointFailoverInterceptor.STATE.get("cvm.tencentcloudapi.com"); + assertNotNull(state); + assertFalse(".com breaker should be Open after 5/5 failures", + state.breakers[0].allow().allowed); + + // Next request: .com short-circuited, goes straight to .cn. + transport.received.clear(); + transport.programOk(); + // Force origin reprobe again — irrelevant here because breaker is + // Open and short-circuits regardless of probe ordering. + state.originProbeAfterMs = 0; + client.DescribeInstances(new DescribeInstancesRequest()); + assertEquals("Open breaker must short-circuit .com without transport hit", + 1, transport.received.size()); + assertEquals("cvm.tencentcloudapi.cn", transport.received.get(0).url().host()); + } + + @Test + public void testBreakerTransitionsOpenToHalfOpenAfterCooldown() throws Exception { + // Pre-place a FailoverState with a *short* breaker timeout so we don't + // have to sleep 60 s. Trip its .com breaker Open, wait for cooldown, + // then verify the next attempt is allowed (HalfOpen) and reaches + // transport against .com again. + long shortTimeoutMs = 100; + EndpointFailoverInterceptor.FailoverState state = + new EndpointFailoverInterceptor.FailoverState(shortTimeoutMs); + EndpointFailoverInterceptor.STATE.put("cvm.tencentcloudapi.com", state); + tripBreaker(state.breakers[0]); + assertFalse("breaker should be Open immediately after trip", + state.breakers[0].allow().allowed); + + // Wait past cooldown — Open → HalfOpen on next allow(). + Thread.sleep(shortTimeoutMs + 50); + CircuitBreaker.Token probeToken = state.breakers[0].allow(); + assertTrue("breaker should permit a probe (HalfOpen) after cooldown elapses", + probeToken.allowed); + // Don't report — leave HalfOpen for the next test scenario; here we + // only care that the cooldown transition worked. + } + + @Test + public void testBreakerReClosesAfterHalfOpenSuccessAndStaysClosed() throws Exception { + // Full lifecycle through the public API: + // Closed → Open (sustained failure) + // Open → HalfOpen (cooldown elapses) + // HalfOpen → Closed (probe succeeds; default maxRequests=0 means + // one success closes the breaker) + // After that the .com breaker should permit unlimited traffic. + long shortTimeoutMs = 100; + EndpointFailoverInterceptor.FailoverState state = + new EndpointFailoverInterceptor.FailoverState(shortTimeoutMs); + EndpointFailoverInterceptor.STATE.put("cvm.tencentcloudapi.com", state); + + CvmClient client = newCvm(); + TransportStub transport = installStub(client); + + // Open .com via direct breaker manipulation (faster than 5 real loops). + tripBreaker(state.breakers[0]); + assertFalse(state.breakers[0].allow().allowed); + + // Wait past cooldown to permit HalfOpen probe. + Thread.sleep(shortTimeoutMs + 50); + + // Force origin reprobe so .com is the first candidate; respond OK. + // candidates() puts .com first, breaker is HalfOpen → permits probe → + // success reports to breaker → Closed. + state.originProbeAfterMs = 0; + transport.programOk(); + client.DescribeInstances(new DescribeInstancesRequest()); + assertEquals(1, transport.received.size()); + assertEquals("cvm.tencentcloudapi.com", transport.received.get(0).url().host()); + + // Breaker must be Closed now — multiple back-to-back allow() calls + // should all succeed without short-circuiting. + for (int i = 0; i < 10; i++) { + assertTrue("breaker should be Closed after HalfOpen success, attempt " + i, + state.breakers[0].allow().allowed); + } + + // End-to-end: a fresh request should reach transport on .com without + // failover, since the breaker is Closed and origin probe was cleared. + transport.received.clear(); + transport.programOk(); + client.DescribeInstances(new DescribeInstancesRequest()); + assertEquals(1, transport.received.size()); + assertEquals("cvm.tencentcloudapi.com", transport.received.get(0).url().host()); + } + + @Test + public void testBreakerReOpensWhenHalfOpenProbeFails() throws Exception { + // Open → HalfOpen → Open: a single failure during HalfOpen reverts + // to Open. The interceptor must surface that failure and on the next + // request short-circuit again. + long shortTimeoutMs = 100; + EndpointFailoverInterceptor.FailoverState state = + new EndpointFailoverInterceptor.FailoverState(shortTimeoutMs); + EndpointFailoverInterceptor.STATE.put("cvm.tencentcloudapi.com", state); + + CvmClient client = newCvm(); + TransportStub transport = installStub(client); + + tripBreaker(state.breakers[0]); + Thread.sleep(shortTimeoutMs + 50); + + // HalfOpen probe: .com first, fails again → re-Open. .cn succeeds. + state.originProbeAfterMs = 0; + transport.programFailure(new UnknownHostException("still down")); + transport.programOk(); + client.DescribeInstances(new DescribeInstancesRequest()); + assertEquals(2, transport.received.size()); + assertEquals("cvm.tencentcloudapi.com", transport.received.get(0).url().host()); + assertEquals("cvm.tencentcloudapi.cn", transport.received.get(1).url().host()); + + // .com breaker must be Open again immediately (not waiting for the + // failure threshold — HalfOpen reverts to Open on a single failure). + assertFalse("HalfOpen failure must re-Open the breaker", + state.breakers[0].allow().allowed); + + // Next request short-circuits .com again. + transport.received.clear(); + state.originProbeAfterMs = 0; + transport.programOk(); + client.DescribeInstances(new DescribeInstancesRequest()); + assertEquals(1, transport.received.size()); + assertEquals("cvm.tencentcloudapi.cn", transport.received.get(0).url().host()); + } + // ---- Followup ordering: known-working TLD preferred; origin reprobed after cooldown ---- @Test