diff --git a/.github/workflows/smoke.yaml b/.github/workflows/smoke.yaml new file mode 100644 index 0000000..3b90e11 --- /dev/null +++ b/.github/workflows/smoke.yaml @@ -0,0 +1,54 @@ +name: smoke + +on: + schedule: + - cron: "17 8 * * *" + workflow_dispatch: + pull_request: + types: + - opened + - synchronize + - reopened + - labeled + +jobs: + smoke: + if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'smoke') + runs-on: ubuntu-latest + timeout-minutes: 90 + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Install smoke tooling + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y ca-certificates curl iproute2 make openssl tar + + mkdir -p "${RUNNER_TEMP}/bin" + echo "${RUNNER_TEMP}/bin" >> "${GITHUB_PATH}" + + curl -fsSL -o "${RUNNER_TEMP}/bin/kind" "https://kind.sigs.k8s.io/dl/v0.29.0/kind-linux-amd64" + chmod +x "${RUNNER_TEMP}/bin/kind" + + curl -fsSL -o "${RUNNER_TEMP}/bin/kubectl" "https://dl.k8s.io/release/v1.33.2/bin/linux/amd64/kubectl" + chmod +x "${RUNNER_TEMP}/bin/kubectl" + + curl -fsSL "https://get.helm.sh/helm-v3.18.3-linux-amd64.tar.gz" | tar -xz -C "${RUNNER_TEMP}" + mv "${RUNNER_TEMP}/linux-amd64/helm" "${RUNNER_TEMP}/bin/helm" + + curl -fsSL -o "${RUNNER_TEMP}/bin/yq" "https://github.com/mikefarah/yq/releases/download/v4.45.4/yq_linux_amd64" + chmod +x "${RUNNER_TEMP}/bin/yq" + + curl -fsSL -o "${RUNNER_TEMP}/bin/devspace" "https://github.com/loft-sh/devspace/releases/download/v6.3.15/devspace-linux-amd64" + chmod +x "${RUNNER_TEMP}/bin/devspace" + + - name: Run smoke + run: make smoke diff --git a/Makefile b/Makefile index 743e872..b8b9f2d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help install-precommit setup-dev lint test test-install clean +.PHONY: help install-precommit setup-dev lint test test-install test-e2e smoke clean help: ## Display this help message @echo "Available targets:" @@ -56,6 +56,11 @@ test: ## Run all tests test-install: ## Run live DevSpace install diagnostics CGO_ENABLED=1 go test -count=1 -v -timeout 5m ./tests/install +test-e2e: ## Run full ephemeral-cluster DevSpace e2e validation + go run ./tests/e2e/cmd/smoke + +smoke: test-e2e ## Run expensive smoke validation + clean: ## Clean up generated files @echo "Cleaning up..." rm -rf charts/*/charts/ diff --git a/README.md b/README.md index 4a5f616..e924945 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,34 @@ dns-sd -q ns.dns.kube **NOTE**: on macOS, do not rely on `dig` for testing DNS resolution. +### Ephemeral Smoke Validation + +Run the full advertised deploy path against a throwaway local cluster: + +```bash +make smoke +``` + +This creates an ephemeral `kind` cluster with an isolated kubeconfig, runs `devspace deploy`, runs the +live install diagnostics, and deletes the cluster. The direct e2e target is also available: + +```bash +make test-e2e +``` + +Useful local overrides: + +```bash +E2E_CLUSTER_NAME=my-smoke E2E_KEEP_CLUSTER=1 make smoke +E2E_DEVSPACE_ARGS="--profile o11y-grafana" make smoke +E2E_TIMEOUT=30m E2E_READY_TIMEOUT=10m make smoke +``` + +`E2E_CLUSTER_PROVIDER=kind` is the current default and only implemented provider. `vind` is reserved +as a future provider name. Timeout knobs use Go duration syntax and include `E2E_TIMEOUT`, +`E2E_CLUSTER_CREATE_WAIT`, `E2E_CLEANUP_TIMEOUT`, `E2E_READY_TIMEOUT`, +`E2E_READY_REPORT_INTERVAL`, `E2E_DIAGNOSTIC_TIMEOUT`, and `E2E_TEST_TIMEOUT`. + ## Available Profiles | Profile | Description | Components | diff --git a/devspace.yaml b/devspace.yaml index e141bf0..0615b26 100644 --- a/devspace.yaml +++ b/devspace.yaml @@ -55,7 +55,7 @@ profiles: description: Network, Gateways, Ingress activation: - vars: - DEVSPACE_CONTEXT: "^(kind|docker-desktop|minikube|rancher-desktop|microk8s)$" + DEVSPACE_CONTEXT: "^(kind(?:-.+)?|docker-desktop|minikube|rancher-desktop|microk8s)$" merge: deployments: metallb: @@ -178,7 +178,7 @@ profiles: description: Local DNS integration for development activation: - vars: - DEVSPACE_CONTEXT: "^(kind|docker-desktop|minikube|rancher-desktop|microk8s)$" + DEVSPACE_CONTEXT: "^(kind(?:-.+)?|docker-desktop|minikube|rancher-desktop|microk8s)$" merge: deployments: etcd: @@ -223,7 +223,7 @@ profiles: description: Certificate Management activation: - vars: - DEVSPACE_CONTEXT: "^(kind|docker-desktop|minikube|rancher-desktop|microk8s)$" + DEVSPACE_CONTEXT: "^(kind(?:-.+)?|docker-desktop|minikube|rancher-desktop|microk8s)$" merge: deployments: cert-manager: @@ -281,7 +281,7 @@ profiles: description: Auxiliary services activation: - vars: - DEVSPACE_CONTEXT: "^(kind|docker-desktop|minikube|rancher-desktop|microk8s)$" + DEVSPACE_CONTEXT: "^(kind(?:-.+)?|docker-desktop|minikube|rancher-desktop|microk8s)$" merge: deployments: reloader: @@ -312,7 +312,7 @@ profiles: description: Cluster observability with Prometheus metrics and lightweight tracing activation: - vars: - DEVSPACE_CONTEXT: "^(kind|docker-desktop|minikube|rancher-desktop|microk8s)$" + DEVSPACE_CONTEXT: "^(kind(?:-.+)?|docker-desktop|minikube|rancher-desktop|microk8s)$" merge: deployments: prometheus: @@ -501,12 +501,18 @@ hooks: silent: true events: ["before:deploy:metallb"] - - name: update-cluster-dns-hook + - name: update-cluster-dns-hook-darwin os: darwin command: devspace run update-cluster-dns events: ["after:deploy:cert-chain"] - - name: cert-chain-hook + - name: update-cluster-dns-hook-linux + os: linux + command: devspace run update-cluster-dns-linux + events: ["after:deploy:cert-chain"] + + - name: cert-chain-hook-darwin + os: darwin command: | while ! devspace run import-root-ca; do echo >&2 "I: Waiting for root CA to be available..." @@ -514,6 +520,15 @@ hooks: done events: ["after:deploy:cert-chain"] + - name: cert-chain-hook-linux + os: linux + command: | + while ! devspace run import-root-ca-linux; do + echo >&2 "I: Waiting for root CA to be available..." + sleep 5 + done + events: ["after:deploy:cert-chain"] + - name: wait-for-trust-manager-hook wait: running: true @@ -597,6 +612,21 @@ commands: quit EOF + update-cluster-dns-linux: + description: Make external cluster DNS available on a Linux host + section: network + command: | + DNS_IP="${DOCKER_CIDR_PREFIX}.254" + echo >&2 "I: Updating systemd-resolved DNS settings..." + LINK="$(ip route get "${DNS_IP}" | awk '{for (i = 1; i <= NF; i++) if ($i == "dev") {print $(i + 1); exit}}')" + if [ -z "${LINK}" ]; then + echo >&2 "E: Could not determine Linux route interface for ${DNS_IP}" + exit 1 + fi + sudo resolvectl dns "${LINK}" "${DNS_IP}" + sudo resolvectl domain "${LINK}" "~kube" + sudo resolvectl flush-caches + reset-cluster-dns: description: Reset DNS service section: network @@ -608,6 +638,20 @@ commands: quit EOF + reset-cluster-dns-linux: + description: Reset Linux systemd-resolved cluster DNS service + section: network + command: | + DNS_IP="${DOCKER_CIDR_PREFIX}.254" + echo >&2 "I: Resetting systemd-resolved DNS settings..." + LINK="$(ip route get "${DNS_IP}" | awk '{for (i = 1; i <= NF; i++) if ($i == "dev") {print $(i + 1); exit}}')" + if [ -z "${LINK}" ]; then + echo >&2 "E: Could not determine Linux route interface for ${DNS_IP}" + exit 1 + fi + sudo resolvectl revert "${LINK}" + sudo resolvectl flush-caches + import-root-ca: description: Import the root CA certificate of the cluster section: network @@ -630,6 +674,28 @@ commands: echo >&2 "I: Root CA certificate imported successfully to system keychain." + import-root-ca-linux: + description: Import the root CA certificate of the cluster into the Linux trust store + section: network + command: | + CERTFILE=$(mktemp -t cluster-root-ca.XXXXXX.crt) + # Ensure cleanup on exit + trap 'rm -f "${CERTFILE}"' EXIT + + echo >&2 "I: Extracting Root CA certificate..." + kubectl get secret -n istio-ingress cluster-root-ca-secret -o jsonpath='{.data.tls\.crt}' | base64 -d > "${CERTFILE}" + if [ ! -s "${CERTFILE}" ]; then + echo >&2 "E: Failed to extract certificate or certificate is empty" + exit 1 + fi + openssl x509 -in "${CERTFILE}" -subject -issuer -dates -noout + + echo >&2 "I: Adding certificate to Linux trust store..." + sudo install -m 0644 "${CERTFILE}" /usr/local/share/ca-certificates/devspace-starter-pack-cluster-root-ca.crt + sudo update-ca-certificates + + echo >&2 "I: Root CA certificate imported successfully to Linux trust store." + port-forward-otel: description: Forward the local OpenTelemetry Collector OTLP ports section: observability diff --git a/tests/e2e/cmd/smoke/main.go b/tests/e2e/cmd/smoke/main.go new file mode 100644 index 0000000..2c5db69 --- /dev/null +++ b/tests/e2e/cmd/smoke/main.go @@ -0,0 +1,320 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +const ( + defaultProvider = "kind" + defaultTimeout = 20 * time.Minute + defaultClusterCreateWait = 5 * time.Minute + defaultCleanupTimeout = 5 * time.Minute + defaultReadyTimeout = 5 * time.Minute + defaultReadyReportInterval = time.Minute + defaultDiagnosticTimeout = 45 * time.Second + defaultTestTimeout = 5 * time.Minute +) + +type config struct { + providerName string + clusterName string + keepCluster bool + devspaceArgs []string + testArgs []string + + timeout time.Duration + clusterCreateWait time.Duration + cleanupTimeout time.Duration + readyTimeout time.Duration + readyReportInterval time.Duration + diagnosticTimeout time.Duration + testTimeout time.Duration +} + +func main() { + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "smoke failed: %v\n", err) + os.Exit(1) + } +} + +func run() error { + cfg := loadConfig() + provider, err := newProvider(cfg.providerName) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), cfg.timeout) + defer cancel() + + tempDir, err := os.MkdirTemp("", "devspace-starter-pack-smoke-*") + if err != nil { + return fmt.Errorf("create temp directory: %w", err) + } + removeTempDir := true + defer func() { + if removeTempDir { + _ = os.RemoveAll(tempDir) + } + }() + + kubeconfig := filepath.Join(tempDir, "kubeconfig") + fmt.Printf("smoke: provider=%s cluster=%s kubeconfig=%s\n", cfg.providerName, cfg.clusterName, kubeconfig) + + if err := provider.preflight(ctx); err != nil { + return err + } + if err := provider.create(ctx, cfg.clusterName, kubeconfig, cfg.clusterCreateWait); err != nil { + return err + } + + clusterCreated := true + defer func() { + if !clusterCreated { + return + } + if cfg.keepCluster { + removeTempDir = false + fmt.Printf("smoke: preserving cluster %q and kubeconfig %s because E2E_KEEP_CLUSTER=1\n", cfg.clusterName, kubeconfig) + return + } + cleanupCtx, cancel := context.WithTimeout(context.Background(), cfg.cleanupTimeout) + defer cancel() + if err := provider.delete(cleanupCtx, cfg.clusterName, kubeconfig); err != nil { + fmt.Fprintf(os.Stderr, "smoke: failed to delete cluster %q: %v\n", cfg.clusterName, err) + } + }() + + env := append(os.Environ(), "KUBECONFIG="+kubeconfig) + if err := assertContext(ctx, env, provider.contextName(cfg.clusterName)); err != nil { + return err + } + + if err := runStep(ctx, env, "devspace", append([]string{"deploy"}, cfg.devspaceArgs...)...); err != nil { + collectDiagnostics(env, cfg.diagnosticTimeout) + return err + } + + if err := waitForPodsReady(ctx, env, cfg); err != nil { + collectDiagnostics(env, cfg.diagnosticTimeout) + return err + } + + testEnv := append(env, "CGO_ENABLED=1") + goArgs := append([]string{"test", "-count=1", "-v", "-timeout", cfg.testTimeout.String()}, cfg.testArgs...) + goArgs = append(goArgs, "./tests/install") + if err := runStep(ctx, testEnv, "go", goArgs...); err != nil { + collectDiagnostics(env, cfg.diagnosticTimeout) + return err + } + + fmt.Println("smoke: completed successfully") + return nil +} + +func waitForPodsReady(ctx context.Context, env []string, cfg config) error { + waitCtx, cancel := context.WithTimeout(ctx, cfg.readyTimeout) + defer cancel() + + ticker := time.NewTicker(cfg.readyReportInterval) + defer ticker.Stop() + + for { + if notReady, ready, err := podReadiness(waitCtx, env); err != nil { + return err + } else if ready { + fmt.Println("smoke: all pods are Ready") + return nil + } else if len(notReady) == 0 { + fmt.Println("smoke: waiting for pods to be created") + } + + select { + case <-waitCtx.Done(): + fmt.Fprintln(os.Stderr, "smoke: timed out waiting for all pods to become Ready") + printPendingPodDiagnostics(env, cfg.diagnosticTimeout) + return waitCtx.Err() + case <-ticker.C: + printPodStatus(env) + } + } +} + +func podReadiness(ctx context.Context, env []string) ([]string, bool, error) { + output, err := commandOutput(ctx, env, "kubectl", "get", "pods", "--all-namespaces", "--no-headers") + if err != nil { + return nil, false, err + } + output = strings.TrimSpace(output) + if output == "" { + return nil, false, nil + } + var notReady []string + for _, line := range strings.Split(output, "\n") { + fields := strings.Fields(line) + if len(fields) < 4 { + continue + } + if fields[3] == "Completed" || fields[3] == "Succeeded" { + continue + } + readyParts := strings.SplitN(fields[2], "/", 2) + if len(readyParts) != 2 || readyParts[0] != readyParts[1] || fields[3] != "Running" { + notReady = append(notReady, line) + } + } + return notReady, len(notReady) == 0, nil +} + +func printPodStatus(env []string) { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + notReady, ready, err := podReadiness(ctx, env) + if err != nil { + fmt.Fprintf(os.Stderr, "smoke: failed to list pod readiness: %v\n", err) + return + } + if ready { + fmt.Println("smoke: all pods are Ready") + return + } + if len(notReady) == 0 { + fmt.Println("smoke: waiting for pods to be created") + return + } + fmt.Printf("smoke: waiting for %d non-ready pod(s)\n", len(notReady)) + for _, line := range notReady { + fmt.Printf("smoke: non-ready pod: %s\n", line) + } +} + +func printPendingPodDiagnostics(env []string, timeout time.Duration) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + _ = runStep(ctx, env, "kubectl", "get", "pods", "--all-namespaces", "-o", "wide") + _ = runStep(ctx, env, "kubectl", "get", "events", "--all-namespaces", "--sort-by=.lastTimestamp") + _ = runStep(ctx, env, "kubectl", "describe", "pods", "--all-namespaces") +} + +func loadConfig() config { + providerName := getenvDefault("E2E_CLUSTER_PROVIDER", defaultProvider) + clusterName := os.Getenv("E2E_CLUSTER_NAME") + if clusterName == "" { + clusterName = fmt.Sprintf("devspace-smoke-%d", time.Now().Unix()) + } + + return config{ + providerName: providerName, + clusterName: clusterName, + keepCluster: os.Getenv("E2E_KEEP_CLUSTER") == "1", + devspaceArgs: splitArgs(os.Getenv("E2E_DEVSPACE_ARGS")), + testArgs: splitArgs(os.Getenv("E2E_TEST_ARGS")), + + timeout: durationFromEnv("E2E_TIMEOUT", defaultTimeout), + clusterCreateWait: durationFromEnv("E2E_CLUSTER_CREATE_WAIT", defaultClusterCreateWait), + cleanupTimeout: durationFromEnv("E2E_CLEANUP_TIMEOUT", defaultCleanupTimeout), + readyTimeout: durationFromEnv("E2E_READY_TIMEOUT", defaultReadyTimeout), + readyReportInterval: durationFromEnv("E2E_READY_REPORT_INTERVAL", defaultReadyReportInterval), + diagnosticTimeout: durationFromEnv("E2E_DIAGNOSTIC_TIMEOUT", defaultDiagnosticTimeout), + testTimeout: durationFromEnv("E2E_TEST_TIMEOUT", defaultTestTimeout), + } +} + +func assertContext(ctx context.Context, env []string, expected string) error { + output, err := commandOutput(ctx, env, "kubectl", "config", "current-context") + if err != nil { + return err + } + got := strings.TrimSpace(output) + if got != expected { + return fmt.Errorf("kubectl current-context is %q, expected ephemeral context %q", got, expected) + } + return runStep(ctx, env, "kubectl", "cluster-info") +} + +func collectDiagnostics(env []string, timeout time.Duration) { + fmt.Fprintln(os.Stderr, "smoke: collecting Kubernetes diagnostics") + diagnosticSteps := [][]string{ + {"kubectl", "get", "pods", "--all-namespaces", "-o", "wide"}, + {"kubectl", "get", "svc", "--all-namespaces", "-o", "wide"}, + {"kubectl", "get", "events", "--all-namespaces", "--sort-by=.lastTimestamp"}, + {"kubectl", "describe", "pods", "--all-namespaces"}, + {"helm", "list", "--all-namespaces"}, + } + + for _, step := range diagnosticSteps { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + _ = runStep(ctx, env, step[0], step[1:]...) + cancel() + } +} + +func requireTool(ctx context.Context, tool string) error { + _, err := exec.LookPath(tool) + if err != nil { + return fmt.Errorf("required tool %q is not available in PATH", tool) + } + return nil +} + +func runStep(ctx context.Context, env []string, name string, args ...string) error { + fmt.Printf("smoke: running %s %s\n", name, strings.Join(args, " ")) + cmd := exec.CommandContext(ctx, name, args...) + cmd.Env = env + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + if ctx.Err() != nil { + return fmt.Errorf("%s %s timed out: %w", name, strings.Join(args, " "), ctx.Err()) + } + return fmt.Errorf("%s %s failed: %w", name, strings.Join(args, " "), err) + } + return nil +} + +func commandOutput(ctx context.Context, env []string, name string, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Env = env + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return stdout.String(), fmt.Errorf("%s %s failed: %w\nstdout:\n%s\nstderr:\n%s", name, strings.Join(args, " "), err, stdout.String(), stderr.String()) + } + return stdout.String(), nil +} + +func splitArgs(value string) []string { + return strings.Fields(value) +} + +func getenvDefault(name, fallback string) string { + value := os.Getenv(name) + if value == "" { + return fallback + } + return value +} + +func durationFromEnv(name string, fallback time.Duration) time.Duration { + value := os.Getenv(name) + if value == "" { + return fallback + } + duration, err := time.ParseDuration(value) + if err != nil { + fmt.Fprintf(os.Stderr, "smoke: invalid duration %s=%q, using %s\n", name, value, fallback) + return fallback + } + return duration +} diff --git a/tests/e2e/cmd/smoke/provider.go b/tests/e2e/cmd/smoke/provider.go new file mode 100644 index 0000000..49e1fdd --- /dev/null +++ b/tests/e2e/cmd/smoke/provider.go @@ -0,0 +1,25 @@ +package main + +import ( + "context" + "fmt" + "time" +) + +type provider interface { + preflight(context.Context) error + create(context.Context, string, string, time.Duration) error + delete(context.Context, string, string) error + contextName(string) string +} + +func newProvider(name string) (provider, error) { + switch name { + case "kind": + return kindProvider{}, nil + case "vind": + return vindProvider{}, nil + default: + return nil, fmt.Errorf("unsupported E2E_CLUSTER_PROVIDER %q", name) + } +} diff --git a/tests/e2e/cmd/smoke/provider_kind.go b/tests/e2e/cmd/smoke/provider_kind.go new file mode 100644 index 0000000..056a063 --- /dev/null +++ b/tests/e2e/cmd/smoke/provider_kind.go @@ -0,0 +1,28 @@ +package main + +import ( + "context" + "os" + "time" +) + +type kindProvider struct{} + +func (kindProvider) preflight(ctx context.Context) error { + if err := requireTool(ctx, "kind"); err != nil { + return err + } + return requireTool(ctx, "kubectl") +} + +func (kindProvider) create(ctx context.Context, clusterName, kubeconfig string, wait time.Duration) error { + return runStep(ctx, os.Environ(), "kind", "create", "cluster", "--name", clusterName, "--kubeconfig", kubeconfig, "--wait", wait.String()) +} + +func (kindProvider) delete(ctx context.Context, clusterName, kubeconfig string) error { + return runStep(ctx, os.Environ(), "kind", "delete", "cluster", "--name", clusterName, "--kubeconfig", kubeconfig) +} + +func (kindProvider) contextName(clusterName string) string { + return "kind-" + clusterName +} diff --git a/tests/e2e/cmd/smoke/provider_vind.go b/tests/e2e/cmd/smoke/provider_vind.go new file mode 100644 index 0000000..76055e0 --- /dev/null +++ b/tests/e2e/cmd/smoke/provider_vind.go @@ -0,0 +1,25 @@ +package main + +import ( + "context" + "errors" + "time" +) + +type vindProvider struct{} + +func (vindProvider) preflight(context.Context) error { + return errors.New("E2E_CLUSTER_PROVIDER=vind is reserved for future support and is not implemented yet") +} + +func (vindProvider) create(context.Context, string, string, time.Duration) error { + return errors.New("E2E_CLUSTER_PROVIDER=vind is reserved for future support and is not implemented yet") +} + +func (vindProvider) delete(context.Context, string, string) error { + return nil +} + +func (vindProvider) contextName(clusterName string) string { + return clusterName +} diff --git a/tests/install/host_linux.go b/tests/install/host_linux.go new file mode 100644 index 0000000..35af0c6 --- /dev/null +++ b/tests/install/host_linux.go @@ -0,0 +1,77 @@ +//go:build linux + +package install_test + +import ( + "crypto/x509" + "strings" + "testing" +) + +func hostRequiredTools() []string { + return []string{"resolvectl"} +} + +func assertHostDNS(t *testing.T, name, expectedIP string) { + t.Helper() + + assertResolvedRouteOnlyDomain(t, "kube", expectedIP) + assertDefaultResolverResolves(t, name, expectedIP) +} + +func assertRootCAImported(t *testing.T) { + t.Helper() + + cert := rootCACertificate(t) + pool, err := x509.SystemCertPool() + if err != nil { + t.Fatalf("failed to load system certificate pool: %v", err) + } + if _, err := cert.Verify(x509.VerifyOptions{Roots: pool}); err != nil { + t.Fatalf("system certificate pool does not trust current cluster root CA: %v", err) + } +} + +func assertOptionalHTTPSRoute(t *testing.T) { + t.Helper() + + if !httpbinRouteInstalled(t) { + t.Skip("optional httpbin route is not installed") + } + assertHTTPSGet(t, "HTTPS route", "https://httpbin.int.kube/get") +} + +func assertOptionalTracingRoute(t *testing.T) { + t.Helper() + + if !httpRouteInstalled(t, "observability", "jaeger") { + t.Fatal("observability/jaeger HTTPRoute is not installed") + } + assertHTTPSGet(t, "Jaeger HTTPS route", "https://jaeger.int.kube/") +} + +func assertOptionalGrafanaRoute(t *testing.T) { + t.Helper() + + if !httpRouteInstalled(t, "observability", "grafana") { + t.Fatal("observability/grafana HTTPRoute is not installed") + } + assertHTTPSGet(t, "Grafana HTTPS route", "https://grafana.int.kube/login") +} + +func assertResolvedRouteOnlyDomain(t *testing.T, domain, expectedIP string) { + t.Helper() + + output := runCommand(t, "resolvectl", "status") + blocks := strings.Split(output, "\nLink ") + for _, block := range blocks { + if !strings.Contains(block, "DNS Servers: "+expectedIP) && !strings.Contains(block, "\n "+expectedIP) { + continue + } + if strings.Contains(block, "DNS Domain: ~"+domain) || strings.Contains(block, "DNS Domain: ") && strings.Contains(block, "~"+domain) { + return + } + } + + t.Fatalf("systemd-resolved does not route ~%s to DNS server %s:\n%s", domain, expectedIP, output) +} diff --git a/tests/install/host_macos.go b/tests/install/host_macos.go index 4d56581..dbbdb53 100644 --- a/tests/install/host_macos.go +++ b/tests/install/host_macos.go @@ -3,13 +3,8 @@ package install_test import ( - "crypto/tls" - "crypto/x509" - "io" - "net/http" "strings" "testing" - "time" ) func hostRequiredTools() []string { @@ -47,26 +42,7 @@ func assertOptionalHTTPSRoute(t *testing.T) { t.Fatal("HTTPS route validation on macOS requires CGO_ENABLED=1 so Go uses the system resolver for .kube names") } - pool, err := x509.SystemCertPool() - if err != nil { - t.Fatalf("failed to load system cert pool: %v", err) - } - - client := &http.Client{ - Timeout: 15 * time.Second, - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, - }, - } - resp, err := client.Get("https://httpbin.int.kube/get") - if err != nil { - t.Fatalf("HTTPS request through gateway failed: %v", err) - } - defer resp.Body.Close() - body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - t.Fatalf("HTTPS route returned %s: %s", resp.Status, strings.TrimSpace(string(body))) - } + assertHTTPSGet(t, "HTTPS route", "https://httpbin.int.kube/get") } func assertOptionalTracingRoute(t *testing.T) { @@ -79,26 +55,7 @@ func assertOptionalTracingRoute(t *testing.T) { t.Fatal("Jaeger route validation on macOS requires CGO_ENABLED=1 so Go uses the system resolver for .kube names") } - pool, err := x509.SystemCertPool() - if err != nil { - t.Fatalf("failed to load system cert pool: %v", err) - } - - client := &http.Client{ - Timeout: 15 * time.Second, - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, - }, - } - resp, err := client.Get("https://jaeger.int.kube/") - if err != nil { - t.Fatalf("Jaeger HTTPS route through gateway failed: %v", err) - } - defer resp.Body.Close() - body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - t.Fatalf("Jaeger HTTPS route returned %s: %s", resp.Status, strings.TrimSpace(string(body))) - } + assertHTTPSGet(t, "Jaeger HTTPS route", "https://jaeger.int.kube/") } func assertOptionalGrafanaRoute(t *testing.T) { @@ -111,26 +68,7 @@ func assertOptionalGrafanaRoute(t *testing.T) { t.Fatal("Grafana route validation on macOS requires CGO_ENABLED=1 so Go uses the system resolver for .kube names") } - pool, err := x509.SystemCertPool() - if err != nil { - t.Fatalf("failed to load system cert pool: %v", err) - } - - client := &http.Client{ - Timeout: 15 * time.Second, - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, - }, - } - resp, err := client.Get("https://grafana.int.kube/login") - if err != nil { - t.Fatalf("Grafana HTTPS route through gateway failed: %v", err) - } - defer resp.Body.Close() - body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - t.Fatalf("Grafana HTTPS route returned %s: %s", resp.Status, strings.TrimSpace(string(body))) - } + assertHTTPSGet(t, "Grafana HTTPS route", "https://grafana.int.kube/login") } func assertScutilResolver(t *testing.T, domain, expectedIP string) { @@ -160,20 +98,3 @@ func assertScutilResolver(t *testing.T, domain, expectedIP string) { t.Fatalf("macOS scutil --dns does not contain a supplemental resolver for %q", domain) } - -func httpbinRouteInstalled(t *testing.T) bool { - t.Helper() - - return httpRouteInstalled(t, "httpbin", "http") -} - -func httpRouteInstalled(t *testing.T, namespace, name string) bool { - t.Helper() - - output, err := runCommandE(defaultCommandTimeout, "kubectl", "get", "httproute", name, "-n", namespace, "-o", "name") - if err != nil { - warningf(t, "HTTPRoute %s/%s check skipped: %v", namespace, name, err) - return false - } - return strings.TrimSpace(output) == "httproute.gateway.networking.k8s.io/"+name || strings.TrimSpace(output) == "httproute/"+name -} diff --git a/tests/install/host_other.go b/tests/install/host_other.go index 2060428..608f857 100644 --- a/tests/install/host_other.go +++ b/tests/install/host_other.go @@ -1,4 +1,4 @@ -//go:build !darwin +//go:build !darwin && !linux package install_test diff --git a/tests/install/host_routes.go b/tests/install/host_routes.go new file mode 100644 index 0000000..e8d91bd --- /dev/null +++ b/tests/install/host_routes.go @@ -0,0 +1,53 @@ +package install_test + +import ( + "crypto/tls" + "crypto/x509" + "io" + "net/http" + "strings" + "testing" + "time" +) + +func assertHTTPSGet(t *testing.T, description, url string) { + t.Helper() + + pool, err := x509.SystemCertPool() + if err != nil { + t.Fatalf("failed to load system cert pool: %v", err) + } + + client := &http.Client{ + Timeout: 15 * time.Second, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, + }, + } + resp, err := client.Get(url) + if err != nil { + t.Fatalf("%s through gateway failed: %v", description, err) + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + t.Fatalf("%s returned %s: %s", description, resp.Status, strings.TrimSpace(string(body))) + } +} + +func httpbinRouteInstalled(t *testing.T) bool { + t.Helper() + + return httpRouteInstalled(t, "httpbin", "http") +} + +func httpRouteInstalled(t *testing.T, namespace, name string) bool { + t.Helper() + + output, err := runCommandE(defaultCommandTimeout, "kubectl", "get", "httproute", name, "-n", namespace, "-o", "name") + if err != nil { + warningf(t, "HTTPRoute %s/%s check skipped: %v", namespace, name, err) + return false + } + return strings.TrimSpace(output) == "httproute.gateway.networking.k8s.io/"+name || strings.TrimSpace(output) == "httproute/"+name +} diff --git a/tests/install/kubernetes.go b/tests/install/kubernetes.go index 3882aa2..fcfefb5 100644 --- a/tests/install/kubernetes.go +++ b/tests/install/kubernetes.go @@ -2,6 +2,7 @@ package install_test import ( "crypto/sha256" + "crypto/x509" "encoding/base64" "encoding/hex" "encoding/pem" @@ -554,6 +555,29 @@ func assertCertManagerResourcesReady(t *testing.T) { func rootCAFingerprint(t *testing.T) string { t.Helper() + cert := rootCACertificate(t) + sum := sha256.Sum256(cert.Raw) + return strings.ToUpper(hex.EncodeToString(sum[:])) +} + +func rootCACertificate(t *testing.T) *x509.Certificate { + t.Helper() + + pemBytes := rootCAPEM(t) + block, _ := pem.Decode(pemBytes) + if block == nil { + t.Fatal("root CA certificate data does not contain a PEM block") + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("failed to parse root CA certificate: %v", err) + } + return cert +} + +func rootCAPEM(t *testing.T) []byte { + t.Helper() + secret := kubectlJSON[secret](t, "get", "secret", rootCASecret, "-n", "cert-manager") encoded := secret.Data["tls.crt"] if encoded == "" { @@ -564,13 +588,7 @@ func rootCAFingerprint(t *testing.T) string { if err != nil { t.Fatalf("failed to base64-decode root CA certificate: %v", err) } - block, _ := pem.Decode(pemBytes) - if block == nil { - t.Fatal("root CA certificate data does not contain a PEM block") - } - - sum := sha256.Sum256(block.Bytes) - return strings.ToUpper(hex.EncodeToString(sum[:])) + return pemBytes } func desiredReplicas(replicas *int) int {