diff --git a/.github/workflows/update-generated-reference-docs.yml b/.github/workflows/update-generated-reference-docs.yml new file mode 100644 index 000000000..91545335f --- /dev/null +++ b/.github/workflows/update-generated-reference-docs.yml @@ -0,0 +1,76 @@ +name: Update generated reference docs + +on: + schedule: + - cron: "17 8 * * *" + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +concurrency: + group: update-generated-reference-docs + cancel-in-progress: false + +jobs: + update-generated-reference-docs: + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.EXTERNAL_REPO_TOKEN || github.token }} + GH_TOKEN: ${{ secrets.EXTERNAL_REPO_TOKEN || github.token }} + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Install Nix + uses: cachix/install-nix-action@v31 + + - name: Install direnv + run: | + sudo apt-get update + sudo apt-get install -y direnv + + - name: Trust direnv + run: direnv allow . + + - name: Update source pins + id: update + run: | + mkdir -p .internal + direnv exec . python3 scripts/update_reference_doc_sources.py \ + --apply \ + --summary .internal/reference-doc-update-summary.json + + - name: Regenerate reference docs + if: steps.update.outputs.has_updates == 'true' + run: direnv exec . npm run generate:all-reference-docs + + - name: Run tests + if: steps.update.outputs.has_updates == 'true' + run: direnv exec . python3 -m pytest tests + + - name: Check whitespace + if: steps.update.outputs.has_updates == 'true' + run: git diff --check + + - name: Upload update artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: generated-reference-doc-update + if-no-files-found: ignore + path: | + .internal/reference-doc-update-summary.json + .internal/generated/x2mdx/**/*.json + + - name: Create pull request + if: steps.update.outputs.has_updates == 'true' + uses: peter-evans/create-pull-request@v8 + with: + token: ${{ secrets.DOCS_PR_TOKEN }} + branch: generated-reference-docs/update + base: main + title: Update generated reference docs from upstream releases + commit-message: Update generated reference docs from upstream releases + body: ${{ steps.update.outputs.pr_body }} diff --git a/config/mintlify-openapi/splice-openapi/source-artifacts.json b/config/mintlify-openapi/splice-openapi/source-artifacts.json index aef63c0d6..9a95e46bc 100644 --- a/config/mintlify-openapi/splice-openapi/source-artifacts.json +++ b/config/mintlify-openapi/splice-openapi/source-artifacts.json @@ -4,6 +4,9 @@ "tag_regex": "^v(?P0\\.[0-9]+\\.[0-9]+)$", "min_version": "0.5.10", "publish_version": "0.5.18", + "versions": [ + "0.5.18" + ], "asset_template": "{version}_openapi.tar.gz", "nav_dropdown": "API Reference", "top_level_group_label": "Splice APIs", diff --git a/config/x2mdx/grpc-ledger-api-reference/source-artifacts.json b/config/x2mdx/grpc-ledger-api-reference/source-artifacts.json index d9bf59929..bfb73270d 100644 --- a/config/x2mdx/grpc-ledger-api-reference/source-artifacts.json +++ b/config/x2mdx/grpc-ledger-api-reference/source-artifacts.json @@ -7,6 +7,16 @@ "web_url": "https://github.com/DACH-NY/canton" }, "min_version": "3.4.4", + "versions": [ + "3.4.4", + "3.4.5", + "3.4.6", + "3.4.7", + "3.4.8", + "3.4.9", + "3.4.10", + "3.4.11" + ], "metadata_path": "config/x2mdx/protobuf-history/metadata.json", "package_prefixes": [ "com.daml.ledger.api.v2" diff --git a/config/x2mdx/protobuf-history/source-artifacts.json b/config/x2mdx/protobuf-history/source-artifacts.json index 1c7365e45..addabe98b 100644 --- a/config/x2mdx/protobuf-history/source-artifacts.json +++ b/config/x2mdx/protobuf-history/source-artifacts.json @@ -8,5 +8,18 @@ }, "min_version": "3.2.0", "excluded_versions": ["3.4.1"], + "versions": [ + "3.4.0", + "3.4.2", + "3.4.3", + "3.4.4", + "3.4.5", + "3.4.6", + "3.4.7", + "3.4.8", + "3.4.9", + "3.4.10", + "3.4.11" + ], "metadata_path": "config/x2mdx/protobuf-history/metadata.json" } diff --git a/config/x2mdx/reference-update-policy.json b/config/x2mdx/reference-update-policy.json new file mode 100644 index 000000000..01538d983 --- /dev/null +++ b/config/x2mdx/reference-update-policy.json @@ -0,0 +1,71 @@ +{ + "surfaces": { + "daml-standard-library": { + "kind": "git-tag-versions", + "config_path": "config/x2mdx/daml-standard-library/source-artifacts.json", + "repository": "digital-asset/daml", + "remote": "https://github.com/digital-asset/daml.git", + "tag_regex": "^v(?P\\d+\\.\\d+\\.\\d+)$", + "keep": 3, + "publish_latest": true + }, + "json-ledger-api": { + "kind": "canton-release-bundle-minors", + "config_paths": [ + "config/x2mdx/ledger-api/source-artifacts.json", + "config/x2mdx/ledger-api-asyncapi/source-artifacts.json" + ], + "repository": "DACH-NY/canton", + "remote": "https://github.com/DACH-NY/canton.git", + "tag_regex": "^v(?P\\d+\\.\\d+\\.\\d+)$", + "keep_minor_count": 2, + "validate_release_url": true + }, + "ledger-bindings": { + "kind": "maven-javadoc-artifact", + "config_path": "config/x2mdx/ledger-bindings/source-artifacts.json", + "keep": 4, + "validate_javadoc": true + }, + "protobuf-history": { + "kind": "canton-tag-versions", + "config_path": "config/x2mdx/protobuf-history/source-artifacts.json", + "repository": "DACH-NY/canton", + "remote": "https://github.com/DACH-NY/canton.git", + "tag_regex": "^v(?P\\d+\\.\\d+\\.\\d+)$", + "validate_release_url": true + }, + "grpc-ledger-api-reference": { + "kind": "canton-tag-versions", + "config_path": "config/x2mdx/grpc-ledger-api-reference/source-artifacts.json", + "repository": "DACH-NY/canton", + "remote": "https://github.com/DACH-NY/canton.git", + "tag_regex": "^v(?P\\d+\\.\\d+\\.\\d+)$", + "validate_release_url": true + }, + "typescript-bindings": { + "kind": "npm-packages", + "config_path": "config/x2mdx/typescript-bindings/source-artifacts.json", + "keep": 4, + "publish_latest": true + }, + "wallet-gateway-openrpc": { + "kind": "github-release-versions", + "config_path": "config/x2mdx/wallet-gateway-openrpc/source-artifacts.json", + "repository": "hyperledger-labs/splice-wallet-kernel", + "tag_prefix": "@canton-network/wallet-gateway-remote@", + "tag_regex": "^@canton-network/wallet-gateway-remote@(?P\\d+\\.\\d+\\.\\d+)$", + "keep": 2, + "publish_latest": true + }, + "splice-openapi": { + "kind": "github-release-asset-versions", + "config_path": "config/mintlify-openapi/splice-openapi/source-artifacts.json", + "repository": "digital-asset/decentralized-canton-sync", + "tag_regex": "^v(?P0\\.[0-9]+\\.[0-9]+)$", + "asset_template": "{version}_openapi.tar.gz", + "keep": 1, + "publish_latest": true + } + } +} diff --git a/config/x2mdx/wallet-gateway-openrpc/source-artifacts.json b/config/x2mdx/wallet-gateway-openrpc/source-artifacts.json index bafd8ccc0..30c6a8473 100644 --- a/config/x2mdx/wallet-gateway-openrpc/source-artifacts.json +++ b/config/x2mdx/wallet-gateway-openrpc/source-artifacts.json @@ -5,6 +5,10 @@ "tag_prefix": "@canton-network/wallet-gateway-remote@", "min_version": "0.24.0", "publish_version": "0.25.0", + "versions": [ + "0.24.0", + "0.25.0" + ], "specs": [ { "spec_id": "dapp-api", diff --git a/scripts/generate_canton_protobuf_history.py b/scripts/generate_canton_protobuf_history.py index c2e50465d..5c53c86ac 100644 --- a/scripts/generate_canton_protobuf_history.py +++ b/scripts/generate_canton_protobuf_history.py @@ -122,6 +122,15 @@ def load_excluded_versions(source_config: dict[str, Any]) -> set[str]: return set(configured) +def configured_versions(source_config: dict[str, Any]) -> set[str] | None: + configured = source_config.get("versions") + if configured is None: + return None + if not isinstance(configured, list) or not all(isinstance(item, str) and item for item in configured): + raise ValueError("Source config versions must be a list of non-empty strings") + return set(configured) + + def run(args: list[str], *, cwd: Path | None = None, capture: bool = False) -> str: kwargs: dict[str, Any] = { "cwd": str(cwd) if cwd else None, @@ -647,7 +656,7 @@ def main() -> int: if not isinstance(bundle_proto_dir, str) or not bundle_proto_dir: raise ValueError("Source config must define bundle_proto_dir") - include_versions = set(args.version) if args.version else None + include_versions = set(args.version) if args.version else configured_versions(source_config) excluded_versions = load_excluded_versions(source_config) if include_versions is not None: include_versions -= excluded_versions diff --git a/scripts/generate_grpc_ledger_api_reference.py b/scripts/generate_grpc_ledger_api_reference.py index 05fec4c81..816b622bf 100644 --- a/scripts/generate_grpc_ledger_api_reference.py +++ b/scripts/generate_grpc_ledger_api_reference.py @@ -92,6 +92,15 @@ def package_prefixes(source_config: dict[str, Any]) -> tuple[str, ...]: return tuple(configured) +def configured_versions(source_config: dict[str, Any]) -> set[str] | None: + configured = source_config.get("versions") + if configured is None: + return None + if not isinstance(configured, list) or not all(isinstance(item, str) and item for item in configured): + raise ValueError("Source config versions must be a list of non-empty strings") + return set(configured) + + def package_matches(package_name: str, *, prefixes: tuple[str, ...]) -> bool: return any(package_name.startswith(prefix) for prefix in prefixes) @@ -641,7 +650,7 @@ def main() -> int: args = parse_args() source_config = load_json(Path(args.source_config).resolve()) prefixes = package_prefixes(source_config) - include_versions = set(args.version) if args.version else None + include_versions = set(args.version) if args.version else configured_versions(source_config) min_version = args.min_version or source_config.get("min_version") or "0.0.0" if not isinstance(min_version, str): raise ValueError("min_version must be a string") diff --git a/scripts/generate_splice_mintlify_openapi.py b/scripts/generate_splice_mintlify_openapi.py index d6306885c..e19a6bfbc 100644 --- a/scripts/generate_splice_mintlify_openapi.py +++ b/scripts/generate_splice_mintlify_openapi.py @@ -37,12 +37,16 @@ def version_key(version: str) -> tuple[int, ...]: def github_json(url: str) -> Any: + headers = { + "Accept": "application/vnd.github+json", + "User-Agent": USER_AGENT, + } + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + if token: + headers["Authorization"] = f"Bearer {token}" request = urllib.request.Request( url, - headers={ - "Accept": "application/vnd.github+json", - "User-Agent": USER_AGENT, - }, + headers=headers, ) with urllib.request.urlopen(request, timeout=180) as response: return json.loads(response.read().decode("utf-8")) @@ -129,6 +133,15 @@ def selected_releases( return releases +def configured_versions(source_config: dict[str, Any]) -> set[str] | None: + configured = source_config.get("versions") + if configured is None: + return None + if not isinstance(configured, list) or not all(isinstance(item, str) and item for item in configured): + raise ValueError("Source config versions must be a list of non-empty strings") + return set(configured) + + def resolve_publish_release( *, source_config: dict[str, Any], @@ -540,7 +553,7 @@ def parse_args() -> argparse.Namespace: def main() -> int: args = parse_args() source_config = load_json(Path(args.source_config).resolve()) - include_versions = set(args.version) if args.version else None + include_versions = set(args.version) if args.version else configured_versions(source_config) releases = selected_releases(source_config=source_config, include_versions=include_versions) publish_release = resolve_publish_release( source_config=source_config, diff --git a/scripts/generate_wallet_gateway_openrpc_reference.py b/scripts/generate_wallet_gateway_openrpc_reference.py index b2f26d6f4..7625dbcd8 100644 --- a/scripts/generate_wallet_gateway_openrpc_reference.py +++ b/scripts/generate_wallet_gateway_openrpc_reference.py @@ -163,6 +163,15 @@ def stable_release_versions( return selected +def configured_versions(source_config: dict[str, Any]) -> set[str] | None: + configured = source_config.get("versions") + if configured is None: + return None + if not isinstance(configured, list) or not all(isinstance(item, str) and item for item in configured): + raise ValueError("Source config versions must be a list of non-empty strings") + return set(configured) + + def docs_json_page_ref(path: Path, docs_json_path: Path) -> str: relative = path.resolve().relative_to(docs_json_path.resolve().parent) if relative.suffix != ".mdx": @@ -364,7 +373,7 @@ def main() -> int: ensure_repo_direnv(repo_root=REPO_ROOT, script_path=Path(__file__).resolve(), argv=sys.argv[1:]) args = parse_args() source_config = load_json(Path(args.source_config).resolve()) - include_versions = set(args.version) if args.version else None + include_versions = set(args.version) if args.version else configured_versions(source_config) remote = str(source_config.get("remote") or "") release_repo = str(source_config.get("release_repo") or DEFAULT_RELEASE_REPO) tag_prefix = str(source_config.get("tag_prefix") or "") diff --git a/scripts/update_reference_doc_sources.py b/scripts/update_reference_doc_sources.py new file mode 100644 index 000000000..7e5f4ed82 --- /dev/null +++ b/scripts/update_reference_doc_sources.py @@ -0,0 +1,651 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import copy +import json +import os +import re +import subprocess +import sys +import urllib.error +import urllib.parse +import urllib.request +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable + +from docs_env import ensure_repo_direnv + + +REPO_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_POLICY = REPO_ROOT / "config" / "x2mdx" / "reference-update-policy.json" +DEFAULT_SUMMARY = REPO_ROOT / ".internal" / "reference-doc-update-summary.json" +USER_AGENT = "digital-asset-docs-reference-updater/1.0" + + +@dataclass(frozen=True) +class SurfaceResult: + name: str + config_paths: tuple[str, ...] + changed: bool + before: dict[str, Any] + after: dict[str, Any] + updates: tuple[str, ...] + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Update generated reference-doc source pins from upstream version sources." + ) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--check", action="store_true", help="Discover updates without modifying config files.") + mode.add_argument("--apply", action="store_true", help="Rewrite source config files when newer versions exist.") + parser.add_argument("--repo-root", default=str(REPO_ROOT)) + parser.add_argument("--policy", default=str(DEFAULT_POLICY)) + parser.add_argument("--summary", default=str(DEFAULT_SUMMARY)) + return parser.parse_args(argv) + + +def json_load(path: Path) -> dict[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"Expected JSON object: {path}") + return payload + + +def json_dump(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + +def version_key(version: str) -> tuple[int, ...]: + return tuple(int(part) for part in version.split(".")) + + +def numeric_version_prefix(version: str) -> str | None: + match = re.match(r"^(?P\d+\.\d+\.\d+)", version) + return match.group("version") if match else None + + +def major_minor(version: str) -> str: + parts = version.split(".") + if len(parts) < 2: + raise ValueError(f"Expected semantic version with major.minor: {version}") + return f"{parts[0]}.{parts[1]}" + + +def request_headers() -> dict[str, str]: + headers = { + "Accept": "application/vnd.github+json", + "User-Agent": USER_AGENT, + } + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + if token: + headers["Authorization"] = f"Bearer {token}" + return headers + + +def fetch_json_url(url: str) -> Any: + request = urllib.request.Request(url, headers=request_headers()) + with urllib.request.urlopen(request, timeout=180) as response: + return json.loads(response.read().decode("utf-8")) + + +def fetch_text_url(url: str) -> str: + request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(request, timeout=180) as response: + return response.read().decode("utf-8") + + +def head_url(url: str) -> bool: + request = urllib.request.Request(url, method="HEAD", headers={"User-Agent": USER_AGENT}) + try: + with urllib.request.urlopen(request, timeout=60) as response: + return 200 <= response.status < 400 + except urllib.error.HTTPError as exc: + if exc.code == 405: + get_request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(get_request, timeout=60) as response: + return 200 <= response.status < 400 + return False + + +def github_paginated(path: str) -> list[dict[str, Any]]: + output: list[dict[str, Any]] = [] + page = 1 + while True: + separator = "&" if "?" in path else "?" + payload = fetch_json_url(f"https://api.github.com/{path}{separator}per_page=100&page={page}") + if not isinstance(payload, list): + raise ValueError(f"Expected GitHub list payload for {path}") + output.extend(item for item in payload if isinstance(item, dict)) + if len(payload) < 100: + return output + page += 1 + + +def github_release_versions(repository: str, tag_regex: str, *, require_asset: str | None = None) -> list[str]: + matcher = re.compile(tag_regex) + versions: list[str] = [] + for release in github_paginated(f"repos/{repository}/releases"): + if release.get("draft") or release.get("prerelease"): + continue + tag_name = release.get("tag_name") + if not isinstance(tag_name, str): + continue + match = matcher.fullmatch(tag_name) + if match is None: + continue + version = match.groupdict().get("version") + if not version: + continue + if require_asset is not None: + assets = release.get("assets") + if not isinstance(assets, list): + continue + wanted = require_asset.format(version=version) + if not any(isinstance(asset, dict) and asset.get("name") == wanted for asset in assets): + continue + versions.append(version) + return sorted(set(versions), key=version_key) + + +def github_tag_versions(repository: str, tag_regex: str, group_name: str = "version") -> list[str]: + matcher = re.compile(tag_regex) + versions: list[str] = [] + for tag in github_paginated(f"repos/{repository}/tags"): + name = tag.get("name") + if not isinstance(name, str): + continue + match = matcher.fullmatch(name) + if match is None: + continue + version = match.groupdict().get(group_name) + if version: + versions.append(version) + return sorted(set(versions), key=version_key) + + +def git_remote_tag_versions(remote: str, tag_regex: str, group_name: str = "version") -> list[str]: + completed = subprocess.run( + ["git", "ls-remote", "--tags", remote], + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + matcher = re.compile(tag_regex) + versions: list[str] = [] + for line in completed.stdout.splitlines(): + ref = line.rsplit("\t", 1)[-1] + if ref.endswith("^{}"): + continue + tag_name = ref.removeprefix("refs/tags/") + match = matcher.fullmatch(tag_name) + if match is None: + continue + version = match.groupdict().get(group_name) + if version: + versions.append(version) + return sorted(set(versions), key=version_key) + + +def tag_versions(surface: dict[str, Any], group_name: str = "version") -> list[str]: + remote = surface.get("remote") + if isinstance(remote, str) and remote: + return git_remote_tag_versions(remote, str(surface["tag_regex"]), group_name=group_name) + return github_tag_versions(str(surface["repository"]), str(surface["tag_regex"]), group_name=group_name) + + +def latest_versions(versions: Iterable[str], keep: int) -> list[str]: + selected = sorted(set(versions), key=version_key) + if keep > 0: + selected = selected[-keep:] + return selected + + +def maven_metadata_versions(repo_base: str, group: str, artifact: str) -> list[str]: + group_path = group.replace(".", "/") + metadata_url = f"{repo_base.rstrip('/')}/{group_path}/{artifact}/maven-metadata.xml" + root = ET.fromstring(fetch_text_url(metadata_url)) + versions = [ + node.text.strip() + for node in root.findall("./versioning/versions/version") + if node.text and re.fullmatch(r"\d+\.\d+\.\d+", node.text.strip()) + ] + return sorted(set(versions), key=version_key) + + +def maven_javadoc_url(repo_base: str, group: str, artifact: str, version: str) -> str: + group_path = group.replace(".", "/") + return f"{repo_base.rstrip('/')}/{group_path}/{artifact}/{version}/{artifact}-{version}-javadoc.jar" + + +def npm_package_versions(package_name: str) -> list[str]: + encoded = urllib.parse.quote(package_name, safe="") + payload = fetch_json_url(f"https://registry.npmjs.org/{encoded}") + versions = payload.get("versions") if isinstance(payload, dict) else None + if not isinstance(versions, dict): + raise ValueError(f"Expected npm versions object for {package_name}") + return sorted( + (version for version in versions if re.fullmatch(r"\d+\.\d+\.\d+", version)), + key=version_key, + ) + + +def release_url(config: dict[str, Any], version: str, *, field: str = "version") -> str: + template = config.get("release_url_template") + if not isinstance(template, str) or not template: + raise ValueError("Config must define release_url_template") + return template.format(**{field: version, "version": version, "canton_version": version}) + + +def validate_urls(urls: Iterable[str]) -> None: + missing = [url for url in urls if not head_url(url)] + if missing: + formatted = "\n".join(f"- {url}" for url in missing) + raise RuntimeError(f"Candidate artifacts are not reachable:\n{formatted}") + + +def surface_payload(payload: dict[str, Any]) -> dict[str, Any]: + return copy.deepcopy(payload) + + +def summarize_change(label: str, before: Any, after: Any) -> str | None: + if before == after: + return None + return f"{label}: {before!r} -> {after!r}" + + +def update_github_release_versions( + *, + repo_root: Path, + name: str, + surface: dict[str, Any], +) -> SurfaceResult: + path = str(surface["config_path"]) + config_path = repo_root / path + before = json_load(config_path) + after = surface_payload(before) + versions = github_release_versions( + str(surface["repository"]), + str(surface["tag_regex"]), + ) + min_version = after.get("min_version") + if isinstance(min_version, str) and min_version: + versions = [version for version in versions if version_key(version) >= version_key(min_version)] + selected = latest_versions(versions, int(surface.get("keep") or 0)) + after["versions"] = selected + if surface.get("publish_latest") and selected: + after["publish_version"] = selected[-1] + updates = tuple( + item + for item in ( + summarize_change("versions", before.get("versions"), after.get("versions")), + summarize_change("publish_version", before.get("publish_version"), after.get("publish_version")), + ) + if item + ) + return SurfaceResult(name, (path,), before != after, before, after, updates) + + +def update_git_tag_versions( + *, + repo_root: Path, + name: str, + surface: dict[str, Any], +) -> SurfaceResult: + path = str(surface["config_path"]) + config_path = repo_root / path + before = json_load(config_path) + after = surface_payload(before) + versions = tag_versions(surface) + min_version = after.get("min_version") + if isinstance(min_version, str) and min_version: + versions = [version for version in versions if version_key(version) >= version_key(min_version)] + selected = latest_versions(versions, int(surface.get("keep") or 0)) + after["versions"] = selected + if surface.get("publish_latest") and selected: + after["publish_version"] = selected[-1] + updates = tuple( + item + for item in ( + summarize_change("versions", before.get("versions"), after.get("versions")), + summarize_change("publish_version", before.get("publish_version"), after.get("publish_version")), + ) + if item + ) + return SurfaceResult(name, (path,), before != after, before, after, updates) + + +def update_github_release_asset_versions( + *, + repo_root: Path, + name: str, + surface: dict[str, Any], +) -> SurfaceResult: + path = str(surface["config_path"]) + config_path = repo_root / path + before = json_load(config_path) + after = surface_payload(before) + versions = github_release_versions( + str(surface["repository"]), + str(surface["tag_regex"]), + require_asset=str(surface["asset_template"]), + ) + min_version = after.get("min_version") + if isinstance(min_version, str) and min_version: + versions = [version for version in versions if version_key(version) >= version_key(min_version)] + selected = latest_versions(versions, int(surface.get("keep") or 0)) + after["versions"] = selected + if surface.get("publish_latest") and selected: + after["publish_version"] = selected[-1] + updates = tuple( + item + for item in ( + summarize_change("versions", before.get("versions"), after.get("versions")), + summarize_change("publish_version", before.get("publish_version"), after.get("publish_version")), + ) + if item + ) + return SurfaceResult(name, (path,), before != after, before, after, updates) + + +def update_canton_tag_versions( + *, + repo_root: Path, + name: str, + surface: dict[str, Any], +) -> SurfaceResult: + path = str(surface["config_path"]) + config_path = repo_root / path + before = json_load(config_path) + after = surface_payload(before) + versions = tag_versions(surface) + min_version = after.get("min_version") + if isinstance(min_version, str) and min_version: + versions = [version for version in versions if version_key(version) >= version_key(min_version)] + excluded = after.get("excluded_versions") or [] + if isinstance(excluded, list): + excluded_set = {item for item in excluded if isinstance(item, str)} + versions = [version for version in versions if version not in excluded_set] + if surface.get("validate_release_url"): + validate_urls(release_url(after, version) for version in versions) + after["versions"] = versions + updates = tuple( + item + for item in (summarize_change("versions", before.get("versions"), after.get("versions")),) + if item + ) + return SurfaceResult(name, (path,), before != after, before, after, updates) + + +def latest_canton_minor_entries( + versions: list[str], + keep_minor_count: int, + current_entries: list[dict[str, str]], +) -> list[dict[str, str]]: + latest_by_minor: dict[str, str] = {} + for version in versions: + minor = major_minor(version) + current = latest_by_minor.get(minor) + if current is None or version_key(version) > version_key(current): + latest_by_minor[minor] = version + for entry in current_entries: + minor = entry.get("version") + current_canton_version = entry.get("canton_version") + if not isinstance(minor, str) or not isinstance(current_canton_version, str): + continue + current_numeric = numeric_version_prefix(current_canton_version) + candidate = latest_by_minor.get(minor) + if current_numeric is None: + latest_by_minor.setdefault(minor, current_canton_version) + elif candidate is None or version_key(current_numeric) > version_key(candidate): + latest_by_minor[minor] = current_canton_version + selected_minors = sorted(latest_by_minor, key=version_key)[-keep_minor_count:] + return [ + { + "version": minor, + "canton_version": latest_by_minor[minor], + } + for minor in selected_minors + ] + + +def update_canton_release_bundle_minors( + *, + repo_root: Path, + name: str, + surface: dict[str, Any], +) -> SurfaceResult: + config_paths = tuple(str(path) for path in surface["config_paths"]) + first_config = json_load(repo_root / config_paths[0]) + versions = tag_versions(surface, group_name="canton_version") + current_entries = first_config.get("versions") + if not isinstance(current_entries, list): + current_entries = [] + entries = latest_canton_minor_entries( + versions, + int(surface["keep_minor_count"]), + [entry for entry in current_entries if isinstance(entry, dict)], + ) + if surface.get("validate_release_url"): + validate_urls(release_url(first_config, entry["canton_version"], field="canton_version") for entry in entries) + publish_version = entries[-1]["version"] if entries else None + + before_combined: dict[str, Any] = {} + after_combined: dict[str, Any] = {} + changed = False + updates: list[str] = [] + for path in config_paths: + before = json_load(repo_root / path) + after = surface_payload(before) + after["versions"] = copy.deepcopy(entries) + if publish_version: + after["publish_version"] = publish_version + before_combined[path] = before + after_combined[path] = after + changed = changed or before != after + for item in ( + summarize_change(f"{path} versions", before.get("versions"), after.get("versions")), + summarize_change(f"{path} publish_version", before.get("publish_version"), after.get("publish_version")), + ): + if item: + updates.append(item) + return SurfaceResult(name, config_paths, changed, before_combined, after_combined, tuple(updates)) + + +def update_maven_javadoc_artifact( + *, + repo_root: Path, + name: str, + surface: dict[str, Any], +) -> SurfaceResult: + path = str(surface["config_path"]) + before = json_load(repo_root / path) + after = surface_payload(before) + repo_base = str(after.get("repo_base") or "https://repo1.maven.org/maven2") + keep = int(surface.get("keep") or 0) + artifacts = after.get("artifacts") + if not isinstance(artifacts, list): + raise ValueError(f"{path} must define artifacts list") + updates: list[str] = [] + for index, artifact in enumerate(artifacts): + if not isinstance(artifact, dict): + continue + group = artifact.get("group") + artifact_name = artifact.get("artifact") + if not isinstance(group, str) or not isinstance(artifact_name, str): + continue + versions = latest_versions(maven_metadata_versions(repo_base, group, artifact_name), keep) + if surface.get("validate_javadoc"): + validate_urls(maven_javadoc_url(repo_base, group, artifact_name, version) for version in versions) + old_versions = artifact.get("versions") + artifact["versions"] = versions + item = summarize_change(f"artifacts[{index}].versions", old_versions, versions) + if item: + updates.append(item) + return SurfaceResult(name, (path,), before != after, before, after, tuple(updates)) + + +def update_npm_packages( + *, + repo_root: Path, + name: str, + surface: dict[str, Any], +) -> SurfaceResult: + path = str(surface["config_path"]) + before = json_load(repo_root / path) + after = surface_payload(before) + keep = int(surface.get("keep") or 0) + packages = after.get("packages") + if not isinstance(packages, list): + raise ValueError(f"{path} must define packages list") + updates: list[str] = [] + for index, package in enumerate(packages): + if not isinstance(package, dict): + continue + package_name = package.get("package_name") + if not isinstance(package_name, str) or not package_name: + continue + versions = latest_versions(npm_package_versions(package_name), keep) + old_versions = package.get("versions") + old_publish_version = package.get("publish_version") + package["versions"] = versions + if surface.get("publish_latest") and versions: + package["publish_version"] = versions[-1] + for item in ( + summarize_change(f"packages[{index}].versions", old_versions, package.get("versions")), + summarize_change( + f"packages[{index}].publish_version", + old_publish_version, + package.get("publish_version"), + ), + ): + if item: + updates.append(item) + return SurfaceResult(name, (path,), before != after, before, after, tuple(updates)) + + +def surface_result(repo_root: Path, name: str, surface: dict[str, Any]) -> SurfaceResult: + kind = surface.get("kind") + if kind == "github-release-versions": + return update_github_release_versions(repo_root=repo_root, name=name, surface=surface) + if kind == "git-tag-versions": + return update_git_tag_versions(repo_root=repo_root, name=name, surface=surface) + if kind == "github-release-asset-versions": + return update_github_release_asset_versions(repo_root=repo_root, name=name, surface=surface) + if kind == "canton-tag-versions": + return update_canton_tag_versions(repo_root=repo_root, name=name, surface=surface) + if kind == "canton-release-bundle-minors": + return update_canton_release_bundle_minors(repo_root=repo_root, name=name, surface=surface) + if kind == "maven-javadoc-artifact": + return update_maven_javadoc_artifact(repo_root=repo_root, name=name, surface=surface) + if kind == "npm-packages": + return update_npm_packages(repo_root=repo_root, name=name, surface=surface) + raise ValueError(f"Unsupported surface kind for {name}: {kind}") + + +def write_results(repo_root: Path, results: list[SurfaceResult]) -> None: + for result in results: + if not result.changed: + continue + if len(result.config_paths) == 1: + json_dump(repo_root / result.config_paths[0], result.after) + continue + for path in result.config_paths: + json_dump(repo_root / path, result.after[path]) + + +def summary_payload(results: list[SurfaceResult]) -> dict[str, Any]: + updated = [result for result in results if result.changed] + return { + "has_updates": bool(updated), + "updated_surfaces": [result.name for result in updated], + "surfaces": [ + { + "name": result.name, + "config_paths": list(result.config_paths), + "changed": result.changed, + "updates": list(result.updates), + } + for result in results + ], + } + + +def pr_body(summary: dict[str, Any]) -> str: + if not summary["has_updates"]: + return "No generated reference-doc source updates were found." + lines = [ + "Generated reference-doc source pins were updated from upstream releases.", + "", + "Updated surfaces:", + ] + for surface in summary["surfaces"]: + if not surface["changed"]: + continue + lines.append(f"- {surface['name']}") + for update in surface["updates"]: + lines.append(f" - {update}") + lines.extend( + [ + "", + "Validation run by the workflow:", + "- `npm run generate:all-reference-docs`", + "- `python3 -m pytest tests`", + "- `git diff --check`", + ] + ) + return "\n".join(lines) + + +def write_github_outputs(summary: dict[str, Any]) -> None: + output_path = os.environ.get("GITHUB_OUTPUT") + if not output_path: + return + body = pr_body(summary) + with Path(output_path).open("a", encoding="utf-8") as handle: + handle.write(f"has_updates={str(summary['has_updates']).lower()}\n") + handle.write(f"updated_surfaces={','.join(summary['updated_surfaces'])}\n") + handle.write("pr_body< int: + args = parse_args(argv) + repo_root = Path(args.repo_root).resolve() + policy = json_load(Path(args.policy).resolve()) + surfaces = policy.get("surfaces") + if not isinstance(surfaces, dict): + raise ValueError("Policy must define a surfaces object") + + results: list[SurfaceResult] = [] + for name, surface in surfaces.items(): + if not isinstance(surface, dict): + continue + try: + results.append(surface_result(repo_root, name, surface)) + except Exception as exc: + raise RuntimeError(f"Failed to update source surface '{name}'") from exc + if args.apply: + write_results(repo_root, results) + + summary = summary_payload(results) + json_dump(Path(args.summary).resolve(), summary) + write_github_outputs(summary) + print(json.dumps(summary, indent=2)) + return 0 + + +def main(argv: list[str] | None = None) -> int: + ensure_repo_direnv(repo_root=REPO_ROOT, script_path=Path(__file__).resolve(), argv=sys.argv[1:] if argv is None else argv) + return run(argv) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_update_reference_doc_sources.py b/tests/test_update_reference_doc_sources.py new file mode 100644 index 000000000..0b4e552d0 --- /dev/null +++ b/tests/test_update_reference_doc_sources.py @@ -0,0 +1,332 @@ +from __future__ import annotations + +import importlib.util +import json +import os +import sys +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def load_script(name: str): + scripts_dir = str(REPO_ROOT / "scripts") + if scripts_dir not in sys.path: + sys.path.insert(0, scripts_dir) + spec = importlib.util.spec_from_file_location(name, REPO_ROOT / "scripts" / f"{name}.py") + assert spec is not None + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + os.environ.setdefault("DIGITAL_ASSET_DOCS_DIRENV", "1") + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +def write_json(path: Path, payload: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + +def read_json(path: Path) -> object: + return json.loads(path.read_text(encoding="utf-8")) + + +def prepare_repo(tmp_path: Path) -> Path: + repo = tmp_path / "repo" + write_json( + repo / "config" / "x2mdx" / "daml-standard-library" / "source-artifacts.json", + {"publish_version": "1.0.1", "versions": ["1.0.0", "1.0.1"]}, + ) + write_json( + repo / "config" / "x2mdx" / "ledger-api" / "source-artifacts.json", + { + "release_url_template": "https://downloads.example/canton-{canton_version}.tar.gz", + "publish_version": "1.0", + "versions": [{"version": "1.0", "canton_version": "1.0.1"}], + }, + ) + write_json( + repo / "config" / "x2mdx" / "ledger-api-asyncapi" / "source-artifacts.json", + { + "release_url_template": "https://downloads.example/canton-{canton_version}.tar.gz", + "publish_version": "1.0", + "versions": [{"version": "1.0", "canton_version": "1.0.1"}], + }, + ) + write_json( + repo / "config" / "x2mdx" / "ledger-bindings" / "source-artifacts.json", + { + "repo_base": "https://repo.example/maven2", + "artifacts": [ + { + "group": "com.daml", + "artifact": "bindings-java", + "language": "java", + "versions": ["1.0.0", "1.0.1"], + } + ], + }, + ) + write_json( + repo / "config" / "x2mdx" / "protobuf-history" / "source-artifacts.json", + { + "release_url_template": "https://downloads.example/canton-{version}.tar.gz", + "min_version": "1.0.0", + "excluded_versions": ["1.0.1"], + }, + ) + write_json( + repo / "config" / "x2mdx" / "grpc-ledger-api-reference" / "source-artifacts.json", + { + "release_url_template": "https://downloads.example/canton-{version}.tar.gz", + "min_version": "1.1.0", + }, + ) + write_json( + repo / "config" / "x2mdx" / "typescript-bindings" / "source-artifacts.json", + { + "packages": [ + { + "package_name": "@daml/types", + "publish_version": "1.0.1", + "versions": ["1.0.0", "1.0.1"], + }, + { + "package_name": "@canton-network/wallet-sdk", + "publish_version": "2.0.0", + "versions": ["2.0.0"], + }, + ] + }, + ) + write_json( + repo / "config" / "x2mdx" / "wallet-gateway-openrpc" / "source-artifacts.json", + { + "release_repo": "owner/wallet", + "tag_prefix": "@pkg@", + "min_version": "0.1.0", + "publish_version": "0.1.0", + }, + ) + write_json( + repo / "config" / "mintlify-openapi" / "splice-openapi" / "source-artifacts.json", + { + "release_repo": "owner/splice", + "tag_regex": "^v(?P\\d+\\.\\d+\\.\\d+)$", + "asset_template": "{version}_openapi.tar.gz", + "min_version": "1.0.0", + "publish_version": "1.0.0", + }, + ) + write_json( + repo / "config" / "x2mdx" / "reference-update-policy.json", + { + "surfaces": { + "daml": { + "kind": "git-tag-versions", + "config_path": "config/x2mdx/daml-standard-library/source-artifacts.json", + "repository": "owner/daml", + "tag_regex": "^v(?P\\d+\\.\\d+\\.\\d+)$", + "keep": 2, + "publish_latest": True, + }, + "json-ledger": { + "kind": "canton-release-bundle-minors", + "config_paths": [ + "config/x2mdx/ledger-api/source-artifacts.json", + "config/x2mdx/ledger-api-asyncapi/source-artifacts.json", + ], + "repository": "owner/canton", + "tag_regex": "^v(?P\\d+\\.\\d+\\.\\d+)$", + "keep_minor_count": 2, + "validate_release_url": True, + }, + "bindings": { + "kind": "maven-javadoc-artifact", + "config_path": "config/x2mdx/ledger-bindings/source-artifacts.json", + "keep": 2, + "validate_javadoc": True, + }, + "protobuf": { + "kind": "canton-tag-versions", + "config_path": "config/x2mdx/protobuf-history/source-artifacts.json", + "repository": "owner/canton", + "tag_regex": "^v(?P\\d+\\.\\d+\\.\\d+)$", + "validate_release_url": True, + }, + "grpc": { + "kind": "canton-tag-versions", + "config_path": "config/x2mdx/grpc-ledger-api-reference/source-artifacts.json", + "repository": "owner/canton", + "tag_regex": "^v(?P\\d+\\.\\d+\\.\\d+)$", + "validate_release_url": True, + }, + "typescript": { + "kind": "npm-packages", + "config_path": "config/x2mdx/typescript-bindings/source-artifacts.json", + "keep": 2, + "publish_latest": True, + }, + "wallet": { + "kind": "github-release-versions", + "config_path": "config/x2mdx/wallet-gateway-openrpc/source-artifacts.json", + "repository": "owner/wallet", + "tag_regex": "^@pkg@(?P\\d+\\.\\d+\\.\\d+)$", + "keep": 2, + "publish_latest": True, + }, + "splice": { + "kind": "github-release-asset-versions", + "config_path": "config/mintlify-openapi/splice-openapi/source-artifacts.json", + "repository": "owner/splice", + "tag_regex": "^v(?P\\d+\\.\\d+\\.\\d+)$", + "asset_template": "{version}_openapi.tar.gz", + "keep": 1, + "publish_latest": True, + }, + } + }, + ) + return repo + + +def install_fake_discovery(module, monkeypatch) -> None: + def fake_releases(path: str): + if path == "repos/owner/daml/releases": + return [{"tag_name": "v1.0.0"}, {"tag_name": "v1.0.1"}, {"tag_name": "v1.0.2"}] + if path == "repos/owner/daml/tags": + return [{"name": "v1.0.0"}, {"name": "v1.0.1"}, {"name": "v1.0.2"}] + if path == "repos/owner/wallet/releases": + return [{"tag_name": "@pkg@0.1.0"}, {"tag_name": "@pkg@0.2.0"}, {"tag_name": "@pkg@0.3.0"}] + if path == "repos/owner/splice/releases": + return [ + { + "tag_name": "v1.0.0", + "assets": [{"name": "1.0.0_openapi.tar.gz"}], + }, + { + "tag_name": "v1.1.0", + "assets": [{"name": "1.1.0_openapi.tar.gz"}], + }, + ] + if path == "repos/owner/canton/tags": + return [ + {"name": "v1.0.0"}, + {"name": "v1.0.1"}, + {"name": "v1.0.2"}, + {"name": "v1.1.0"}, + {"name": "v1.1.1"}, + ] + raise AssertionError(path) + + def fake_text(url: str) -> str: + assert url == "https://repo.example/maven2/com/daml/bindings-java/maven-metadata.xml" + return """ + + + + 1.0.0 + 1.0.1 + 1.0.2 + + + +""" + + def fake_json(url: str): + if url == "https://registry.npmjs.org/%40daml%2Ftypes": + return {"versions": {"1.0.0": {}, "1.0.1": {}, "1.0.2": {}}} + if url == "https://registry.npmjs.org/%40canton-network%2Fwallet-sdk": + return {"versions": {"2.0.0": {}, "2.0.1": {}, "2.0.2": {}}} + raise AssertionError(url) + + monkeypatch.setattr(module, "github_paginated", fake_releases) + monkeypatch.setattr(module, "fetch_text_url", fake_text) + monkeypatch.setattr(module, "fetch_json_url", fake_json) + monkeypatch.setattr(module, "head_url", lambda url: True) + + +def test_apply_updates_source_configs_from_discovered_versions(tmp_path: Path, monkeypatch) -> None: + module = load_script("update_reference_doc_sources") + repo = prepare_repo(tmp_path) + install_fake_discovery(module, monkeypatch) + summary_path = repo / ".internal" / "summary.json" + + result = module.run( + [ + "--apply", + "--repo-root", + str(repo), + "--policy", + str(repo / "config" / "x2mdx" / "reference-update-policy.json"), + "--summary", + str(summary_path), + ] + ) + + assert result == 0 + summary = read_json(summary_path) + assert summary["has_updates"] is True + assert summary["updated_surfaces"] == [ + "daml", + "json-ledger", + "bindings", + "protobuf", + "grpc", + "typescript", + "wallet", + "splice", + ] + assert read_json(repo / "config" / "x2mdx" / "daml-standard-library" / "source-artifacts.json") == { + "publish_version": "1.0.2", + "versions": ["1.0.1", "1.0.2"], + } + assert read_json(repo / "config" / "x2mdx" / "ledger-api" / "source-artifacts.json")["versions"] == [ + {"version": "1.0", "canton_version": "1.0.2"}, + {"version": "1.1", "canton_version": "1.1.1"}, + ] + assert read_json(repo / "config" / "x2mdx" / "protobuf-history" / "source-artifacts.json")["versions"] == [ + "1.0.0", + "1.0.2", + "1.1.0", + "1.1.1", + ] + assert read_json(repo / "config" / "x2mdx" / "grpc-ledger-api-reference" / "source-artifacts.json")["versions"] == [ + "1.1.0", + "1.1.1", + ] + typescript = read_json(repo / "config" / "x2mdx" / "typescript-bindings" / "source-artifacts.json") + assert typescript["packages"][0]["versions"] == ["1.0.1", "1.0.2"] + assert typescript["packages"][1]["publish_version"] == "2.0.2" + + +def test_check_mode_reports_no_updates_without_rewriting(tmp_path: Path, monkeypatch) -> None: + module = load_script("update_reference_doc_sources") + repo = prepare_repo(tmp_path) + install_fake_discovery(module, monkeypatch) + policy = repo / "config" / "x2mdx" / "reference-update-policy.json" + summary_path = repo / ".internal" / "summary.json" + module.run(["--apply", "--repo-root", str(repo), "--policy", str(policy), "--summary", str(summary_path)]) + before = (repo / "config" / "x2mdx" / "typescript-bindings" / "source-artifacts.json").read_text( + encoding="utf-8" + ) + + result = module.run(["--check", "--repo-root", str(repo), "--policy", str(policy), "--summary", str(summary_path)]) + + assert result == 0 + assert read_json(summary_path)["has_updates"] is False + after = (repo / "config" / "x2mdx" / "typescript-bindings" / "source-artifacts.json").read_text( + encoding="utf-8" + ) + assert after == before + + +def test_generators_use_configured_versions_as_default_pin() -> None: + assert load_script("generate_canton_protobuf_history").configured_versions({"versions": ["1.2.3"]}) == {"1.2.3"} + assert load_script("generate_grpc_ledger_api_reference").configured_versions({"versions": ["1.2.3"]}) == {"1.2.3"} + assert load_script("generate_wallet_gateway_openrpc_reference").configured_versions({"versions": ["1.2.3"]}) == { + "1.2.3" + } + assert load_script("generate_splice_mintlify_openapi").configured_versions({"versions": ["1.2.3"]}) == {"1.2.3"}