Automate generated reference doc source updates

.github/workflows/update-generated-reference-docs.yml

@@ -0,0 +1,76 @@
+name: Update generated reference docs
+
+on:
+  schedule:
+    - cron: "17 8 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+concurrency:
+  group: update-generated-reference-docs
+  cancel-in-progress: false
+
+jobs:
+  update-generated-reference-docs:
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{ secrets.EXTERNAL_REPO_TOKEN || github.token }}
+      GH_TOKEN: ${{ secrets.EXTERNAL_REPO_TOKEN || github.token }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+
+      - name: Install Nix
+        uses: cachix/install-nix-action@v31
+
+      - name: Install direnv
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y direnv
+
+      - name: Trust direnv
+        run: direnv allow .
+
+      - name: Update source pins
+        id: update
+        run: |
+          mkdir -p .internal
+          direnv exec . python3 scripts/update_reference_doc_sources.py \
+            --apply \
+            --summary .internal/reference-doc-update-summary.json
+
+      - name: Regenerate reference docs
+        if: steps.update.outputs.has_updates == 'true'
+        run: direnv exec . npm run generate:all-reference-docs
+
+      - name: Run tests
+        if: steps.update.outputs.has_updates == 'true'
+        run: direnv exec . python3 -m pytest tests
+
+      - name: Check whitespace
+        if: steps.update.outputs.has_updates == 'true'
+        run: git diff --check
+
+      - name: Upload update artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: generated-reference-doc-update
+          if-no-files-found: ignore
+          path: |
+            .internal/reference-doc-update-summary.json
+            .internal/generated/x2mdx/**/*.json
+
+      - name: Create pull request
+        if: steps.update.outputs.has_updates == 'true'
+        uses: peter-evans/create-pull-request@v8
+        with:
+          token: ${{ secrets.DOCS_PR_TOKEN }}
+          branch: generated-reference-docs/update
+          base: main
+          title: Update generated reference docs from upstream releases
+          commit-message: Update generated reference docs from upstream releases
+          body: ${{ steps.update.outputs.pr_body }}

config/mintlify-openapi/splice-openapi/source-artifacts.json

@@ -4,6 +4,9 @@
   "tag_regex": "^v(?P<version>0\\.[0-9]+\\.[0-9]+)$",
   "min_version": "0.5.10",
   "publish_version": "0.5.18",
+  "versions": [
+    "0.5.18"
+  ],
   "asset_template": "{version}_openapi.tar.gz",
   "nav_dropdown": "API Reference",
   "top_level_group_label": "Splice APIs",

config/x2mdx/grpc-ledger-api-reference/source-artifacts.json

@@ -7,6 +7,16 @@
     "web_url": "https://github.com/DACH-NY/canton"
   },
   "min_version": "3.4.4",
+  "versions": [
+    "3.4.4",
+    "3.4.5",
+    "3.4.6",
+    "3.4.7",
+    "3.4.8",
+    "3.4.9",
+    "3.4.10",
+    "3.4.11"
+  ],
   "metadata_path": "config/x2mdx/protobuf-history/metadata.json",
   "package_prefixes": [
     "com.daml.ledger.api.v2"

config/x2mdx/protobuf-history/source-artifacts.json

@@ -8,5 +8,18 @@
   },
   "min_version": "3.2.0",
   "excluded_versions": ["3.4.1"],
+  "versions": [
+    "3.4.0",
+    "3.4.2",
+    "3.4.3",
+    "3.4.4",
+    "3.4.5",
+    "3.4.6",
+    "3.4.7",
+    "3.4.8",
+    "3.4.9",
+    "3.4.10",
+    "3.4.11"
+  ],
   "metadata_path": "config/x2mdx/protobuf-history/metadata.json"
 }

config/x2mdx/reference-update-policy.json

@@ -0,0 +1,71 @@
+{
+  "surfaces": {
+    "daml-standard-library": {
+      "kind": "git-tag-versions",
+      "config_path": "config/x2mdx/daml-standard-library/source-artifacts.json",
+      "repository": "digital-asset/daml",
+      "remote": "https://github.com/digital-asset/daml.git",
+      "tag_regex": "^v(?P<version>\\d+\\.\\d+\\.\\d+)$",
+      "keep": 3,
+      "publish_latest": true
+    },
+    "json-ledger-api": {
+      "kind": "canton-release-bundle-minors",
+      "config_paths": [
+        "config/x2mdx/ledger-api/source-artifacts.json",
+        "config/x2mdx/ledger-api-asyncapi/source-artifacts.json"
+      ],
+      "repository": "DACH-NY/canton",
+      "remote": "https://github.com/DACH-NY/canton.git",
+      "tag_regex": "^v(?P<canton_version>\\d+\\.\\d+\\.\\d+)$",
+      "keep_minor_count": 2,
+      "validate_release_url": true
+    },
+    "ledger-bindings": {
+      "kind": "maven-javadoc-artifact",
+      "config_path": "config/x2mdx/ledger-bindings/source-artifacts.json",
+      "keep": 4,
+      "validate_javadoc": true
+    },
+    "protobuf-history": {
+      "kind": "canton-tag-versions",
+      "config_path": "config/x2mdx/protobuf-history/source-artifacts.json",
+      "repository": "DACH-NY/canton",
+      "remote": "https://github.com/DACH-NY/canton.git",
+      "tag_regex": "^v(?P<version>\\d+\\.\\d+\\.\\d+)$",
+      "validate_release_url": true
+    },
+    "grpc-ledger-api-reference": {
+      "kind": "canton-tag-versions",
+      "config_path": "config/x2mdx/grpc-ledger-api-reference/source-artifacts.json",
+      "repository": "DACH-NY/canton",
+      "remote": "https://github.com/DACH-NY/canton.git",
+      "tag_regex": "^v(?P<version>\\d+\\.\\d+\\.\\d+)$",
+      "validate_release_url": true
+    },
+    "typescript-bindings": {
+      "kind": "npm-packages",
+      "config_path": "config/x2mdx/typescript-bindings/source-artifacts.json",
+      "keep": 4,
+      "publish_latest": true
+    },
+    "wallet-gateway-openrpc": {
+      "kind": "github-release-versions",
+      "config_path": "config/x2mdx/wallet-gateway-openrpc/source-artifacts.json",
+      "repository": "hyperledger-labs/splice-wallet-kernel",
+      "tag_prefix": "@canton-network/wallet-gateway-remote@",
+      "tag_regex": "^@canton-network/wallet-gateway-remote@(?P<version>\\d+\\.\\d+\\.\\d+)$",
+      "keep": 2,
+      "publish_latest": true
+    },
+    "splice-openapi": {
+      "kind": "github-release-asset-versions",
+      "config_path": "config/mintlify-openapi/splice-openapi/source-artifacts.json",
+      "repository": "digital-asset/decentralized-canton-sync",
+      "tag_regex": "^v(?P<version>0\\.[0-9]+\\.[0-9]+)$",
+      "asset_template": "{version}_openapi.tar.gz",
+      "keep": 1,
+      "publish_latest": true
+    }
+  }
+}

config/x2mdx/wallet-gateway-openrpc/source-artifacts.json

@@ -5,6 +5,10 @@
   "tag_prefix": "@canton-network/wallet-gateway-remote@",
   "min_version": "0.24.0",
   "publish_version": "0.25.0",
+  "versions": [
+    "0.24.0",
+    "0.25.0"
+  ],
   "specs": [
     {
       "spec_id": "dapp-api",

scripts/generate_canton_protobuf_history.py

@@ -122,6 +122,15 @@ def load_excluded_versions(source_config: dict[str, Any]) -> set[str]:
     return set(configured)
 
 
+def configured_versions(source_config: dict[str, Any]) -> set[str] | None:
+    configured = source_config.get("versions")
+    if configured is None:
+        return None
+    if not isinstance(configured, list) or not all(isinstance(item, str) and item for item in configured):
+        raise ValueError("Source config versions must be a list of non-empty strings")
+    return set(configured)
+
+
 def run(args: list[str], *, cwd: Path | None = None, capture: bool = False) -> str:
     kwargs: dict[str, Any] = {
         "cwd": str(cwd) if cwd else None,
@@ -647,7 +656,7 @@ def main() -> int:
     if not isinstance(bundle_proto_dir, str) or not bundle_proto_dir:
         raise ValueError("Source config must define bundle_proto_dir")
 
-    include_versions = set(args.version) if args.version else None
+    include_versions = set(args.version) if args.version else configured_versions(source_config)
     excluded_versions = load_excluded_versions(source_config)
     if include_versions is not None:
         include_versions -= excluded_versions

scripts/generate_grpc_ledger_api_reference.py

@@ -92,6 +92,15 @@ def package_prefixes(source_config: dict[str, Any]) -> tuple[str, ...]:
     return tuple(configured)
 
 
+def configured_versions(source_config: dict[str, Any]) -> set[str] | None:
+    configured = source_config.get("versions")
+    if configured is None:
+        return None
+    if not isinstance(configured, list) or not all(isinstance(item, str) and item for item in configured):
+        raise ValueError("Source config versions must be a list of non-empty strings")
+    return set(configured)
+
+
 def package_matches(package_name: str, *, prefixes: tuple[str, ...]) -> bool:
     return any(package_name.startswith(prefix) for prefix in prefixes)
 
@@ -641,7 +650,7 @@ def main() -> int:
     args = parse_args()
     source_config = load_json(Path(args.source_config).resolve())
     prefixes = package_prefixes(source_config)
-    include_versions = set(args.version) if args.version else None
+    include_versions = set(args.version) if args.version else configured_versions(source_config)
     min_version = args.min_version or source_config.get("min_version") or "0.0.0"
     if not isinstance(min_version, str):
         raise ValueError("min_version must be a string")

scripts/generate_splice_mintlify_openapi.py

@@ -37,12 +37,16 @@ def version_key(version: str) -> tuple[int, ...]:
 
 
 def github_json(url: str) -> Any:
+    headers = {
+        "Accept": "application/vnd.github+json",
+        "User-Agent": USER_AGENT,
+    }
+    token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
     request = urllib.request.Request(
         url,
-        headers={
-            "Accept": "application/vnd.github+json",
-            "User-Agent": USER_AGENT,
-        },
+        headers=headers,
     )
     with urllib.request.urlopen(request, timeout=180) as response:
         return json.loads(response.read().decode("utf-8"))
@@ -129,6 +133,15 @@ def selected_releases(
     return releases
 
 
+def configured_versions(source_config: dict[str, Any]) -> set[str] | None:
+    configured = source_config.get("versions")
+    if configured is None:
+        return None
+    if not isinstance(configured, list) or not all(isinstance(item, str) and item for item in configured):
+        raise ValueError("Source config versions must be a list of non-empty strings")
+    return set(configured)
+
+
 def resolve_publish_release(
     *,
     source_config: dict[str, Any],
@@ -540,7 +553,7 @@ def parse_args() -> argparse.Namespace:
 def main() -> int:
     args = parse_args()
     source_config = load_json(Path(args.source_config).resolve())
-    include_versions = set(args.version) if args.version else None
+    include_versions = set(args.version) if args.version else configured_versions(source_config)
     releases = selected_releases(source_config=source_config, include_versions=include_versions)
     publish_release = resolve_publish_release(
         source_config=source_config,

scripts/generate_wallet_gateway_openrpc_reference.py

@@ -163,6 +163,15 @@ def stable_release_versions(
     return selected
 
 
+def configured_versions(source_config: dict[str, Any]) -> set[str] | None:
+    configured = source_config.get("versions")
+    if configured is None:
+        return None
+    if not isinstance(configured, list) or not all(isinstance(item, str) and item for item in configured):
+        raise ValueError("Source config versions must be a list of non-empty strings")
+    return set(configured)
+
+
 def docs_json_page_ref(path: Path, docs_json_path: Path) -> str:
     relative = path.resolve().relative_to(docs_json_path.resolve().parent)
     if relative.suffix != ".mdx":
@@ -364,7 +373,7 @@ def main() -> int:
     ensure_repo_direnv(repo_root=REPO_ROOT, script_path=Path(__file__).resolve(), argv=sys.argv[1:])
     args = parse_args()
     source_config = load_json(Path(args.source_config).resolve())
-    include_versions = set(args.version) if args.version else None
+    include_versions = set(args.version) if args.version else configured_versions(source_config)
     remote = str(source_config.get("remote") or "")
     release_repo = str(source_config.get("release_repo") or DEFAULT_RELEASE_REPO)
     tag_prefix = str(source_config.get("tag_prefix") or "")