From 59d7e7cfcfd408c881cd5b798945dd4c059597a4 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Wed, 22 Apr 2026 20:36:08 +0100 Subject: [PATCH 1/2] Cache Deno development cache in CI (#14408) * Gate vendor.sh cache wipe behind QUARTO_SKIP_DENO_CACHE_WIPE Default behavior unchanged: local ./configure.sh still wipes deno_cache before re-vendoring. CI opts into cache preservation by setting QUARTO_SKIP_DENO_CACHE_WIPE=1, enabling actions/cache to restore the cache and have it survive vendor.sh. * Gate vendor.cmd cache wipe behind QUARTO_SKIP_DENO_CACHE_WIPE Windows counterpart to the vendor.sh change. Behavior matches: default wipes, QUARTO_SKIP_DENO_CACHE_WIPE=1 preserves. * Cache Deno development cache in CI Adds actions/cache restore for ./package/dist/bin/deno_cache, keyed on the five files that influence Deno dep resolution: configuration, src/import_map.json, src/vendor_deps.ts, tests/test-deps.ts, and package/scripts/deno_std/deno_std.ts. Both configure steps set QUARTO_SKIP_DENO_CACHE_WIPE=1 so the restored cache survives the configure.sh / configure.cmd run. Adds dev-docs/ci-deno-caching.md explaining the cache layout, key composition, env var contract, and how to force invalidation. Adds a cross-link from dev-docs/upgrade-dependencies.md so a contributor bumping deno_std version sees the cache consequences. Expected per-OS cache size ~150-200 MB; total footprint ~600 MB across Linux/macOS/Windows, well under GitHub's 10 GB repo quota. (cherry picked from commit e4b75c1a32902f6d6e01ac62752a5b7d02ec0a12) --- .../workflows/actions/quarto-dev/action.yml | 12 ++ dev-docs/ci-deno-caching.md | 107 ++++++++++++++++++ dev-docs/upgrade-dependencies.md | 2 + package/scripts/vendoring/vendor.cmd | 7 +- package/scripts/vendoring/vendor.sh | 5 +- 5 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 dev-docs/ci-deno-caching.md diff --git a/.github/workflows/actions/quarto-dev/action.yml b/.github/workflows/actions/quarto-dev/action.yml index cc5292af525..42d1aa8884a 100644 --- a/.github/workflows/actions/quarto-dev/action.yml +++ b/.github/workflows/actions/quarto-dev/action.yml @@ -23,9 +23,19 @@ runs: restore-keys: | ${{ runner.os }}-cargo-typst-gather- + - name: Cache Deno development cache + uses: actions/cache@v5 + with: + path: ./package/dist/bin/deno_cache + key: ${{ runner.os }}-${{ runner.arch }}-deno-cache-v1-${{ hashFiles('configuration', 'src/import_map.json', 'src/vendor_deps.ts', 'tests/test-deps.ts', 'package/scripts/deno_std/deno_std.ts') }} + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-deno-cache-v1- + - name: Configure Quarto (.sh) if: runner.os != 'Windows' shell: bash + env: + QUARTO_SKIP_DENO_CACHE_WIPE: "1" run: | # install with symlink in /usr/local/bin which in PATH on CI ./configure.sh @@ -33,6 +43,8 @@ runs: - name: Configure Quarto (.ps1) if: runner.os == 'Windows' shell: pwsh + env: + QUARTO_SKIP_DENO_CACHE_WIPE: "1" run: | ./configure.cmd "$(Get-ChildItem -Path ./package/dist/bin/quarto.cmd | %{ $_.FullName } | Split-Path)" >> $env:GITHUB_PATH diff --git a/dev-docs/ci-deno-caching.md b/dev-docs/ci-deno-caching.md new file mode 100644 index 00000000000..b368d127745 --- /dev/null +++ b/dev-docs/ci-deno-caching.md @@ -0,0 +1,107 @@ +# CI Deno cache + +How the Quarto development Deno cache is cached in GitHub Actions, how +invalidation works, and how to force a fresh download. + +## What gets cached + +Two Deno caches are restored by the `quarto-dev` composite action +(`.github/workflows/actions/quarto-dev/action.yml`): + +| Cache | Path | Populated by | +|-------|------|--------------| +| Deno standard library | `./src/resources/deno_std/cache` | `package/scripts/deno_std/deno_std.ts` | +| Deno development cache | `./package/dist/bin/deno_cache` | `package/scripts/vendoring/vendor.sh` / `vendor.cmd` | + +This document covers the second one. The first predates it and is keyed on +`deno_std.lock` + `deno_std.ts`. + +## Cache key + +``` +${{ runner.os }}-${{ runner.arch }}-deno-cache-v1- +``` + +`` is `hashFiles()` over five files: + +| File | Why it's in the key | +|------|---------------------| +| `configuration` | Pins the Deno binary version. | +| `src/import_map.json` | Top-level dep version map — the primary driver of what gets downloaded. | +| `src/vendor_deps.ts` | Explicit "things to vendor" entrypoint. | +| `tests/test-deps.ts` | Test-only deps entrypoint (iterated by vendor.sh / vendor.cmd). | +| `package/scripts/deno_std/deno_std.ts` | deno_std entrypoint iterated by the vendor scripts. | + +`restore-keys` falls back to the same prefix without the hash suffix, so a +partial-match cache from a sibling branch is reused if the exact key misses. + +### Not in the key + +- `quarto.ts` and other `src/**/*.ts` files: these are consumer code, not + dep-resolution inputs. `deno install` is additive on top of an existing + cache, so new imports in consumer code download without needing a key change. + Hashing `src/**` would invalidate on every commit. +- `vendor.sh` / `vendor.cmd`: they control *how* `deno install` runs, not + *what* it resolves. Including them would cross-invalidate (a Unix-only + `vendor.sh` edit would bust the Windows cache too, since `hashFiles()` is + OS-independent). +- `deno.lock` / `tests/deno.lock`: both are gitignored (`.gitignore:26`) and + are generated by `deno install` at run time, so they do not exist on a + fresh checkout and cannot serve as key inputs. + +## Wipe gate + +`vendor.sh` and `vendor.cmd` delete `./package/dist/bin/deno_cache` before +running `deno install`. That default is preserved for local development. In +CI we need the opposite — the cache was just restored by `actions/cache` and +must survive until `deno install` uses it. + +The coordination uses a single environment variable: + +- **Name:** `QUARTO_SKIP_DENO_CACHE_WIPE` +- **Truthy value:** `"1"` (anything else, including unset, means "wipe"). +- **Set by:** both configure steps in `action.yml` (`env: QUARTO_SKIP_DENO_CACHE_WIPE: "1"`). +- **Read by:** `vendor.sh` and `vendor.cmd`. Nothing else. +- **Do not set locally.** The variable only makes sense when paired with a + restore step. Without one, skipping the wipe just leaves a stale cache on + disk. + +## Forcing invalidation + +### When it happens automatically + +Editing any of the five keyed files changes the hash, which produces a new +cache key. Next CI run misses, re-downloads, then saves under the new key. +Old keys age out via GitHub's 7-day LRU. + +### When you have to force it manually + +Only if the cache contents go bad in a way the key can't detect — e.g. a +partial download that wasn't corrected, or a silent shape change from a new +Deno version that still resolves to the same `configuration` string. In that +case, bump the `-v1-` version prefix in both the `key` and `restore-keys` of +the cache step in `action.yml`: + +```diff +- key: ${{ runner.os }}-${{ runner.arch }}-deno-cache-v1-... ++ key: ${{ runner.os }}-${{ runner.arch }}-deno-cache-v2-... + restore-keys: | +- ${{ runner.os }}-${{ runner.arch }}-deno-cache-v1- ++ ${{ runner.os }}-${{ runner.arch }}-deno-cache-v2- +``` + +This avoids editing any of the five hashed files (which may not even be +involved in the regression) and leaves a clear audit trail in git history. + +## Rollback + +- **Cache misbehaves:** delete the "Cache Deno development cache" step in + `action.yml`. `configure.sh` / `configure.cmd` run `vendor.sh` / + `vendor.cmd` as before and redownload from scratch. +- **Wipe gate misbehaves:** remove the `QUARTO_SKIP_DENO_CACHE_WIPE` env + from the configure steps. Vendor scripts fall back to the default wipe. + +## Expected footprint + +~150–200 MB per OS, three OSes → ~600 MB total repo cache usage, well under +GitHub's 10 GB per-repo quota. diff --git a/dev-docs/upgrade-dependencies.md b/dev-docs/upgrade-dependencies.md index fe52814715d..6c888c15c87 100644 --- a/dev-docs/upgrade-dependencies.md +++ b/dev-docs/upgrade-dependencies.md @@ -12,6 +12,8 @@ Contact Carlos so he uploads the binaries to the S3 bucket. - run `./configure.sh`. +Bumping a version in `src/import_map.json` (or any of the other keyed files) automatically invalidates the CI Deno cache on next run. See [ci-deno-caching.md](ci-deno-caching.md) for the key composition and how to force invalidation manually. + ### Upgrade Deno download link for RHEL build from conda-forge - Go to and find the version of Deno required. diff --git a/package/scripts/vendoring/vendor.cmd b/package/scripts/vendoring/vendor.cmd index f9ab84973fa..e43fae3577b 100644 --- a/package/scripts/vendoring/vendor.cmd +++ b/package/scripts/vendoring/vendor.cmd @@ -18,9 +18,12 @@ IF NOT DEFINED DENO_DIR ( ECHO Revendoring quarto dependencies -REM remove deno_cache directory first +REM remove deno_cache directory first, unless explicitly told to preserve +REM (CI sets QUARTO_SKIP_DENO_CACHE_WIPE=1 so a restored cache survives vendor.cmd) IF EXIST "!DENO_DIR!" ( - RMDIR /S /Q "!DENO_DIR!" + IF NOT "!QUARTO_SKIP_DENO_CACHE_WIPE!"=="1" ( + RMDIR /S /Q "!DENO_DIR!" + ) ) PUSHD "!QUARTO_SRC_PATH!" diff --git a/package/scripts/vendoring/vendor.sh b/package/scripts/vendoring/vendor.sh index 04d899a51ed..976eb80d6b5 100755 --- a/package/scripts/vendoring/vendor.sh +++ b/package/scripts/vendoring/vendor.sh @@ -15,8 +15,9 @@ fi echo Revendoring quarto dependencies -# remove deno_cache directory first -if [ -d "$DENO_DIR" ]; then +# remove deno_cache directory first, unless explicitly told to preserve +# (CI sets QUARTO_SKIP_DENO_CACHE_WIPE=1 so a restored cache survives vendor.sh) +if [ -d "$DENO_DIR" ] && [ "${QUARTO_SKIP_DENO_CACHE_WIPE}" != "1" ]; then rm -rf "$DENO_DIR" fi From 7061f9a1b2447a186b01f88dd27f4b4ffa0bcbf9 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Tue, 28 Apr 2026 15:58:44 +0200 Subject: [PATCH 2/2] Fix configure scripts destroying restored Deno CI cache (#14450) The CI Deno dev cache (Cache Deno development cache step in .github/workflows/actions/quarto-dev/action.yml) restores package/dist/bin/deno_cache before configure runs. configure.sh and configure.cmd then unconditionally wipe package/dist when bootstrapping Deno, destroying the just-restored cache before vendor.sh reaches it. The vendor scripts already gate their own cache wipe with QUARTO_SKIP_DENO_CACHE_WIPE=1; the configure-time wipe was missed. Apply the same gate to configure.sh and configure.cmd so the restored cache survives. Local devs (env var unset) keep the original wipe behavior. (cherry picked from commit 3e11b832cc00a7eef2a37882d8ac3b6c016ccc37) --- configure.cmd | 42 +++++++++++++++++++++++------------------- configure.sh | 6 +++++- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/configure.cmd b/configure.cmd index f1eccc9c3ed..6c7d3632940 100644 --- a/configure.cmd +++ b/configure.cmd @@ -15,27 +15,31 @@ if NOT DEFINED QUARTO_VENDOR_BINARIES ( if "%QUARTO_VENDOR_BINARIES%" == "true" ( - REM Windows-specific: Check if deno.exe is running before deleting package/dist - REM Extracted to package/scripts/windows/check-deno-in-use.cmd for maintainability - call package\scripts\windows\check-deno-in-use.cmd "!QUARTO_DIST_PATH!" - if "!ERRORLEVEL!"=="1" exit /B 1 - - echo Removing package/dist/ directory... - RMDIR /S /Q "!QUARTO_DIST_PATH!" 2>NUL - - REM Fallback: Verify deletion succeeded (defense in depth) - if exist "!QUARTO_DIST_PATH!" ( - echo. - echo ============================================================ - echo Error: Could not delete package/dist/ directory - echo This may be due to permissions, antivirus, or another process holding files - echo ============================================================ - echo. - echo Try closing applications and run configure.cmd again - exit /B 1 + REM CI sets QUARTO_SKIP_DENO_CACHE_WIPE=1 so the restored deno_cache survives + REM (matches the gate used by package\scripts\vendoring\vendor.cmd). + IF NOT "!QUARTO_SKIP_DENO_CACHE_WIPE!"=="1" ( + REM Windows-specific: Check if deno.exe is running before deleting package/dist + REM Extracted to package/scripts/windows/check-deno-in-use.cmd for maintainability + call package\scripts\windows\check-deno-in-use.cmd "!QUARTO_DIST_PATH!" + if "!ERRORLEVEL!"=="1" exit /B 1 + + echo Removing package/dist/ directory... + RMDIR /S /Q "!QUARTO_DIST_PATH!" 2>NUL + + REM Fallback: Verify deletion succeeded (defense in depth) + if exist "!QUARTO_DIST_PATH!" ( + echo. + echo ============================================================ + echo Error: Could not delete package/dist/ directory + echo This may be due to permissions, antivirus, or another process holding files + echo ============================================================ + echo. + echo Try closing applications and run configure.cmd again + exit /B 1 + ) ) - MKDIR !QUARTO_BIN_PATH!\tools + IF NOT EXIST "!QUARTO_BIN_PATH!\tools" MKDIR !QUARTO_BIN_PATH!\tools PUSHD !QUARTO_BIN_PATH!\tools ECHO Bootstrapping Deno... diff --git a/configure.sh b/configure.sh index 64f54e790e0..d505fd68dd2 100755 --- a/configure.sh +++ b/configure.sh @@ -37,7 +37,11 @@ if [[ "${QUARTO_VENDOR_BINARIES}" = "true" ]]; then # Ensure directory is there for Deno echo "Bootstrapping Deno..." - rm -rf "$QUARTO_DIST_PATH" + # CI sets QUARTO_SKIP_DENO_CACHE_WIPE=1 so the restored deno_cache survives + # (matches the gate used by package/scripts/vendoring/vendor.sh). + if [ "${QUARTO_SKIP_DENO_CACHE_WIPE}" != "1" ]; then + rm -rf "$QUARTO_DIST_PATH" + fi ## Binary Directory mkdir -p "$QUARTO_BIN_PATH/tools"