From 3b2bc9e686fc61d1872d7aa4d8e2ca4444f562f6 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 14 May 2026 12:44:28 -0600 Subject: [PATCH] Document storage tuning, thread debugging, MQTT durable sessions, analytics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fill in documentation gaps identified by reviewing harper-pro/core against the reference docs: - New Storage Tuning page covering writeAsync, compression dictionary/ threshold, blobPaths, prefetchWrites, noReadAhead, pageSize, caching, maxTransactionQueueTime, and the storage.reclamation.* sub-properties (threshold/interval/evictionFactor) — previously schema-only. - New Worker Thread Debugging page covering threads.debug.* (port, startingPort, host, waitForDebugger), Chrome DevTools and VS Code attach instructions, SSH-tunneled remote debugging, and an expanded treatment of threads.heapSnapshotNearLimit. - MQTT overview: full Durable Sessions section (cleanStart/ sessionExpiryInterval, hdb_durable_session storage, audit-log catch-up, per-node scope) and a note on hdb_session_will for Last Will durability. Fix a duplicate Last Will heading. - Analytics overview: full analytics.* config reference including the previously undocumented storageInterval, plus replicate, logging, and aggregatePeriod. - configuration/options.md: annotate threads.heapSnapshotNearLimit and threads.debug, expand storage.reclamation.* with per-property defaults, add analytics.storageInterval and analytics.logging, link storage section to the new Storage Tuning page. - Cross-links from configuration/overview.md and database/overview.md; sidebar entries for both new pages. Co-Authored-By: Claude Opus 4.7 (1M context) --- reference/analytics/overview.md | 35 ++++- reference/configuration/debugging.md | 111 ++++++++++++++ reference/configuration/options.md | 12 +- reference/configuration/overview.md | 7 + reference/database/overview.md | 1 + reference/database/storage-tuning.md | 209 +++++++++++++++++++++++++++ reference/mqtt/overview.md | 30 +++- sidebarsReference.ts | 10 ++ 8 files changed, 407 insertions(+), 8 deletions(-) create mode 100644 reference/configuration/debugging.md create mode 100644 reference/database/storage-tuning.md diff --git a/reference/analytics/overview.md b/reference/analytics/overview.md index b3f4fb68..06c2d27e 100644 --- a/reference/analytics/overview.md +++ b/reference/analytics/overview.md @@ -196,9 +196,40 @@ Applications can record custom metrics using the `server.recordAnalytics()` API. ## Analytics Configuration -The `analytics.aggregatePeriod` configuration option controls how frequently aggregate summaries are written. See [Configuration Overview](../configuration/overview.md) for details. +The `analytics` configuration section controls aggregation, replication, and storage-volume sampling. All options are optional. + +```yaml +analytics: + aggregatePeriod: 60 + storageInterval: 10 + replicate: false + logging: + level: info +``` + +### `analytics.aggregatePeriod` + +Type: `number` (seconds)  •  Default: `60` + +How frequently Harper aggregates raw per-second entries into the `hdb_analytics` summary table. Lowering this gives higher-resolution aggregate data at the cost of more frequent aggregation work and more rows in `hdb_analytics`. + +### `analytics.storageInterval` + +Type: `number`  •  Default: `10` + +Number of aggregation cycles between disk-volume measurements. With the default `aggregatePeriod` of `60` and `storageInterval` of `10`, Harper records `database-size`, `table-size`, and `storage-volume` metrics every 10 minutes. Set to `0` to disable storage-volume sampling entirely — useful when running on systems where `statfs` is expensive or unavailable (e.g., some FUSE mounts). + +### `analytics.replicate` + +Type: `boolean`  •  Default: `false` + +When enabled, aggregate analytics entries are replicated across the cluster so a single peer can answer aggregate queries for the whole topology. Raw per-thread entries (`hdb_raw_analytics`) are always node-local. Enable when running a centralized analytics consumer; leave disabled in large clusters to avoid replication overhead for high-cardinality metrics. + +### `analytics.logging` + +Type: `object` -Per-component analytics logging can be configured via `analytics.logging`. See [Logging Configuration](../logging/configuration.md) for details. +Per-subsystem logging override for the analytics writer. See [Logging Configuration — `analytics.logging`](../logging/configuration.md#analyticslogging). ## Related diff --git a/reference/configuration/debugging.md b/reference/configuration/debugging.md new file mode 100644 index 00000000..41155dd6 --- /dev/null +++ b/reference/configuration/debugging.md @@ -0,0 +1,111 @@ +--- +title: Worker Thread Debugging +--- + +# Worker Thread Debugging + +Harper runs as a main thread plus a pool of worker threads (configurable via `threads.count`). The `threads.debug` option exposes the Node.js inspector on each thread so you can attach Chrome DevTools, VS Code, or any [Chrome DevTools Protocol](https://chromedevtools.github.io/devtools-protocol/) (CDP) client to step through component code, inspect heap state, or capture CPU profiles. + +For the worker thread architecture, see [Architecture Overview](../database/overview.md#architecture-overview). + +## Enabling the Debugger + +The simplest form starts the inspector on the main thread at the default port (`9229`): + +```yaml +threads: + debug: true +``` + +For per-thread debugging, expand to an object: + +```yaml +threads: + debug: + startingPort: 9229 + host: 127.0.0.1 + waitForDebugger: false +``` + +| Property | Type | Default | Description | +| ----------------- | --------- | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `port` | `integer` | `9229` | Port for the main thread inspector. Use this when you only need to debug startup or main-thread behavior. | +| `startingPort` | `integer` | _(none)_ | When set, each worker thread gets a sequential inspector port starting from this value. Thread N uses port `startingPort + N`. The main thread keeps `port`. | +| `host` | `string` | `127.0.0.1` | Interface the inspector binds to. Leave on loopback in production; use `0.0.0.0` only when tunneling over SSH or operating in a trusted network. | +| `waitForDebugger` | `boolean` | `false` | Pause each thread at startup until a debugger attaches. Useful for catching bugs that occur during component initialization. | + +## Attaching Chrome DevTools + +1. Set `threads.debug.startingPort: 9230` (so worker threads use 9230, 9231, … and the main thread keeps 9229). +2. Start Harper. +3. Open `chrome://inspect` in Chrome. +4. Click **Configure** and add `localhost:9229`, `localhost:9230`, … for each thread you want to inspect. +5. The threads appear under **Remote Target**. Click **inspect** to open DevTools for that thread. + +## Attaching VS Code + +Add an entry per thread to `.vscode/launch.json`. The example below attaches to the main thread and two workers: + +```json +{ + "version": "0.2.0", + "configurations": [ + { "type": "node", "request": "attach", "name": "Harper main", "port": 9229 }, + { "type": "node", "request": "attach", "name": "Harper worker 1", "port": 9230 }, + { "type": "node", "request": "attach", "name": "Harper worker 2", "port": 9231 } + ], + "compounds": [ + { "name": "Harper (all threads)", "configurations": ["Harper main", "Harper worker 1", "Harper worker 2"] } + ] +} +``` + +Run the compound configuration to attach to every thread at once. + +## Debugging Remote Instances + +Inspector ports must remain on `127.0.0.1` in production. To reach them from a developer workstation, tunnel each port over SSH: + +```bash +ssh -L 9229:127.0.0.1:9229 \ + -L 9230:127.0.0.1:9230 \ + -L 9231:127.0.0.1:9231 \ + harper.example.com +``` + +Then point Chrome DevTools or VS Code at `localhost:9229–9231` as if they were local. + +## Waiting for the Debugger at Startup + +When a bug only reproduces during component initialization, set `waitForDebugger: true`. Each thread starts paused on its first line until a debugger attaches and resumes execution. This is also the safest way to debug an initialization sequence that completes too quickly to manually attach. + +```yaml +threads: + debug: + startingPort: 9230 + waitForDebugger: true +``` + +**Health checks and load balancers** will fail while the threads are paused — only enable `waitForDebugger` in dedicated debug environments. + +## Heap Snapshots Near the Limit + +### `threads.heapSnapshotNearLimit` + +Type: `boolean`  •  Default: `false` + +When the V8 heap approaches the limit set by `threads.maxHeapMemory`, the thread writes a `.heapsnapshot` file to the Harper root directory before the process exits with an out-of-memory error. The snapshot can be loaded into Chrome DevTools (Memory tab → **Load profile**) to identify retained objects responsible for the leak. + +```yaml +threads: + maxHeapMemory: 1024 + heapSnapshotNearLimit: true +``` + +Snapshots can be large (often a sizable fraction of the heap limit) and writing them blocks the thread briefly — leave disabled for normal operation and enable only when investigating an out-of-memory pattern. + +## Related + +- [Configuration Options — `threads`](./options.md#threads) — full thread configuration reference +- [Architecture Overview](../database/overview.md#architecture-overview) — how worker threads fit into Harper +- [Node.js Inspector documentation](https://nodejs.org/en/learn/getting-started/debugging) — debugger protocol details diff --git a/reference/configuration/options.md b/reference/configuration/options.md index 4bee19cc..fc7cdb20 100644 --- a/reference/configuration/options.md +++ b/reference/configuration/options.md @@ -63,8 +63,8 @@ threads: - `count` — Number of worker threads; _Default_: CPU count minus one - `maxHeapMemory` — Heap limit per thread (MB) -- `heapSnapshotNearLimit` — Take heap snapshot when approaching limit -- `debug` — Enable debugging; sub-options: `port`, `startingPort`, `host`, `waitForDebugger` +- `heapSnapshotNearLimit` — Write a `.heapsnapshot` file when a thread nears its heap limit (loadable in Chrome DevTools Memory tab); _Default_: `false`. See [Worker Thread Debugging](./debugging.md#heap-snapshots-near-the-limit) +- `debug` — Enable Node.js inspector; sub-options: `port`, `startingPort`, `host`, `waitForDebugger`. See [Worker Thread Debugging](./debugging.md) --- @@ -212,7 +212,7 @@ replication: ## `storage` -Database storage configuration. See [Database Overview](../database/overview.md) and [Compaction](../database/compaction.md). +Database storage configuration. See [Storage Tuning](../database/storage-tuning.md) for guidance on tuning these options for production workloads, [Database Overview](../database/overview.md) for general database concepts, and [Compaction](../database/compaction.md) for reclaiming space inside existing files. ```yaml storage: @@ -235,7 +235,9 @@ storage: - `path` — Database files directory; _Default_: `/database` - `blobPaths` — Blob storage directory or directories; _Default_: `/blobs` (Added in: v4.5.0) - `pageSize` — Database page size (bytes); _Default_: OS default -- `reclamation.threshold` / `reclamation.interval` / `reclamation.evictionFactor` — Background storage reclamation settings (Added in: v4.5.0) +- `reclamation.threshold` — Free-space ratio below which reclamation begins evicting from caching tables; _Default_: `0.4` (Added in: v4.5.0) +- `reclamation.interval` — Free-space check interval; _Default_: `1h` +- `reclamation.evictionFactor` — Heuristic factor for early eviction under disk pressure; _Default_: `100000`. See [Storage Tuning — Reclamation](../database/storage-tuning.md#storage-reclamation) --- @@ -270,7 +272,9 @@ analytics: ``` - `aggregatePeriod` — Aggregation interval (seconds); _Default_: `60` (Added in: v4.5.0) +- `storageInterval` — Aggregation cycles between storage-volume measurements (`0` disables); _Default_: `10` - `replicate` — Replicate analytics data across cluster; _Default_: `false` +- `logging` — Per-subsystem logger override for analytics writes. See [Logging Configuration](../logging/configuration.md#analyticslogging) --- diff --git a/reference/configuration/overview.md b/reference/configuration/overview.md index 27737986..3966744f 100644 --- a/reference/configuration/overview.md +++ b/reference/configuration/overview.md @@ -227,3 +227,10 @@ Safe mode skips: - Loading package-based extensions defined in component configs Built-in plugins (REST, HTTP, operations API, etc.) and the core database are unaffected. + +## See Also + +- [Configuration Options](./options.md) — complete reference for every `harper-config.yaml` key +- [Worker Thread Debugging](./debugging.md) — attaching Node.js inspector to Harper's worker threads +- [Storage Tuning](../database/storage-tuning.md) — production tuning of `storage.*` options +- [Logging Configuration](../logging/configuration.md) — main and per-subsystem log settings diff --git a/reference/database/overview.md b/reference/database/overview.md index a73d6b47..b7269f85 100644 --- a/reference/database/overview.md +++ b/reference/database/overview.md @@ -110,6 +110,7 @@ For deeper coverage of each database feature, see the dedicated pages in this se - **[API](./api.md)** — The `tables`, `databases`, `transaction()`, and `createBlob()` globals for interacting with the database from code - **[Data Loader](./data-loader.md)** — Loading seed or initial data into tables as part of component deployment - **[Storage Algorithm](./storage-algorithm.md)** — How Harper stores data using LMDB with universal indexing and ACID compliance +- **[Storage Tuning](./storage-tuning.md)** — Tuning `storage.*` options for production: durability vs. throughput, compression, blob paths, reclamation - **[Jobs](./jobs.md)** — Asynchronous bulk data operations (CSV import/export, S3 import/export) - **[System Tables](./system-tables.md)** — Harper internal tables for analytics, data loader state, and other system features - **[Compaction](./compaction.md)** — Reducing database file size by eliminating fragmentation and free space diff --git a/reference/database/storage-tuning.md b/reference/database/storage-tuning.md new file mode 100644 index 00000000..d59267e8 --- /dev/null +++ b/reference/database/storage-tuning.md @@ -0,0 +1,209 @@ +--- +title: Storage Tuning +--- + +# Storage Tuning + +Harper's `storage` configuration section controls how database files are written, cached, and reclaimed on disk. Defaults are tuned for safety and balanced throughput; this page covers the knobs that matter for production deployments with specific workload profiles. + +For a quick reference of every option, see [Configuration Options — `storage`](../configuration/options.md#storage). For the underlying mechanics, see [Storage Algorithm](./storage-algorithm.md). + +## Durability vs. Throughput + +### `storage.writeAsync` + +Type: `boolean`  •  Default: `false` + +Disables `fsync` on commit. Writes return as soon as data is queued to the page cache, dramatically increasing throughput on write-heavy workloads. + +**This disables durability guarantees.** A power loss or OS crash between the application commit and the OS flushing pages to disk can lose recently committed transactions. The database itself remains structurally consistent — only the most recent writes are at risk. + +Enable only when: + +- The data is reproducible from an upstream source (e.g., caches with `sourcedFrom`). +- The workload is bulk ingest where some loss on crash is acceptable and the operation can be re-run. +- Replication provides durability — peers acknowledge writes before they could be lost. + +```yaml +storage: + writeAsync: true +``` + +### `storage.maxTransactionQueueTime` + +Type: `duration string`  •  Default: `45s` + +The maximum estimated time a write may wait in the commit queue before Harper rejects new writes with HTTP 503. Acts as backpressure when downstream disk I/O cannot keep up with incoming writes. + +Lower this in latency-sensitive systems where it is better to shed load early than to let request queues grow. Raise it when occasional disk-write bursts are expected and the application can tolerate longer commit latency. + +```yaml +storage: + maxTransactionQueueTime: 30s +``` + +## Compression + +### `storage.compression` + +Type: `boolean | object`  •  Default: `true` + +LZ4 record compression is enabled by default. It typically reduces on-disk size by 2–4× for JSON-like records with modest CPU cost. + +For object form: + +| Property | Type | Description | +| ------------ | -------- | ---------------------------------------------------------------------------------------------------------------------------- | +| `dictionary` | `string` | Path to a [Zstd-style](https://github.com/facebook/zstd) compression dictionary. Training a dictionary on representative records improves the compression ratio for small records. | +| `threshold` | `number` | Records smaller than this many bytes are stored uncompressed. Useful when small records dominate and the overhead of compression headers outweighs gains. | + +```yaml +storage: + compression: + threshold: 256 + dictionary: ~/hdb/keys/records.dict +``` + +Disable entirely for workloads dominated by small numeric records or pre-compressed payloads (e.g., images, video): + +```yaml +storage: + compression: false +``` + +## Blob Storage Paths + +### `storage.blobPaths` + +Type: `string | string[]`  •  Default: `/blobs` + +Blob attributes (declared with `Blob` in `schema.graphql` or written via [`createBlob`](./api.md)) are stored outside the main database files. `blobPaths` accepts a single path or an array — Harper distributes blob writes across the listed paths. + +Common configurations: + +- **Separate fast disk for blobs:** Place `blobPaths` on a higher-bandwidth volume (e.g., NVMe) while keeping the index on a smaller, lower-latency drive. +- **Multiple volumes:** Provide an array to spread storage across drives. Harper picks the path with the most free space for each new blob, providing crude load-balancing without RAID. +- **Storage tiers:** Mount large but slower object storage at one of the paths to absorb older blobs while keeping hot blobs on fast storage. + +```yaml +storage: + blobPaths: + - /mnt/nvme0/harper-blobs + - /mnt/nvme1/harper-blobs +``` + +Blobs are not relocated when `blobPaths` changes — only new blobs honor the updated configuration. Existing blob references continue to resolve at their original path. + +## Read & Write Behavior + +### `storage.prefetchWrites` + +Type: `boolean`  •  Default: `true` + +Before a write transaction commits, Harper loads the affected pages into memory if not already present. This avoids stalling the commit on a page fault. + +Disable only when memory pressure makes the prefetch counterproductive — for example, when transactions touch records on cold pages that are not expected to be re-read soon. + +### `storage.noReadAhead` + +Type: `boolean`  •  Default: `false` + +Advises the OS via `posix_fadvise` not to read ahead beyond the requested pages. Useful for random-access workloads on rotational disks where speculative reads pollute the page cache. Leave at the default for sequential or scan-heavy workloads. + +### `storage.pageSize` + +Type: `number`  •  Default: OS page size (typically 4096 bytes) + +Changes the database page size. Larger pages can reduce write amplification for large records but increase the minimum I/O unit. **Only set this on a fresh database** — existing files cannot be migrated to a different page size. + +### `storage.caching` + +Type: `boolean`  •  Default: `true` + +In-memory record caching of decoded records. Disable to reduce heap usage when records are large and unlikely to be re-read in the same process. + +## Storage Reclamation + +`storage.reclamation` controls how Harper evicts data from caching tables (tables with [`sourcedFrom`](../resources/resource-api.md#sourcedfromresource-options)) when disk usage runs high. Reclamation does **not** affect non-caching tables — those rely on explicit deletion, TTL expiration, or [compaction](./compaction.md). + +### `storage.reclamation.threshold` + +Type: `number` (ratio)  •  Default: `0.4` + +Minimum fraction of the volume that should remain free. When free space falls below this ratio, reclamation begins evicting expired and lightly-used entries from caching tables. A larger value reclaims earlier and more aggressively; a smaller value defers reclamation closer to the volume filling. + +### `storage.reclamation.interval` + +Type: `duration string`  •  Default: `1h` + +How often Harper checks free space against the threshold. Lower intervals catch fast-filling volumes sooner at the cost of more periodic I/O. + +### `storage.reclamation.evictionFactor` + +Type: `number`  •  Default: `100000` + +Tunes the heuristic used to evict entries early when reclamation priority is high. The heuristic considers each entry's remaining time-to-expiration, record size, and how long ago it was last refreshed. Lowering this evicts more aggressively when free space is critical; raising it preserves entries longer. + +```yaml +storage: + reclamation: + threshold: 0.3 # start reclaiming at 30% free + interval: 30m + evictionFactor: 50000 +``` + +For a deeper discussion of how `sourcedFrom` interacts with reclamation, see [Resource API — `sourcedFrom`](../resources/resource-api.md#sourcedfromresource-options). + +## Compaction on Start + +### `storage.compactOnStart` + +Type: `boolean`  •  Default: `false` + +Runs [compaction](./compaction.md) on all non-system databases at startup. Useful when deployments include scheduled restarts and you want to reclaim fragmented space as part of the maintenance window. + +### `storage.compactOnStartKeepBackup` + +Type: `boolean`  •  Default: `false` + +Retains the pre-compaction backup files after `compactOnStart` runs. Recommended for the first few cycles in production while validating compaction behavior; the backups can be removed manually once confidence is established. + +## Workload Recipes + +**Write-heavy ingest, durability via replication:** + +```yaml +storage: + writeAsync: true + prefetchWrites: true + maxTransactionQueueTime: 60s +``` + +**Read-heavy cache layer with large blobs:** + +```yaml +storage: + caching: true + blobPaths: + - /mnt/fast-ssd/harper-blobs + reclamation: + threshold: 0.2 + interval: 15m +``` + +**Memory-constrained edge deployment:** + +```yaml +storage: + caching: false + noReadAhead: true + compression: true +``` + +## Related + +- [Configuration Options](../configuration/options.md) — full list of `storage` options +- [Storage Algorithm](./storage-algorithm.md) — how Harper stores records and indexes on disk +- [Compaction](./compaction.md) — reclaiming space inside existing database files +- [Resource API — `sourcedFrom`](../resources/resource-api.md#sourcedfromresource-options) — caching tables that interact with reclamation +- [Database API — `createBlob`](./api.md) — creating blobs that live under `blobPaths` diff --git a/reference/mqtt/overview.md b/reference/mqtt/overview.md index ffa6537b..c4bfb85a 100644 --- a/reference/mqtt/overview.md +++ b/reference/mqtt/overview.md @@ -49,13 +49,39 @@ Harper supports multi-level topics for both publishing and subscribing: ### Sessions - **Clean sessions** — Subscriptions and queued messages are discarded on disconnect. -- **Durable sessions** — Subscriptions and queued messages are persisted across reconnects. +- **Durable sessions** — Subscriptions and queued messages are persisted across reconnects. See [Durable Sessions](#durable-sessions) below. + +### Durable Sessions + +A durable session retains a client's subscription list and any unacknowledged messages across disconnects. When the client reconnects with the same client ID, it picks up from where it left off — including any messages published while it was offline. + +Durable sessions in Harper are persisted as records in the `hdb_durable_session` system table, indexed by client ID. The session record holds the list of subscriptions (topic + QoS) and the timestamp of the last delivered message per topic. Because durable sessions are records rather than in-memory state, an abandoned session sits idle with no runtime cost until the client reconnects or the record is deleted. + +**Establishing a durable session** — Connect with a stable client ID and `cleanSession: false` (MQTT v3.1.1) or `cleanStart: false` (MQTT v5): + +```javascript +// MQTT v5 with the `mqtt` npm package +mqtt.connect('mqtts://harper.example.com:8883', { + clientId: 'sensor-42', + clean: false, // request a durable session + protocolVersion: 5, + properties: { + sessionExpiryInterval: 86400, // keep session for 24h after disconnect + }, +}); +``` + +**Catch-up on reconnect** — When the client reconnects, Harper replays missed messages on subscribed topics by reading the audit log. For this to work, audit logging must be enabled on the tables backing the subscribed topics. See [Transaction Logging](../database/transaction.md) and [`logging.auditLog`](../logging/configuration.md#loggingauditlog). + +**Session expiry** — In MQTT v5, the `sessionExpiryInterval` property on `CONNECT` controls how long the session is retained after the client disconnects. With `sessionExpiryInterval: 0` (or a clean session connect), Harper deletes the session record at disconnect. Connecting with the same client ID and `clean: true` also explicitly deletes any existing durable session. + +**Distributed durable sessions** — Harper does not currently maintain a single logical durable session across multiple cluster nodes; a durable session record lives on the node where it was created. Clients that may reconnect to different nodes (e.g., behind a load balancer with round-robin DNS) should use sticky routing or terminate on a fixed node. ### Last Will -Harper supports the MQTT Last Will and Testament feature. If a client disconnects unexpectedly, the broker publishes the configured will message on its behalf. +Harper supports the MQTT Last Will and Testament feature. If a client disconnects unexpectedly, the broker publishes the configured will message on its behalf. Will messages are persisted in the `hdb_session_will` system table at CONNECT time, so they survive a broker restart and fire reliably on unexpected disconnect. ## Content Negotiation diff --git a/sidebarsReference.ts b/sidebarsReference.ts index a4061285..c7457f84 100644 --- a/sidebarsReference.ts +++ b/sidebarsReference.ts @@ -39,6 +39,11 @@ const sidebars: SidebarsConfig = { id: 'database/storage-algorithm', label: 'Storage Algorithm', }, + { + type: 'doc', + id: 'database/storage-tuning', + label: 'Storage Tuning', + }, { type: 'doc', id: 'database/jobs', @@ -308,6 +313,11 @@ const sidebars: SidebarsConfig = { id: 'configuration/operations', label: 'Operations', }, + { + type: 'doc', + id: 'configuration/debugging', + label: 'Worker Thread Debugging', + }, ], }, {