From fe0678fac25806d78b7320af2f2f2b6830754da1 Mon Sep 17 00:00:00 2001 From: Keith Solomon Date: Sun, 17 May 2026 14:05:25 -0500 Subject: [PATCH] feat: add sqlite catalog web app --- .gitignore | 1 + README.md | 17 + config.example.yaml | 4 + .../plans/2026-05-17-sqlite-web-app.md | 86 +++ .../specs/2026-05-17-sqlite-web-app-design.md | 2 +- package-lock.json | 624 +++++++++++++++++- package.json | 2 + scripts/smoke.mjs | 1 + src/cli/program.ts | 19 + src/config/config.ts | 6 + src/database/schema.ts | 63 ++ src/database/store.ts | 202 ++++++ src/output/databaseWriter.ts | 23 + src/output/sheets.ts | 2 +- src/parsing/generic.ts | 8 + src/run/runCatalog.ts | 27 +- src/web/app.ts | 101 +++ src/web/views.ts | 63 ++ tests/config.test.ts | 2 + tests/database.test.ts | 90 +++ tests/run.test.ts | 42 ++ tests/web.test.ts | 79 +++ 22 files changed, 1452 insertions(+), 12 deletions(-) create mode 100644 docs/superpowers/plans/2026-05-17-sqlite-web-app.md create mode 100644 src/database/schema.ts create mode 100644 src/database/store.ts create mode 100644 src/output/databaseWriter.ts create mode 100644 src/web/app.ts create mode 100644 src/web/views.ts create mode 100644 tests/database.test.ts create mode 100644 tests/web.test.ts diff --git a/.gitignore b/.gitignore index 41f4829..6fea4ce 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ node_modules dist config.yaml output +data diff --git a/README.md b/README.md index 9696294..706b16e 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ nlc run nlc run --from 2026-05-01 --to 2026-05-16 nlc run --last 30d nlc run --enrich-only +nlc serve ``` ## Setup @@ -22,6 +23,8 @@ nlc run --enrich-only - `~/.nlc/gmail-credentials.json` - `~/.nlc/sheets-credentials.json` 5. Run `node dist/index.js run --dry-run` before live writes. +6. Run `node dist/index.js run` to import into SQLite. +7. Run `node dist/index.js serve` and open . Tokens are persisted locally under `~/.nlc` and must not be committed. @@ -81,9 +84,23 @@ Start from [config.example.yaml](config.example.yaml). The important choices are - `gmail.folder`: the single Gmail label/folder to process. - `output.excel.enabled`: writes a local `.xlsx` file. - `output.sheets_api.enabled`: enables Google Sheets integration when credentials and spreadsheet ID are configured. +- `database.enabled`: writes SQLite catalog data during `nlc run`; defaults to `true`. +- `database.path`: SQLite database path; defaults to `./data/newsletter-catalog.sqlite`. - `links.tracking_params`: query parameters stripped during URL normalization. - `categories.llm`: optional BYOK categorization provider. +## SQLite and Web App + +SQLite is the default catalog store. `nlc run` writes imported links, sponsors, dead links, and run history to the configured database even when spreadsheet outputs are disabled. + +Start the local read-only web app with: + +```bash +nlc serve --host 127.0.0.1 --port 3000 +``` + +The first web UI is intentionally functional: dashboard, links, sponsored links, dead links, and runs. + ## Build and Distribution The build uses `tsup` for the JavaScript bundle and `@yao-pkg/pkg` for the standalone executable: diff --git a/config.example.yaml b/config.example.yaml index 22cf2e8..86d34a4 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -14,6 +14,10 @@ output: enabled: true path: './output/newsletter-catalog.xlsx' +database: + enabled: true + path: './data/newsletter-catalog.sqlite' + newsletters: 'sender@example.com': display_name: 'Example Newsletter' diff --git a/docs/superpowers/plans/2026-05-17-sqlite-web-app.md b/docs/superpowers/plans/2026-05-17-sqlite-web-app.md new file mode 100644 index 0000000..f271e0a --- /dev/null +++ b/docs/superpowers/plans/2026-05-17-sqlite-web-app.md @@ -0,0 +1,86 @@ +# SQLite Web App Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make SQLite the default catalog store and add a functional local web app for browsing imported newsletter data. + +**Architecture:** Use Node's built-in `node:sqlite` `DatabaseSync` API for a packaged-executable-friendly SQLite layer. Keep the existing Gmail ingestion pipeline and spreadsheet writers, add a `DatabaseWriter` as the default writer, and add an Express server that renders read-only catalog pages from SQLite. + +**Tech Stack:** TypeScript, Node.js `node:sqlite`, Express, Commander, Vitest, existing parser/output modules. + +--- + +### Task 1: Config Defaults + +**Files:** + +- Modify: `src/config/config.ts` +- Modify: `config.example.yaml` +- Test: `tests/config.test.ts` + +- [ ] Write a failing test that `database.enabled` defaults to `true` and `database.path` defaults to `./data/newsletter-catalog.sqlite`. +- [ ] Add the database schema to config validation. +- [ ] Add `database` to `config.example.yaml`. +- [ ] Run `npm test -- tests/config.test.ts`. + +### Task 2: SQLite Schema and Store + +**Files:** + +- Create: `src/database/schema.ts` +- Create: `src/database/store.ts` +- Test: `tests/database.test.ts` + +- [ ] Write failing tests that migrations create `newsletters`, `issues`, `links`, `link_occurrences`, `sponsors`, `dead_links`, and `runs`. +- [ ] Write failing tests that saving a catalog payload deduplicates canonical URLs and stores occurrences per issue. +- [ ] Implement migration and transactional persistence with `node:sqlite`. +- [ ] Run `npm test -- tests/database.test.ts`. + +### Task 3: Default Database Writer + +**Files:** + +- Create: `src/output/databaseWriter.ts` +- Modify: `src/run/runCatalog.ts` +- Modify: `src/cli/program.ts` +- Test: `tests/run.test.ts` + +- [ ] Write a failing test that `runCatalog` can write rows to a database writer and still preserves spreadsheet writer behavior. +- [ ] Add a `DatabaseWriter` implementing the existing `OutputWriter` interface. +- [ ] Make `nlc run` add `DatabaseWriter` by default when `database.enabled` is true. +- [ ] Run `npm test -- tests/run.test.ts tests/database.test.ts`. + +### Task 4: Web App + +**Files:** + +- Create: `src/web/app.ts` +- Create: `src/web/views.ts` +- Modify: `src/cli/program.ts` +- Test: `tests/web.test.ts` + +- [ ] Write failing tests that dashboard, links, sponsored links, dead links, and runs routes render data from a fixture database. +- [ ] Add `nlc serve --config --host --port`. +- [ ] Implement read-only Express routes and simple server-rendered HTML. +- [ ] Run `npm test -- tests/web.test.ts`. + +### Task 5: Docs and Verification + +**Files:** + +- Modify: `README.md` +- Modify: `package.json` + +- [ ] Document SQLite default storage and `nlc serve`. +- [ ] Add Express dependencies and update lockfile. +- [ ] Run `npm install`. +- [ ] Run `npm run lint`. +- [ ] Run `npm run format:check`. +- [ ] Run `npm run typecheck`. +- [ ] Run `npm test`. +- [ ] Run `npm run build`. +- [ ] Run `npm run smoke`. + +## Self-Review + +This plan covers all accepted spec requirements: SQLite defaults, schema, default import persistence, optional spreadsheet output, `nlc serve`, functional catalog views, docs, and automated tests. No unresolved placeholders remain. Types and paths align with the existing project structure. diff --git a/docs/superpowers/specs/2026-05-17-sqlite-web-app-design.md b/docs/superpowers/specs/2026-05-17-sqlite-web-app-design.md index 2e297d8..322cc40 100644 --- a/docs/superpowers/specs/2026-05-17-sqlite-web-app-design.md +++ b/docs/superpowers/specs/2026-05-17-sqlite-web-app-design.md @@ -52,7 +52,7 @@ Extend `config.yaml`: ```yaml database: enabled: true - path: "./data/newsletter-catalog.sqlite" + path: './data/newsletter-catalog.sqlite' ``` Behavior: diff --git a/package-lock.json b/package-lock.json index a17b85a..9b8e7f0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@commander-js/extra-typings": "^12.1.0", "cheerio": "^1.0.0", "commander": "^12.1.0", + "express": "^5.2.1", "googleapis": "^140.0.1", "ora": "^8.1.1", "xlsx": "^0.18.5", @@ -22,6 +23,7 @@ "nlc": "dist/index.js" }, "devDependencies": { + "@types/express": "^5.0.6", "@types/node": "^22.9.0", "@typescript-eslint/eslint-plugin": "^8.11.0", "@typescript-eslint/parser": "^8.11.0", @@ -1244,6 +1246,27 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/body-parser": { + "version": "1.19.6", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz", + "integrity": "sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "node_modules/@types/connect": { + "version": "3.4.38", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", + "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -1251,6 +1274,38 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/express": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.6.tgz", + "integrity": "sha512-sKYVuV7Sv9fbPIt/442koC7+IIwK5olP1KWeD88e/idgoJqDm3JV/YUiPwkoKK92ylff2MGxSz1CSjsXelx0YA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^5.0.0", + "@types/serve-static": "^2" + } + }, + "node_modules/@types/express-serve-static-core": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.1.1.tgz", + "integrity": "sha512-v4zIMr/cX7/d2BpAEX3KNKL/JrT1s43s96lLvvdTmza1oEvDudCqK9aF/djc/SWgy8Yh0h30TZx5VpzqFCxk5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "node_modules/@types/http-errors": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.5.tgz", + "integrity": "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/json5": { "version": "0.0.29", "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", @@ -1268,6 +1323,41 @@ "undici-types": "~6.21.0" } }, + "node_modules/@types/qs": { + "version": "6.15.1", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.15.1.tgz", + "integrity": "sha512-GZHUBZR9hckSUhrxmp1nG6NwdpM9fCunJwyThLW1X3AyHgd9IlHb6VANpQQqDr2o/qQp6McZ3y/IA2rVzKzSbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/range-parser": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", + "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@types/send/-/send-1.2.1.tgz", + "integrity": "sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/serve-static": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-2.2.0.tgz", + "integrity": "sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/http-errors": "*", + "@types/node": "*" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "8.59.3", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.59.3.tgz", @@ -1737,6 +1827,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/acorn": { "version": "8.16.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", @@ -2163,6 +2266,46 @@ "dev": true, "license": "MIT" }, + "node_modules/body-parser": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.2.tgz", + "integrity": "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^1.0.5", + "debug": "^4.4.3", + "http-errors": "^2.0.0", + "iconv-lite": "^0.7.0", + "on-finished": "^2.4.1", + "qs": "^6.14.1", + "raw-body": "^3.0.1", + "type-is": "^2.0.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/body-parser/node_modules/iconv-lite": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", + "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/boolbase": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", @@ -2229,6 +2372,15 @@ "esbuild": ">=0.18" } }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/cac": { "version": "6.7.14", "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", @@ -2552,6 +2704,46 @@ "node": "^14.18.0 || >=16.10.0" } }, + "node_modules/content-disposition": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", + "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, "node_modules/core-util-is": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", @@ -2764,6 +2956,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/detect-libc": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", @@ -2915,12 +3116,27 @@ "safe-buffer": "^5.0.1" } }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, "node_modules/emoji-regex": { "version": "10.6.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", "license": "MIT" }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/encoding-sniffer": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.1.tgz", @@ -3161,6 +3377,12 @@ "node": ">=6" } }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, "node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -3567,6 +3789,15 @@ "node": ">=0.10.0" } }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/events-universal": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz", @@ -3597,6 +3828,49 @@ "node": ">=12.0.0" } }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -3672,6 +3946,27 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -3739,6 +4034,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/frac": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz", @@ -3748,6 +4052,15 @@ "node": ">=0.8" } }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", @@ -4273,6 +4586,26 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/https-proxy-agent": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", @@ -4372,7 +4705,6 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true, "license": "ISC" }, "node_modules/ini": { @@ -4410,6 +4742,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, "node_modules/is-array-buffer": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.5.tgz", @@ -4679,6 +5020,12 @@ "node": ">=8" } }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, "node_modules/is-regex": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", @@ -5125,6 +5472,52 @@ "node": ">= 0.4" } }, + "node_modules/media-typer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/mimic-function": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/mimic-function/-/mimic-function-5.0.1.tgz", @@ -5295,6 +5688,15 @@ "dev": true, "license": "MIT" }, + "node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/node-abi": { "version": "3.92.0", "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.92.0.tgz", @@ -5498,11 +5900,22 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, "license": "ISC", "dependencies": { "wrappy": "1" @@ -5715,6 +6128,15 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -5752,6 +6174,16 @@ "dev": true, "license": "MIT" }, + "node_modules/path-to-regexp": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/pathe": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", @@ -6027,6 +6459,19 @@ "node": ">=0.4.0" } }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/pump": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", @@ -6084,6 +6529,46 @@ ], "license": "MIT" }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/raw-body/node_modules/iconv-lite": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", + "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/rc": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", @@ -6326,6 +6811,22 @@ "fsevents": "~2.3.2" } }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -6444,6 +6945,51 @@ "node": ">=10" } }, + "node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/set-function-length": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", @@ -6493,6 +7039,12 @@ "node": ">= 0.4" } }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -6693,6 +7245,15 @@ "dev": true, "license": "MIT" }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/std-env": { "version": "3.10.0", "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", @@ -7152,6 +7713,15 @@ "node": ">=14.0.0" } }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", @@ -7806,6 +8376,37 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/type-is": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", + "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", + "license": "MIT", + "dependencies": { + "content-type": "^2.0.0", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/type-is/node_modules/content-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", + "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/typed-array-buffer": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.3.tgz", @@ -7950,6 +8551,15 @@ "node": ">= 10.0.0" } }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/unzipper": { "version": "0.12.3", "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.12.3.tgz", @@ -8001,6 +8611,15 @@ "uuid": "dist/bin/uuid" } }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/vite": { "version": "5.4.21", "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.21.tgz", @@ -8826,7 +9445,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true, "license": "ISC" }, "node_modules/xlsx": { diff --git a/package.json b/package.json index afa06f0..58bdf7a 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "@commander-js/extra-typings": "^12.1.0", "cheerio": "^1.0.0", "commander": "^12.1.0", + "express": "^5.2.1", "googleapis": "^140.0.1", "ora": "^8.1.1", "xlsx": "^0.18.5", @@ -35,6 +36,7 @@ "zod": "^3.23.8" }, "devDependencies": { + "@types/express": "^5.0.6", "@types/node": "^22.9.0", "@typescript-eslint/eslint-plugin": "^8.11.0", "@typescript-eslint/parser": "^8.11.0", diff --git a/scripts/smoke.mjs b/scripts/smoke.mjs index 724a8dd..ca636f4 100644 --- a/scripts/smoke.mjs +++ b/scripts/smoke.mjs @@ -28,6 +28,7 @@ state_file: ${JSON.stringify(join(dir, 'state.json'))} await exec(binary, ['--help']); await exec('node', [cli, 'init', '--help']); await exec('node', [cli, 'run', '--help']); + await exec('node', [cli, 'serve', '--help']); await exec('node', [cli, 'run', '--config', config, '--dry-run'], { env: { ...process.env, NLC_FIXTURE: '1' } }); diff --git a/src/cli/program.ts b/src/cli/program.ts index 214e9b8..24560e8 100644 --- a/src/cli/program.ts +++ b/src/cli/program.ts @@ -2,10 +2,12 @@ import { Command, Option } from 'commander'; import { writeFile } from 'node:fs/promises'; import { loadConfig } from '../config/config.js'; import { createGmailClient } from '../gmail/client.js'; +import { DatabaseWriter } from '../output/databaseWriter.js'; import { ExcelWriter } from '../output/excel.js'; import { createGoogleSheetsWriter } from '../output/googleSheets.js'; import { OutputWriter } from '../output/sheets.js'; import { runCatalog } from '../run/runCatalog.js'; +import { createWebApp } from '../web/app.js'; import { validateDateFilters } from './flags.js'; const sampleConfig = `gmail: @@ -79,6 +81,20 @@ export function createProgram(): Command { console.log(JSON.stringify(summary, null, 2)); }); + program + .command('serve') + .description('Start a local web app for browsing the SQLite catalog') + .option('--config ', 'Config path', './config.yaml') + .option('--host ', 'Host to bind', '127.0.0.1') + .option('--port ', 'Port to bind', (value) => Number(value), 3000) + .action(async (options) => { + const config = await loadConfig(options.config); + const app = createWebApp(config.database.path); + app.listen(options.port, options.host, () => { + console.log(`Newsletter Link Catalog listening at http://${options.host}:${options.port}`); + }); + }); + return program; } @@ -86,6 +102,9 @@ async function createWriters( config: Awaited> ): Promise { const writers: OutputWriter[] = []; + if (config.database.enabled) { + writers.push(new DatabaseWriter(config.database.path)); + } if (config.output.excel.enabled) { writers.push(new ExcelWriter(config.output.excel.path)); } diff --git a/src/config/config.ts b/src/config/config.ts index d52cc12..fb6ce1c 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -81,6 +81,12 @@ const configSchema = z linkConcurrency: z.number().int().positive().default(3) }) .default({}), + database: z + .object({ + enabled: z.boolean().default(true), + path: z.string().default('./data/newsletter-catalog.sqlite') + }) + .default({}), stateFile: z.string().default('~/.nlc/state.json'), plugins: z.record(z.string(), z.any()).default({}) }) diff --git a/src/database/schema.ts b/src/database/schema.ts new file mode 100644 index 0000000..1209e73 --- /dev/null +++ b/src/database/schema.ts @@ -0,0 +1,63 @@ +export const catalogSchema = ` +PRAGMA foreign_keys = ON; + +CREATE TABLE IF NOT EXISTS newsletters ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE +); + +CREATE TABLE IF NOT EXISTS issues ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + newsletter_id INTEGER NOT NULL REFERENCES newsletters(id) ON DELETE CASCADE, + issue_date TEXT NOT NULL, + title TEXT NOT NULL DEFAULT '', + message_id TEXT, + UNIQUE(newsletter_id, issue_date, title) +); + +CREATE TABLE IF NOT EXISTS links ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT NOT NULL UNIQUE +); + +CREATE TABLE IF NOT EXISTS link_occurrences ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + issue_id INTEGER NOT NULL REFERENCES issues(id) ON DELETE CASCADE, + link_id INTEGER NOT NULL REFERENCES links(id) ON DELETE CASCADE, + category TEXT NOT NULL DEFAULT '', + title TEXT NOT NULL DEFAULT '', + description TEXT NOT NULL DEFAULT '', + page_title_meta TEXT NOT NULL DEFAULT '', + also_in TEXT NOT NULL DEFAULT '', + UNIQUE(issue_id, link_id) +); + +CREATE TABLE IF NOT EXISTS sponsors ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + issue_id INTEGER REFERENCES issues(id) ON DELETE SET NULL, + link_id INTEGER REFERENCES links(id) ON DELETE SET NULL, + newsletter TEXT NOT NULL DEFAULT '', + sponsor TEXT NOT NULL DEFAULT '', + description TEXT NOT NULL DEFAULT '' +); + +CREATE TABLE IF NOT EXISTS dead_links ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + link_id INTEGER REFERENCES links(id) ON DELETE SET NULL, + url TEXT NOT NULL, + status TEXT NOT NULL DEFAULT '', + source TEXT NOT NULL DEFAULT '', + date TEXT NOT NULL DEFAULT '' +); + +CREATE TABLE IF NOT EXISTS runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + started_at TEXT NOT NULL, + mode TEXT NOT NULL DEFAULT '', + newsletters_processed INTEGER NOT NULL DEFAULT 0, + links_extracted INTEGER NOT NULL DEFAULT 0, + sponsors INTEGER NOT NULL DEFAULT 0, + dead_links INTEGER NOT NULL DEFAULT 0, + errors INTEGER NOT NULL DEFAULT 0 +); +`; diff --git a/src/database/store.ts b/src/database/store.ts new file mode 100644 index 0000000..45777a4 --- /dev/null +++ b/src/database/store.ts @@ -0,0 +1,202 @@ +import { mkdirSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { dirname } from 'node:path'; +import { CatalogPayload } from '../output/sheets.js'; +import { catalogSchema } from './schema.js'; + +const require = createRequire(import.meta.url); +const sqlite = require('node:sqlite') as typeof import('node:sqlite'); + +export interface CatalogRunPayload extends CatalogPayload { + mode: string; + newslettersProcessed: number; + linksExtracted: number; + sponsorCount: number; + deadLinkCount: number; + errors: number; +} + +export class CatalogDatabase { + private readonly db: import('node:sqlite').DatabaseSync; + + public constructor(private readonly path: string) { + if (path !== ':memory:') { + mkdirSync(dirname(path), { recursive: true }); + } + this.db = new sqlite.DatabaseSync(path); + this.db.exec('PRAGMA foreign_keys = ON'); + } + + public migrate(): void { + this.db.exec(catalogSchema); + } + + public close(): void { + this.db.close(); + } + + public tableNames(): string[] { + return this.db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name" + ) + .all() + .map((row: any) => row.name); + } + + public count(table: string): number { + const safeTable = table.replace(/[^a-z_]/g, ''); + const row = this.db.prepare(`SELECT COUNT(*) AS count FROM ${safeTable}`).get() as { + count: number; + }; + return row.count; + } + + public saveCatalogRun(payload: CatalogRunPayload): void { + this.migrate(); + this.db.exec('BEGIN'); + try { + this.insertRun(payload); + for (const row of payload.rows) { + this.insertOccurrence(row); + } + for (const sponsor of payload.sponsors) { + this.insertSponsor(sponsor); + } + for (const deadLink of payload.deadLinks) { + this.insertDeadLink(deadLink); + } + this.db.exec('COMMIT'); + } catch (error) { + this.db.exec('ROLLBACK'); + throw error; + } + } + + public dashboardCounts(): Record { + return { + newsletters: this.count('newsletters'), + issues: this.count('issues'), + links: this.count('links'), + sponsors: this.count('sponsors'), + deadLinks: this.count('dead_links'), + runs: this.count('runs') + }; + } + + public contentLinks(): any[] { + return this.db + .prepare( + `SELECT n.name AS newsletter, i.issue_date AS issueDate, o.category, o.title, l.url, o.description + FROM link_occurrences o + JOIN issues i ON i.id = o.issue_id + JOIN newsletters n ON n.id = i.newsletter_id + JOIN links l ON l.id = o.link_id + ORDER BY i.issue_date DESC, n.name, o.title` + ) + .all(); + } + + public sponsoredLinks(): any[] { + return this.db + .prepare('SELECT newsletter, sponsor, description FROM sponsors ORDER BY newsletter, sponsor') + .all(); + } + + public deadLinks(): any[] { + return this.db + .prepare('SELECT url, status, source, date FROM dead_links ORDER BY date DESC, url') + .all(); + } + + public runs(): any[] { + return this.db.prepare('SELECT * FROM runs ORDER BY started_at DESC, id DESC').all(); + } + + private insertRun(payload: CatalogRunPayload): void { + this.db + .prepare( + `INSERT INTO runs + (started_at, mode, newsletters_processed, links_extracted, sponsors, dead_links, errors) + VALUES (?, ?, ?, ?, ?, ?, ?)` + ) + .run( + new Date().toISOString(), + payload.mode, + payload.newslettersProcessed, + payload.linksExtracted, + payload.sponsorCount, + payload.deadLinkCount, + payload.errors + ); + } + + private insertOccurrence(row: Record): void { + const newsletterId = this.upsertNewsletter(String(row['Source Newsletter'] ?? 'Newsletter')); + const issueId = this.upsertIssue(newsletterId, String(row['Issue Date'] ?? ''), ''); + const linkId = this.upsertLink(String(row['Link URL'] ?? '')); + this.db + .prepare( + `INSERT INTO link_occurrences + (issue_id, link_id, category, title, description, page_title_meta, also_in) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(issue_id, link_id) DO UPDATE SET + category = excluded.category, + title = excluded.title, + description = excluded.description, + page_title_meta = excluded.page_title_meta, + also_in = excluded.also_in` + ) + .run( + issueId, + linkId, + String(row.Category ?? ''), + String(row.Title ?? ''), + String(row.Description ?? ''), + String(row['Page Title + Meta'] ?? ''), + String(row['Also In'] ?? '') + ); + } + + private insertSponsor(row: Record): void { + const newsletter = String(row.Newsletter ?? ''); + const linkId = this.upsertLink(String(row.Link ?? '')); + this.db + .prepare( + `INSERT INTO sponsors (issue_id, link_id, newsletter, sponsor, description) + VALUES (?, ?, ?, ?, ?)` + ) + .run(null, linkId, newsletter, String(row.Sponsor ?? ''), String(row.Description ?? '')); + } + + private insertDeadLink(row: Record): void { + const url = String(row.URL ?? ''); + const linkId = this.upsertLink(url); + this.db + .prepare('INSERT INTO dead_links (link_id, url, status, source, date) VALUES (?, ?, ?, ?, ?)') + .run(linkId, url, String(row.Status ?? ''), String(row.Source ?? ''), String(row.Date ?? '')); + } + + private upsertNewsletter(name: string): number { + this.db.prepare('INSERT OR IGNORE INTO newsletters (name) VALUES (?)').run(name); + return ( + this.db.prepare('SELECT id FROM newsletters WHERE name = ?').get(name) as { id: number } + ).id; + } + + private upsertIssue(newsletterId: number, issueDate: string, title: string): number { + this.db + .prepare('INSERT OR IGNORE INTO issues (newsletter_id, issue_date, title) VALUES (?, ?, ?)') + .run(newsletterId, issueDate, title); + return ( + this.db + .prepare('SELECT id FROM issues WHERE newsletter_id = ? AND issue_date = ? AND title = ?') + .get(newsletterId, issueDate, title) as { id: number } + ).id; + } + + private upsertLink(url: string): number { + this.db.prepare('INSERT OR IGNORE INTO links (url) VALUES (?)').run(url); + return (this.db.prepare('SELECT id FROM links WHERE url = ?').get(url) as { id: number }).id; + } +} diff --git a/src/output/databaseWriter.ts b/src/output/databaseWriter.ts new file mode 100644 index 0000000..4533ef4 --- /dev/null +++ b/src/output/databaseWriter.ts @@ -0,0 +1,23 @@ +import { CatalogDatabase } from '../database/store.js'; +import { CatalogPayload, OutputWriter } from './sheets.js'; + +export class DatabaseWriter implements OutputWriter { + public constructor(private readonly path: string) {} + + public async write(payload: CatalogPayload, summary: Record = {}): Promise { + const db = new CatalogDatabase(this.path); + try { + db.saveCatalogRun({ + mode: String(summary.mode ?? 'run'), + newslettersProcessed: Number(summary.newslettersProcessed ?? 0), + linksExtracted: Number(summary.linksExtracted ?? payload.rows.length), + sponsorCount: Number(summary.sponsors ?? payload.sponsors.length), + deadLinkCount: Number(summary.deadLinks ?? payload.deadLinks.length), + errors: Number(summary.errors ?? 0), + ...payload + }); + } finally { + db.close(); + } + } +} diff --git a/src/output/sheets.ts b/src/output/sheets.ts index a34a527..96ff327 100644 --- a/src/output/sheets.ts +++ b/src/output/sheets.ts @@ -19,5 +19,5 @@ export interface CatalogPayload { } export interface OutputWriter { - write(payload: CatalogPayload): Promise; + write(payload: CatalogPayload, summary?: Record): Promise; } diff --git a/src/parsing/generic.ts b/src/parsing/generic.ts index 0531b79..58281eb 100644 --- a/src/parsing/generic.ts +++ b/src/parsing/generic.ts @@ -32,6 +32,10 @@ function sponsorMarkerText(value: string): string | undefined { } function blockTokens($: cheerio.CheerioAPI, node: any): Token[] { + if (!node) { + return []; + } + if (node.type === 'text') { const text = compactText(node.data ?? ''); return text ? [{ type: 'text', text }] : []; @@ -49,6 +53,10 @@ function blockTokens($: cheerio.CheerioAPI, node: any): Token[] { function localContext($: cheerio.CheerioAPI, element: any, title: string): string { const block = $(element).closest('p,li,td,div').first(); + if (block.length === 0) { + return title; + } + const tokens = blockTokens($, block.get(0)); const anchorIndex = tokens.findIndex( (token) => token.type === 'anchor' && token.element === element diff --git a/src/run/runCatalog.ts b/src/run/runCatalog.ts index edb024a..f77a801 100644 --- a/src/run/runCatalog.ts +++ b/src/run/runCatalog.ts @@ -47,6 +47,16 @@ function escapeRegExp(value: string): string { return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); } +function runMode(options: RunOptions): string { + if (options.full) { + return 'full'; + } + if (options.dryRun) { + return 'dry-run'; + } + return 'incremental'; +} + export async function runCatalog(options: RunOptions): Promise { const config = normalizeConfig(options.config); const state = new StateStore(config.stateFile); @@ -113,17 +123,20 @@ export async function runCatalog(options: RunOptions): Promise { } } - if (!options.dryRun) { - for (const writer of options.writers) { - await writer.write({ rows, sponsors, deadLinks: [] }); - } - } - - return { + const summary = { + mode: runMode(options), newslettersProcessed: messages.length, linksExtracted: rows.length, sponsors: sponsors.length, deadLinks: 0, errors }; + + if (!options.dryRun) { + for (const writer of options.writers) { + await writer.write({ rows, sponsors, deadLinks: [] }, summary); + } + } + + return summary; } diff --git a/src/web/app.ts b/src/web/app.ts new file mode 100644 index 0000000..b8b4228 --- /dev/null +++ b/src/web/app.ts @@ -0,0 +1,101 @@ +import express from 'express'; +import { CatalogDatabase } from '../database/store.js'; +import { dashboard, page, table } from './views.js'; + +export function createWebApp(databasePath: string) { + const app = express(); + + app.get('/', (_req, res, next) => { + withDatabase(databasePath, (db) => res.send(dashboard(db.dashboardCounts()))).catch(next); + }); + + app.get('/links', (_req, res, next) => { + withDatabase(databasePath, (db) => + res.send( + page( + 'Links', + `

Links

${table(db.contentLinks(), [ + ['newsletter', 'Newsletter'], + ['issueDate', 'Issue Date'], + ['category', 'Category'], + ['title', 'Title'], + ['url', 'URL'], + ['description', 'Description'] + ])}` + ) + ) + ).catch(next); + }); + + app.get('/sponsors', (_req, res, next) => { + withDatabase(databasePath, (db) => + res.send( + page( + 'Sponsored Links', + `

Sponsored Links

${table(db.sponsoredLinks(), [ + ['newsletter', 'Newsletter'], + ['sponsor', 'Sponsor'], + ['description', 'Description'] + ])}` + ) + ) + ).catch(next); + }); + + app.get('/dead-links', (_req, res, next) => { + withDatabase(databasePath, (db) => + res.send( + page( + 'Dead Links', + `

Dead Links

${table(db.deadLinks(), [ + ['url', 'URL'], + ['status', 'Status'], + ['source', 'Source'], + ['date', 'Date'] + ])}` + ) + ) + ).catch(next); + }); + + app.get('/runs', (_req, res, next) => { + withDatabase(databasePath, (db) => + res.send( + page( + 'Runs', + `

Runs

${table(db.runs(), [ + ['started_at', 'Started'], + ['mode', 'Mode'], + ['newsletters_processed', 'Newsletters'], + ['links_extracted', 'Links'], + ['sponsors', 'Sponsors'], + ['dead_links', 'Dead Links'], + ['errors', 'Errors'] + ])}` + ) + ) + ).catch(next); + }); + + app.use( + (error: Error, _req: express.Request, res: express.Response, _next: express.NextFunction) => { + console.error(error); + res.status(500).send(page('Error', '

Error

Something went wrong.

')); + } + ); + + return app; +} + +async function withDatabase( + databasePath: string, + callback: (database: CatalogDatabase) => void +): Promise { + const db = new CatalogDatabase(databasePath); + try { + db.migrate(); + callback(db); + } finally { + db.close(); + } +} diff --git a/src/web/views.ts b/src/web/views.ts new file mode 100644 index 0000000..794d44a --- /dev/null +++ b/src/web/views.ts @@ -0,0 +1,63 @@ +function escapeHtml(value: unknown): string { + return String(value ?? '') + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"') + .replaceAll("'", '''); +} + +export function page(title: string, body: string): string { + return ` + + + + + ${escapeHtml(title)} + + + + + ${body} + +`; +} + +export function table(rows: Record[], columns: Array<[string, string]>): string { + if (rows.length === 0) { + return '

No rows yet.

'; + } + return `${columns.map(([, label]) => ``).join('')}${rows + .map((row) => `${columns.map(([key]) => ``).join('')}`) + .join('')}
${escapeHtml(label)}
${escapeHtml(row[key])}
`; +} + +export function dashboard(counts: Record): string { + return page( + 'Newsletter Link Catalog', + `

Newsletter Link Catalog

+
+ ${Object.entries(counts) + .map( + ([key, value]) => + `
${escapeHtml(value)}
${escapeHtml(key)}
` + ) + .join('')} +
` + ); +} diff --git a/tests/config.test.ts b/tests/config.test.ts index bbdc427..a59928f 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -17,6 +17,8 @@ output: expect(config.gmail.folder).toBe('Newsletters'); expect(config.links.trackingParams).toContain('utm_*'); expect(config.enrichment.concurrency).toBe(3); + expect(config.database.enabled).toBe(true); + expect(config.database.path).toBe('./data/newsletter-catalog.sqlite'); }); it('treats comment-only YAML maps as empty objects', () => { diff --git a/tests/database.test.ts b/tests/database.test.ts new file mode 100644 index 0000000..4141f0a --- /dev/null +++ b/tests/database.test.ts @@ -0,0 +1,90 @@ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { CatalogDatabase } from '../src/database/store.js'; + +let dir = ''; + +beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'nlc-db-')); +}); + +afterEach(async () => { + await rm(dir, { force: true, recursive: true }); +}); + +describe('CatalogDatabase', () => { + it('creates the catalog schema', () => { + const db = new CatalogDatabase(join(dir, 'catalog.sqlite')); + db.migrate(); + + expect(db.tableNames()).toEqual([ + 'dead_links', + 'issues', + 'link_occurrences', + 'links', + 'newsletters', + 'runs', + 'sponsors' + ]); + + db.close(); + }); + + it('persists catalog payloads and deduplicates canonical URLs', () => { + const db = new CatalogDatabase(join(dir, 'catalog.sqlite')); + db.migrate(); + db.saveCatalogRun({ + mode: 'test', + newslettersProcessed: 2, + linksExtracted: 2, + sponsorCount: 1, + deadLinkCount: 1, + errors: 0, + rows: [ + { + 'Issue Date': '2026-05-17', + Category: 'JavaScript', + 'Link URL': 'https://example.com/post', + Title: 'First mention', + Description: 'One', + 'Page Title + Meta': '', + 'Source Newsletter': 'Alpha Weekly', + 'Also In': '' + }, + { + 'Issue Date': '2026-05-18', + Category: 'DevOps', + 'Link URL': 'https://example.com/post', + Title: 'Second mention', + Description: 'Two', + 'Page Title + Meta': '', + 'Source Newsletter': 'Beta Weekly', + 'Also In': '' + } + ], + sponsors: [ + { + Newsletter: 'Alpha Weekly', + Sponsor: 'Acme', + Link: 'https://sponsor.example', + Description: 'Sponsor blurb' + } + ], + deadLinks: [ + { URL: 'https://dead.example', Status: '404', Source: 'Alpha Weekly', Date: '2026-05-17' } + ] + }); + + expect(db.count('links')).toBe(3); + expect(db.count('link_occurrences')).toBe(2); + expect(db.count('newsletters')).toBe(2); + expect(db.count('issues')).toBe(2); + expect(db.count('sponsors')).toBe(1); + expect(db.count('dead_links')).toBe(1); + expect(db.count('runs')).toBe(1); + + db.close(); + }); +}); diff --git a/tests/run.test.ts b/tests/run.test.ts index e9c81ae..bae5221 100644 --- a/tests/run.test.ts +++ b/tests/run.test.ts @@ -2,6 +2,7 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { CatalogDatabase } from '../src/database/store.js'; import { runCatalog } from '../src/run/runCatalog.js'; let dir = ''; @@ -86,4 +87,45 @@ describe('run orchestration', () => { 'Typescale AI' ]); }); + + it('can persist run output to SQLite through a database writer', async () => { + const path = join(dir, 'catalog.sqlite'); + const db = new CatalogDatabase(path); + db.migrate(); + + const result = await runCatalog({ + config: { + gmail: { folder: 'Newsletters' }, + output: { name: 'Catalog', excel: { enabled: false } }, + stateFile: join(dir, 'state.json') + }, + messages: [ + { + id: 'msg-1', + messageId: '', + from: 'SQLite Weekly ', + date: '2026-05-17T00:00:00.000Z', + html: '

Databases

SQLite Post - Local data.' + } + ], + writers: [ + { + write: async (payload) => + db.saveCatalogRun({ + mode: 'test', + newslettersProcessed: 1, + linksExtracted: payload.rows.length, + sponsorCount: payload.sponsors.length, + deadLinkCount: payload.deadLinks.length, + errors: 0, + ...payload + }) + } + ] + }); + + expect(result.linksExtracted).toBe(1); + expect(db.count('link_occurrences')).toBe(1); + db.close(); + }); }); diff --git a/tests/web.test.ts b/tests/web.test.ts new file mode 100644 index 0000000..34ad192 --- /dev/null +++ b/tests/web.test.ts @@ -0,0 +1,79 @@ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { AddressInfo } from 'node:net'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { CatalogDatabase } from '../src/database/store.js'; +import { createWebApp } from '../src/web/app.js'; + +let dir = ''; + +beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'nlc-web-')); +}); + +afterEach(async () => { + await rm(dir, { force: true, recursive: true }); +}); + +async function withServer(path: string, pathname: string): Promise { + const app = createWebApp(path); + const server = app.listen(0); + try { + const { port } = server.address() as AddressInfo; + const response = await fetch(`http://127.0.0.1:${port}${pathname}`); + return response.text(); + } finally { + server.close(); + } +} + +function fixtureDatabase(): string { + const path = join(dir, 'catalog.sqlite'); + const db = new CatalogDatabase(path); + db.saveCatalogRun({ + mode: 'test', + newslettersProcessed: 1, + linksExtracted: 1, + sponsorCount: 1, + deadLinkCount: 1, + errors: 0, + rows: [ + { + 'Issue Date': '2026-05-17', + Category: 'SQLite', + 'Link URL': 'https://sqlite.example', + Title: 'SQLite Post', + Description: 'A local database post', + 'Page Title + Meta': '', + 'Source Newsletter': 'DB Weekly', + 'Also In': '' + } + ], + sponsors: [ + { + Newsletter: 'DB Weekly', + Sponsor: 'Acme', + Link: 'https://sponsor.example', + Description: 'Blurb' + } + ], + deadLinks: [ + { URL: 'https://dead.example', Status: '404', Source: 'DB Weekly', Date: '2026-05-17' } + ] + }); + db.close(); + return path; +} + +describe('web app', () => { + it('renders dashboard and catalog pages from SQLite', async () => { + const path = fixtureDatabase(); + + await expect(withServer(path, '/')).resolves.toContain('Newsletter Link Catalog'); + await expect(withServer(path, '/links')).resolves.toContain('SQLite Post'); + await expect(withServer(path, '/sponsors')).resolves.toContain('Acme'); + await expect(withServer(path, '/dead-links')).resolves.toContain('https://dead.example'); + await expect(withServer(path, '/runs')).resolves.toContain('test'); + }); +});