feat: add sqlite catalog web app

tests/config.test.ts

@@ -17,6 +17,8 @@ output:
     expect(config.gmail.folder).toBe('Newsletters');
     expect(config.links.trackingParams).toContain('utm_*');
     expect(config.enrichment.concurrency).toBe(3);
+    expect(config.database.enabled).toBe(true);
+    expect(config.database.path).toBe('./data/newsletter-catalog.sqlite');
   });
 
   it('treats comment-only YAML maps as empty objects', () => {

tests/database.test.ts

@@ -0,0 +1,90 @@
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { CatalogDatabase } from '../src/database/store.js';
+
+let dir = '';
+
+beforeEach(async () => {
+  dir = await mkdtemp(join(tmpdir(), 'nlc-db-'));
+});
+
+afterEach(async () => {
+  await rm(dir, { force: true, recursive: true });
+});
+
+describe('CatalogDatabase', () => {
+  it('creates the catalog schema', () => {
+    const db = new CatalogDatabase(join(dir, 'catalog.sqlite'));
+    db.migrate();
+
+    expect(db.tableNames()).toEqual([
+      'dead_links',
+      'issues',
+      'link_occurrences',
+      'links',
+      'newsletters',
+      'runs',
+      'sponsors'
+    ]);
+
+    db.close();
+  });
+
+  it('persists catalog payloads and deduplicates canonical URLs', () => {
+    const db = new CatalogDatabase(join(dir, 'catalog.sqlite'));
+    db.migrate();
+    db.saveCatalogRun({
+      mode: 'test',
+      newslettersProcessed: 2,
+      linksExtracted: 2,
+      sponsorCount: 1,
+      deadLinkCount: 1,
+      errors: 0,
+      rows: [
+        {
+          'Issue Date': '2026-05-17',
+          Category: 'JavaScript',
+          'Link URL': 'https://example.com/post',
+          Title: 'First mention',
+          Description: 'One',
+          'Page Title + Meta': '',
+          'Source Newsletter': 'Alpha Weekly',
+          'Also In': ''
+        },
+        {
+          'Issue Date': '2026-05-18',
+          Category: 'DevOps',
+          'Link URL': 'https://example.com/post',
+          Title: 'Second mention',
+          Description: 'Two',
+          'Page Title + Meta': '',
+          'Source Newsletter': 'Beta Weekly',
+          'Also In': ''
+        }
+      ],
+      sponsors: [
+        {
+          Newsletter: 'Alpha Weekly',
+          Sponsor: 'Acme',
+          Link: 'https://sponsor.example',
+          Description: 'Sponsor blurb'
+        }
+      ],
+      deadLinks: [
+        { URL: 'https://dead.example', Status: '404', Source: 'Alpha Weekly', Date: '2026-05-17' }
+      ]
+    });
+
+    expect(db.count('links')).toBe(3);
+    expect(db.count('link_occurrences')).toBe(2);
+    expect(db.count('newsletters')).toBe(2);
+    expect(db.count('issues')).toBe(2);
+    expect(db.count('sponsors')).toBe(1);
+    expect(db.count('dead_links')).toBe(1);
+    expect(db.count('runs')).toBe(1);
+
+    db.close();
+  });
+});

tests/run.test.ts

@@ -2,6 +2,7 @@ import { mkdtemp, rm } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { CatalogDatabase } from '../src/database/store.js';
 import { runCatalog } from '../src/run/runCatalog.js';
 
 let dir = '';
@@ -86,4 +87,45 @@ describe('run orchestration', () => {
       'Typescale AI'
     ]);
   });
+
+  it('can persist run output to SQLite through a database writer', async () => {
+    const path = join(dir, 'catalog.sqlite');
+    const db = new CatalogDatabase(path);
+    db.migrate();
+
+    const result = await runCatalog({
+      config: {
+        gmail: { folder: 'Newsletters' },
+        output: { name: 'Catalog', excel: { enabled: false } },
+        stateFile: join(dir, 'state.json')
+      },
+      messages: [
+        {
+          id: 'msg-1',
+          messageId: '<msg-1>',
+          from: 'SQLite Weekly <sqlite@example.com>',
+          date: '2026-05-17T00:00:00.000Z',
+          html: '<h2>Databases</h2><a href="https://sqlite.example">SQLite Post</a> - Local data.'
+        }
+      ],
+      writers: [
+        {
+          write: async (payload) =>
+            db.saveCatalogRun({
+              mode: 'test',
+              newslettersProcessed: 1,
+              linksExtracted: payload.rows.length,
+              sponsorCount: payload.sponsors.length,
+              deadLinkCount: payload.deadLinks.length,
+              errors: 0,
+              ...payload
+            })
+        }
+      ]
+    });
+
+    expect(result.linksExtracted).toBe(1);
+    expect(db.count('link_occurrences')).toBe(1);
+    db.close();
+  });
 });

tests/web.test.ts

@@ -0,0 +1,79 @@
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { AddressInfo } from 'node:net';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { CatalogDatabase } from '../src/database/store.js';
+import { createWebApp } from '../src/web/app.js';
+
+let dir = '';
+
+beforeEach(async () => {
+  dir = await mkdtemp(join(tmpdir(), 'nlc-web-'));
+});
+
+afterEach(async () => {
+  await rm(dir, { force: true, recursive: true });
+});
+
+async function withServer(path: string, pathname: string): Promise<string> {
+  const app = createWebApp(path);
+  const server = app.listen(0);
+  try {
+    const { port } = server.address() as AddressInfo;
+    const response = await fetch(`http://127.0.0.1:${port}${pathname}`);
+    return response.text();
+  } finally {
+    server.close();
+  }
+}
+
+function fixtureDatabase(): string {
+  const path = join(dir, 'catalog.sqlite');
+  const db = new CatalogDatabase(path);
+  db.saveCatalogRun({
+    mode: 'test',
+    newslettersProcessed: 1,
+    linksExtracted: 1,
+    sponsorCount: 1,
+    deadLinkCount: 1,
+    errors: 0,
+    rows: [
+      {
+        'Issue Date': '2026-05-17',
+        Category: 'SQLite',
+        'Link URL': 'https://sqlite.example',
+        Title: 'SQLite Post',
+        Description: 'A local database post',
+        'Page Title + Meta': '',
+        'Source Newsletter': 'DB Weekly',
+        'Also In': ''
+      }
+    ],
+    sponsors: [
+      {
+        Newsletter: 'DB Weekly',
+        Sponsor: 'Acme',
+        Link: 'https://sponsor.example',
+        Description: 'Blurb'
+      }
+    ],
+    deadLinks: [
+      { URL: 'https://dead.example', Status: '404', Source: 'DB Weekly', Date: '2026-05-17' }
+    ]
+  });
+  db.close();
+  return path;
+}
+
+describe('web app', () => {
+  it('renders dashboard and catalog pages from SQLite', async () => {
+    const path = fixtureDatabase();
+
+    await expect(withServer(path, '/')).resolves.toContain('Newsletter Link Catalog');
+    await expect(withServer(path, '/links')).resolves.toContain('SQLite Post');
+    await expect(withServer(path, '/sponsors')).resolves.toContain('Acme');
+    await expect(withServer(path, '/dead-links')).resolves.toContain('https://dead.example');
+    await expect(withServer(path, '/runs')).resolves.toContain('test');
+  });
+});