diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index c91f55c..a09f015 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -115,7 +115,7 @@ | Component | Technology | Purpose | |-----------|------------|---------| | Database | SQLite via `better-sqlite3` | Single file `ghostgrid.db`, WAL journal mode, synchronous queries | -| Schema | Idempotent `CREATE TABLE IF NOT EXISTS` | 8 tables defined in full and created on boot (fresh-install model, no migrations) | +| Schema | Idempotent `CREATE TABLE IF NOT EXISTS` | Baseline 8 tables created on boot via `server-db.ts`; additive changes to live DBs go in `server-migrations.ts` (see §4.4) | | Settings store | key/value `settings` table | Runtime config for all integrations, seeded with `INSERT OR IGNORE` | --- @@ -195,7 +195,7 @@ Networking (optional, managed in-app) ## 4. Database Schema Design -SQLite file `ghostgrid.db` in `process.cwd()`, opened with `journal_mode = WAL`. Every table is defined in full and created idempotently on boot (`CREATE TABLE IF NOT EXISTS`) — the app assumes a fresh install, so there is no migration layer. +SQLite file `ghostgrid.db` in `process.cwd()`, opened with `journal_mode = WAL`. Baseline tables are created idempotently on boot (`CREATE TABLE IF NOT EXISTS` in `server-db.ts`). Additive changes to live DBs (new columns, tables, default settings) are applied by the migration runner in `server-migrations.ts`. See §4.4. ### 4.1 Schema (as created in `server-db.ts`) @@ -298,7 +298,7 @@ CREATE TABLE IF NOT EXISTS caddy ( | JSON columns | `labs.deviceIds` and `labs.topology` are JSON strings, parsed in the API layer | | Booleans | Booking flags (`notified`, `emailSent`, `ansible*Triggered`) are `INTEGER` 0/1, mapped to JS booleans on read | | Cascade | No FK cascades; referential cleanup is done in code (e.g. deleting a device scrubs it from every lab's `deviceIds`/`topology`) | -| Schema changes | Fresh-install model — edit the `CREATE TABLE` block in `server-db.ts` directly. To add columns to an existing DB, place idempotent `ALTER TABLE … ADD COLUMN` calls (wrapped in try/catch) in `server-db.ts` after the `db.exec()` block — they run on every startup and are no-ops when the column already exists | +| Schema changes | Baseline schema in `server-db.ts` (`CREATE TABLE IF NOT EXISTS`). New columns / tables / default settings for live DBs go in `server-migrations.ts` as an appended migration object. See §4.4. | ### 4.3 Settings (key/value config) @@ -313,6 +313,29 @@ Seeded with `INSERT OR IGNORE` (defaults below). Secret keys are never returned 🔒 = in `SECRET_KEYS`, masked on read, only updated when a non-`__SET__` value is sent. +### 4.4 Migration System + +Additive schema changes to live databases are handled by `server-migrations.ts`. The runner executes once at the start of `startServer()`, before routes or background jobs, and is synchronous (`better-sqlite3`). + +**How it works:** + +1. Creates (idempotently) a `_migrations` table: `id TEXT PRIMARY KEY, applied_at TEXT`. +2. Iterates the `migrations` array in order. For each entry whose `id` is not in `_migrations`, runs `up(db)` inside a transaction and records the id on success. +3. If `up()` throws, the transaction rolls back and the id is not recorded — the migration retries on the next start. +4. Already-applied migrations are skipped forever. + +**Rules for adding a migration:** + +- Append to the end of the `migrations` array in `server-migrations.ts`. **Never reorder or remove entries.** +- Use a unique, descriptive `id` — format: `NNNN_short_description` (e.g. `0001_bookings_add_color`). +- Only additive DDL: `ALTER TABLE … ADD COLUMN`, `CREATE TABLE IF NOT EXISTS`, `INSERT OR IGNORE INTO settings`. +- New default settings for **live DBs** go here. Day-0 seeds (fresh install) stay in `DEFAULT_SETTINGS` in `server-db.ts`. +- New settings still need allow-listing in `PUT /api/settings` (and `SECRET_KEYS` if secret) — the migration only seeds the value. + +**Fresh-install behavior:** `server-db.ts` creates all baseline tables first (at module import). `runMigrations` then applies every migration in the array — safe because all migrations are additive over the baseline. + +**`_migrations` is excluded** from `/api/database/import` and `/api/database/info` (not in their explicit table lists) — migration state belongs to the current install, not the imported data. + --- ## 5. API Design @@ -784,7 +807,7 @@ GhostGrid/ | CheckMK/Semaphore outage | Integration loops catch errors, log them, and retry next cycle; non-fatal | | Caddy admin API unreachable | `pushCaddyConfig()` failures are logged as warnings; routes apply when Caddy starts | | Data loss | Back up `ghostgrid.db` + `-wal`/`-shm`; each instance has its own DB | -| Schema evolution | Edit the `CREATE TABLE` block in `server-db.ts` (fresh-install model). For live DBs, add new columns via idempotent `ALTER TABLE … ADD COLUMN` (try/catch) after the `db.exec()` block. New settings need seed + allow-list (+ `SECRET_KEYS` if secret) | +| Schema evolution | For fresh installs, edit the `CREATE TABLE` block in `server-db.ts`. For live DBs (new columns, tables, default settings), append a migration to `server-migrations.ts` (see §4.4). New settings still need allow-listing in `PUT /api/settings` (+ `SECRET_KEYS` if secret). | --- @@ -852,7 +875,7 @@ Express (server.ts) ──► better-sqlite3 (ghostgrid.db, WAL) - `labs.deviceIds` / `labs.topology` are JSON strings in SQLite, parsed in the API. - Booking boolean flags are 0/1 integers in SQLite, mapped on read. - A new settings key must be: **seeded** in `server-db.ts`, **allow-listed** in `PUT /api/settings`, and (if secret) added to `SECRET_KEYS`. -- Schema changes go into the `CREATE TABLE` block in `server-db.ts` (fresh-install). For adding columns to live DBs, use idempotent `ALTER TABLE … ADD COLUMN` (try/catch) placed after the `db.exec()` block. +- Baseline schema in `server-db.ts`. Additive schema changes (new columns, tables, default settings) go in `server-migrations.ts` as an appended migration object — never edit existing entries (see §4.4). - The SPA catch-all (`app.get('*')`) + static serving are registered **last** in `startServer()`, after every `/api` route — otherwise GET `/api/*` falls through to `index.html`. All `/api` responses carry `Cache-Control: no-store`. - One Caddy per container; `POST /load` replaces the whole config. Only the `CADDY_MANAGER=true` instance may push/seed/edit routes — never let the non-manager push. - All user-facing strings are in **English**. diff --git a/server-migrations.ts b/server-migrations.ts new file mode 100644 index 0000000..dee4042 --- /dev/null +++ b/server-migrations.ts @@ -0,0 +1,40 @@ +import Database from 'better-sqlite3'; + +interface Migration { + id: string; // unique, immutable — format: NNNN_short_description + up: (db: InstanceType) => void; +} + +// Append only. Never reorder or remove entries — that would corrupt tracking. +// Each `up` function receives the open DB handle inside an already-open transaction. +const migrations: Migration[] = [ + // Example: + // { + // id: '0001_bookings_add_color', + // up: (db) => { + // db.exec(`ALTER TABLE bookings ADD COLUMN color TEXT NOT NULL DEFAULT 'blue'`); + // }, + // }, +]; + +export function runMigrations(db: InstanceType): void { + db.exec(` + CREATE TABLE IF NOT EXISTS _migrations ( + id TEXT PRIMARY KEY, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) + ) + `); + + const isApplied = db.prepare('SELECT 1 FROM _migrations WHERE id = ?'); + const markApplied = db.prepare('INSERT INTO _migrations (id) VALUES (?)'); + + for (const migration of migrations) { + if (isApplied.get(migration.id)) continue; + console.log(`[Migrations] Applying: ${migration.id}`); + db.transaction(() => { + migration.up(db); + markApplied.run(migration.id); + })(); + console.log(`[Migrations] Applied: ${migration.id}`); + } +} diff --git a/server.ts b/server.ts index 8f625a6..c8b5176 100644 --- a/server.ts +++ b/server.ts @@ -8,6 +8,7 @@ import jwt from 'jsonwebtoken'; import DatabaseConstructor from 'better-sqlite3'; import { ConfidentialClientApplication } from '@azure/msal-node'; import db, { uid, addLog, getSetting, setSetting, getAllSettings, getCaddyRoutes, addCaddyRoute, updateCaddyRoute, deleteCaddyRoute, getCaddyRouteById, DB_FILE } from './server-db'; +import { runMigrations } from './server-migrations'; import { Device, LabTemplate, Booking, LogEntry, User, QuickLink } from './src/types'; const JWT_SECRET = process.env.JWT_SECRET || 'ghostgrid-dev-secret-change-in-production'; @@ -164,6 +165,8 @@ async function pushCaddyConfig(): Promise { } async function startServer() { + runMigrations(db); + const app = express(); const PORT = Number(process.env.PORT) || 3000;