diff --git a/.github/workflows/bulldozer-serialization-compat.yaml b/.github/workflows/bulldozer-serialization-compat.yaml new file mode 100644 index 000000000..e5ffbba58 --- /dev/null +++ b/.github/workflows/bulldozer-serialization-compat.yaml @@ -0,0 +1,133 @@ +name: Bulldozer serialization compat + +# Guards the on-disk format of the bulldozer/piledriver augmented B-tree. Bulldozer data is persisted +# in production, so a change that makes the current code unable to read previously-written data (or +# makes an older build unable to read data the new code writes, which matters for rollbacks) would +# corrupt or lose data. This workflow proves both directions by having each code version read a dump +# written by the other. The per-version golden-fixture unit test (serialization-compat.test.ts) is the +# first line of defence; this cross-version job is the belt-and-suspenders check. + +on: + push: + branches-ignore: + - main + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/dev' }} + +# DATA_STRUCTURES_DIR is repo-relative (used for diffing/overlaying). HARNESS_REL is relative to the +# bulldozer-js package, since the harness is run via `pnpm -C apps/bulldozer-js exec tsx`. +env: + DATA_STRUCTURES_DIR: apps/bulldozer-js/src/databases/piledriver/data-structures + HARNESS_REL: src/databases/piledriver/data-structures/__fixtures__/serialization-cross-version.ts + +jobs: + check-changed: + name: Check if bulldozer serialization changed + runs-on: ubuntu-latest + outputs: + changed: ${{ steps.check-diff.outputs.changed }} + base_branch: ${{ steps.check-diff.outputs.base_branch }} + steps: + - name: Checkout current branch + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 0 + + - name: Check for serialization-relevant changes + id: check-diff + run: | + # dev compares to main, everything else compares to dev. + if [ "${{ github.ref }}" = "refs/heads/dev" ]; then + BASE_BRANCH="main" + else + BASE_BRANCH="dev" + fi + echo "base_branch=$BASE_BRANCH" >> $GITHUB_OUTPUT + + git fetch origin "$BASE_BRANCH" + MERGE_BASE=$(git merge-base HEAD "origin/$BASE_BRANCH") + + # Any change under the piledriver data-structures dir could touch the persisted node format. + if git diff --quiet "$MERGE_BASE" HEAD -- "$DATA_STRUCTURES_DIR"; then + echo "No serialization-relevant changes" + echo "changed=false" >> $GITHUB_OUTPUT + else + echo "Serialization-relevant changes detected" + echo "changed=true" >> $GITHUB_OUTPUT + fi + + cross-version: + name: Cross-version read (base ↔ current) + needs: check-changed + if: needs.check-changed.outputs.changed == 'true' + runs-on: ubuntu-latest + steps: + - name: Checkout current branch + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + path: current + + - name: Checkout base branch (${{ needs.check-changed.outputs.base_branch }}) + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + ref: ${{ needs.check-changed.outputs.base_branch }} + path: base + + # The harness + its utils are new, so the base checkout doesn't have them. Overlay the current + # branch's copy onto the base tree; the harness only uses the long-stable AugmentedTreeMap / + # piledriver public API, so it compiles and runs against the base branch's implementation. + - name: Overlay harness onto base checkout + run: | + rm -rf "base/$DATA_STRUCTURES_DIR/__fixtures__" + cp -r "current/$DATA_STRUCTURES_DIR/__fixtures__" "base/$DATA_STRUCTURES_DIR/__fixtures__" + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: 22.x + + - name: Setup pnpm + uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 + + - name: Install and build @hexclave/shared (current) + working-directory: current + run: | + pnpm install --frozen-lockfile + pnpm exec turbo run build --filter=@hexclave/shared + + - name: Install and build @hexclave/shared (base) + working-directory: base + run: | + pnpm install --frozen-lockfile + pnpm exec turbo run build --filter=@hexclave/shared + + - name: Emit dump with current code + working-directory: current + run: pnpm -C apps/bulldozer-js exec tsx "$HARNESS_REL" emit "$GITHUB_WORKSPACE/new.json" + + - name: Emit dump with base code + working-directory: base + run: pnpm -C apps/bulldozer-js exec tsx "$HARNESS_REL" emit "$GITHUB_WORKSPACE/old.json" + + # Backward compatibility (production upgrade): the new code must read what the old code wrote. + - name: Current code reads base-written dump + working-directory: current + run: pnpm -C apps/bulldozer-js exec tsx "$HARNESS_REL" verify "$GITHUB_WORKSPACE/old.json" + + # Forward compatibility (production rollback): the old code must read what the new code wrote. + - name: Base code reads current-written dump + working-directory: base + run: pnpm -C apps/bulldozer-js exec tsx "$HARNESS_REL" verify "$GITHUB_WORKSPACE/new.json" + + # Keeps the workflow (and the aggregate `all-good` check) green when serialization wasn't touched. + skip-unchanged: + name: No serialization changes (skipped) + needs: check-changed + if: needs.check-changed.outputs.changed == 'false' + runs-on: ubuntu-latest + steps: + - name: No serialization-relevant changes detected + run: echo "No changes under $DATA_STRUCTURES_DIR detected. Skipping cross-version compat." diff --git a/apps/bulldozer-js/src/databases/piledriver/data-structures/__fixtures__/generate-serialization-fixtures.ts b/apps/bulldozer-js/src/databases/piledriver/data-structures/__fixtures__/generate-serialization-fixtures.ts new file mode 100644 index 000000000..5a1c43445 --- /dev/null +++ b/apps/bulldozer-js/src/databases/piledriver/data-structures/__fixtures__/generate-serialization-fixtures.ts @@ -0,0 +1,64 @@ +/** + * Regenerates the golden serialization fixtures used by `serialization-compat.test.ts`. + * + * These fixtures freeze the on-disk shape of a persisted `AugmentedTreeMap` for each node format + * version, so that if a future change alters how nodes are (de)serialized in a way that breaks + * reading previously-persisted data, the compat test fails immediately. + * + * IMPORTANT: never edit or delete an existing fixture by hand — that would defeat the purpose. + * When you introduce a new node format version, bump `nodeFormatVersion` in augmented-tree-map.ts, + * add a new `writeFixture(...)` call below, and run: + * + * pnpm -C apps/bulldozer-js exec tsx src/databases/piledriver/data-structures/__fixtures__/generate-serialization-fixtures.ts + * + * The v0 fixture (no `version`/`entryAugmentations` fields) reproduces the exact node shape that + * builds before format versioning wrote, so we keep proving that legacy production data still reads. + */ +import { mkdirSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + FIXTURE_ARITY, + FIXTURE_ROW_COUNT, + Json, + SerializationDump, + buildSampleTree, + computeQueries, + resolveHeap, +} from "./serialization-fixture-utils.js"; + +const here = dirname(fileURLToPath(import.meta.url)); + +// Produces the pre-versioning node shape by dropping the fields v0 nodes never had, everywhere they +// appear in the graph. +function stripToV0(value: Json): Json { + if (Array.isArray(value)) return value.map(stripToV0); + if (value === null || typeof value !== "object") return value; + const out: { [key: string]: Json } = {}; + for (const [key, child] of Object.entries(value)) { + if (key === "version" || key === "entryAugmentations") continue; + out[key] = stripToV0(child); + } + return out; +} + +function writeFixture(name: string, body: SerializationDump) { + const path = join(here, `${name}.json`); + writeFileSync(path, `${JSON.stringify(body, null, 2)}\n`); + process.stdout.write(`wrote ${path}\n`); +} + +async function main() { + mkdirSync(here, { recursive: true }); + const tree = await buildSampleTree(FIXTURE_ARITY, FIXTURE_ROW_COUNT); + const treeObject = await resolveHeap(tree.toPiledriverObject()); + const queries = await computeQueries(tree); + + writeFixture("v1", { arity: FIXTURE_ARITY, tree: treeObject, queries }); + writeFixture("v0", { arity: FIXTURE_ARITY, tree: stripToV0(treeObject), queries }); +} + +main().catch((error) => { + process.stderr.write(`${error?.stack ?? error}\n`); + process.exit(1); +}); diff --git a/apps/bulldozer-js/src/databases/piledriver/data-structures/__fixtures__/serialization-cross-version.ts b/apps/bulldozer-js/src/databases/piledriver/data-structures/__fixtures__/serialization-cross-version.ts new file mode 100644 index 000000000..03fb7a5df --- /dev/null +++ b/apps/bulldozer-js/src/databases/piledriver/data-structures/__fixtures__/serialization-cross-version.ts @@ -0,0 +1,65 @@ +/** + * Cross-version serialization compatibility harness, used by the `serialization-compat` GitHub + * Actions workflow. Unlike the golden fixtures (which pin a frozen shape), this proves that two + * *different code versions* interoperate on the wire: + * + * emit build the deterministic sample tree, persist it, and write a self-describing dump + * verify read a dump written by (possibly) another code version and assert it reproduces + * the exact query results its writer observed + * + * The workflow runs `emit` under one checkout and `verify` under the other, in both directions: + * - new code reads a dump written by base code (backward compat: prod upgrade) + * - base code reads a dump written by new code (forward compat: prod rollback) + * + * This file (and serialization-fixture-utils.ts) is copied into the base-branch checkout by the + * workflow, so it must only rely on the long-stable AugmentedTreeMap / piledriver public API. + */ +import { readFileSync, writeFileSync } from "node:fs"; +import { + FIXTURE_ARITY, + FIXTURE_ROW_COUNT, + SerializationDump, + buildSampleTree, + computeQueries, + resolveHeap, + treeFromDump, +} from "./serialization-fixture-utils.js"; + +async function emit(path: string) { + const tree = await buildSampleTree(FIXTURE_ARITY, FIXTURE_ROW_COUNT); + const dump: SerializationDump = { + arity: FIXTURE_ARITY, + tree: await resolveHeap(tree.toPiledriverObject()), + queries: await computeQueries(tree), + }; + writeFileSync(path, `${JSON.stringify(dump, null, 2)}\n`); + process.stdout.write(`emitted serialization dump to ${path}\n`); +} + +async function verify(path: string) { + // Boundary parse of a dump written by this or another code version; mismatches surface below. + const dump = JSON.parse(readFileSync(path, "utf8")) as SerializationDump; + const actual = await computeQueries(treeFromDump(dump)); + const expected = JSON.stringify(dump.queries); + if (JSON.stringify(actual) !== expected) { + process.stderr.write(`serialization cross-version MISMATCH reading ${path}\n expected: ${expected}\n actual: ${JSON.stringify(actual)}\n`); + process.exit(1); + } + process.stdout.write(`verified serialization dump ${path} (queries reproduced exactly)\n`); +} + +async function main() { + const [mode, path] = process.argv.slice(2); + if ((mode !== "emit" && mode !== "verify") || !path) { + process.stderr.write("usage: serialization-cross-version.ts \n"); + process.exit(2); + return; + } + if (mode === "emit") await emit(path); + else await verify(path); +} + +main().catch((error) => { + process.stderr.write(`${error?.stack ?? error}\n`); + process.exit(1); +}); diff --git a/apps/bulldozer-js/src/databases/piledriver/data-structures/__fixtures__/serialization-fixture-utils.ts b/apps/bulldozer-js/src/databases/piledriver/data-structures/__fixtures__/serialization-fixture-utils.ts new file mode 100644 index 000000000..e437f776a --- /dev/null +++ b/apps/bulldozer-js/src/databases/piledriver/data-structures/__fixtures__/serialization-fixture-utils.ts @@ -0,0 +1,111 @@ +/** + * Shared helpers for the augmented-tree-map serialization compatibility checks: + * - the golden fixtures + `serialization-compat.test.ts` (checked-in, per-version regression), and + * - `serialization-cross-version.ts` (CI harness that has one code version write a dump and another + * read it back). + * + * Everything here depends only on the long-stable `AugmentedTreeMap` / piledriver public API, so this + * module also imports cleanly under older checkouts of the repo (which the cross-version CI job relies + * on when it runs the harness against the base branch's code). + */ +import { AugmentedTreeMap } from "../augmented-tree-map.js"; +import { PiledriverHeapObject, PiledriverObject, isPiledriverHeapObjectSymbol } from "../../index.js"; + +export type Json = null | boolean | number | string | Json[] | { [key: string]: Json }; + +export type SerializationQueries = { + size: number, + entriesInRange: [number, number][], + reversedLimited: [number, number][], + augmentationAll: number, + augmentationRange: number, +}; + +// A self-describing serialized tree: the persisted object graph plus the query results the writer +// observed. A reader passes the check iff, after deserialising `tree`, it reproduces `queries`. +export type SerializationDump = { + arity: number, + tree: Json, + queries: SerializationQueries, +}; + +export const FIXTURE_ARITY = 8; +export const FIXTURE_ROW_COUNT = 40; + +export function optionsFor(arity: number) { + return { + arity, + comparator: (a: number, b: number) => a - b, + initialAugmentation: 0, + extractAugmentation: (value: number) => value, + mergeAugmentations: (...values: number[]) => values.reduce((sum, value) => sum + value, 0), + }; +} + +// The deterministic dataset every writer produces, so dumps from different code versions are directly +// comparable: keys 1..FIXTURE_ROW_COUNT mapped to key*2. +export async function buildSampleTree(arity: number, rowCount: number): Promise> { + let tree = new AugmentedTreeMap(optionsFor(arity)); + for (let key = 1; key <= rowCount; key++) tree = await tree.set(key, key * 2); + return tree; +} + +async function arrayFrom(iterable: AsyncIterable): Promise { + const result: T[] = []; + for await (const item of iterable) result.push(item); + return result; +} + +// The fixed set of queries used to attest that a deserialised tree behaves identically regardless of +// which code version wrote it. Covers forward/reverse iteration, limits, and full/partial aggregation. +export async function computeQueries(tree: AugmentedTreeMap): Promise { + return { + size: await tree.size(), + entriesInRange: await arrayFrom(tree.entries({ gte: 10, lte: 15 })), + reversedLimited: await arrayFrom(tree.entries({ lte: 30, reverse: true, limit: 5 })), + augmentationAll: await tree.getAugmentation({}), + augmentationRange: await tree.getAugmentation({ gte: 10, lte: 20 }), + }; +} + +// Recursively resolves every heap object into its inline content, producing a plain-JSON snapshot of +// the persisted object graph (heap references inlined where they point). Mirrors the real on-disk node +// shape without needing the internal Node/Child types. +export async function resolveHeap(value: PiledriverObject): Promise { + if (value === null || typeof value !== "object") return value; + if (isPiledriverHeapObjectSymbol in value) return await resolveHeap(await value.get()); + if (Array.isArray(value)) return await Promise.all(value.map(resolveHeap)); + const out: { [key: string]: Json } = {}; + for (const [key, child] of Object.entries(value)) out[key] = await resolveHeap(child); + return out; +} + +// Wraps a plain object as a heap object whose `get()` resolves to it, mirroring how piledriver hands +// deserialised heap objects back to the tree. +function asFrozenHeapObject(object: PiledriverObject): PiledriverHeapObject { + return { + async get() { + return object; + }, + [isPiledriverHeapObjectSymbol]: true, + }; +} + +// Reconstructs the persisted object graph, wrapping every `ref` field back into a heap object (the +// only place the tree stores heap references). Keying on the real field name is intentional: if a +// future format renames/relocates it, the fixture's `ref` becomes unreadable and the check fails — +// which is exactly the regression we want to surface. +export function inflate(value: Json): PiledriverObject { + if (value === null || typeof value !== "object") return value; + if (Array.isArray(value)) return value.map(inflate); + const out: { [key: string]: PiledriverObject } = {}; + for (const [key, child] of Object.entries(value)) { + const inflated = inflate(child); + out[key] = key === "ref" ? asFrozenHeapObject(inflated) : inflated; + } + return out; +} + +export function treeFromDump(dump: SerializationDump): AugmentedTreeMap { + return AugmentedTreeMap.fromPiledriverObject(inflate(dump.tree), optionsFor(dump.arity)); +} diff --git a/apps/bulldozer-js/src/databases/piledriver/data-structures/__fixtures__/v0.json b/apps/bulldozer-js/src/databases/piledriver/data-structures/__fixtures__/v0.json new file mode 100644 index 000000000..b675eb7ca --- /dev/null +++ b/apps/bulldozer-js/src/databases/piledriver/data-structures/__fixtures__/v0.json @@ -0,0 +1,592 @@ +{ + "arity": 8, + "tree": { + "type": "AugmentedTreeMap", + "root": { + "ref": { + "entries": [ + [ + { + "key": 5, + "id": "" + }, + 10 + ], + [ + { + "key": 10, + "id": "" + }, + 20 + ], + [ + { + "key": 15, + "id": "" + }, + 30 + ], + [ + { + "key": 20, + "id": "" + }, + 40 + ], + [ + { + "key": 25, + "id": "" + }, + 50 + ], + [ + { + "key": 30, + "id": "" + }, + 60 + ], + [ + { + "key": 35, + "id": "" + }, + 70 + ] + ], + "children": [ + { + "ref": { + "entries": [ + [ + { + "key": 1, + "id": "" + }, + 2 + ], + [ + { + "key": 2, + "id": "" + }, + 4 + ], + [ + { + "key": 3, + "id": "" + }, + 6 + ], + [ + { + "key": 4, + "id": "" + }, + 8 + ] + ], + "children": [], + "augmentation": 20, + "size": 4, + "minKey": { + "key": 1, + "id": "" + }, + "maxKey": { + "key": 4, + "id": "" + } + }, + "augmentation": 20, + "size": 4, + "entryCount": 4, + "minKey": { + "key": 1, + "id": "" + }, + "maxKey": { + "key": 4, + "id": "" + } + }, + { + "ref": { + "entries": [ + [ + { + "key": 6, + "id": "" + }, + 12 + ], + [ + { + "key": 7, + "id": "" + }, + 14 + ], + [ + { + "key": 8, + "id": "" + }, + 16 + ], + [ + { + "key": 9, + "id": "" + }, + 18 + ] + ], + "children": [], + "augmentation": 60, + "size": 4, + "minKey": { + "key": 6, + "id": "" + }, + "maxKey": { + "key": 9, + "id": "" + } + }, + "augmentation": 60, + "size": 4, + "entryCount": 4, + "minKey": { + "key": 6, + "id": "" + }, + "maxKey": { + "key": 9, + "id": "" + } + }, + { + "ref": { + "entries": [ + [ + { + "key": 11, + "id": "" + }, + 22 + ], + [ + { + "key": 12, + "id": "" + }, + 24 + ], + [ + { + "key": 13, + "id": "" + }, + 26 + ], + [ + { + "key": 14, + "id": "" + }, + 28 + ] + ], + "children": [], + "augmentation": 100, + "size": 4, + "minKey": { + "key": 11, + "id": "" + }, + "maxKey": { + "key": 14, + "id": "" + } + }, + "augmentation": 100, + "size": 4, + "entryCount": 4, + "minKey": { + "key": 11, + "id": "" + }, + "maxKey": { + "key": 14, + "id": "" + } + }, + { + "ref": { + "entries": [ + [ + { + "key": 16, + "id": "" + }, + 32 + ], + [ + { + "key": 17, + "id": "" + }, + 34 + ], + [ + { + "key": 18, + "id": "" + }, + 36 + ], + [ + { + "key": 19, + "id": "" + }, + 38 + ] + ], + "children": [], + "augmentation": 140, + "size": 4, + "minKey": { + "key": 16, + "id": "" + }, + "maxKey": { + "key": 19, + "id": "" + } + }, + "augmentation": 140, + "size": 4, + "entryCount": 4, + "minKey": { + "key": 16, + "id": "" + }, + "maxKey": { + "key": 19, + "id": "" + } + }, + { + "ref": { + "entries": [ + [ + { + "key": 21, + "id": "" + }, + 42 + ], + [ + { + "key": 22, + "id": "" + }, + 44 + ], + [ + { + "key": 23, + "id": "" + }, + 46 + ], + [ + { + "key": 24, + "id": "" + }, + 48 + ] + ], + "children": [], + "augmentation": 180, + "size": 4, + "minKey": { + "key": 21, + "id": "" + }, + "maxKey": { + "key": 24, + "id": "" + } + }, + "augmentation": 180, + "size": 4, + "entryCount": 4, + "minKey": { + "key": 21, + "id": "" + }, + "maxKey": { + "key": 24, + "id": "" + } + }, + { + "ref": { + "entries": [ + [ + { + "key": 26, + "id": "" + }, + 52 + ], + [ + { + "key": 27, + "id": "" + }, + 54 + ], + [ + { + "key": 28, + "id": "" + }, + 56 + ], + [ + { + "key": 29, + "id": "" + }, + 58 + ] + ], + "children": [], + "augmentation": 220, + "size": 4, + "minKey": { + "key": 26, + "id": "" + }, + "maxKey": { + "key": 29, + "id": "" + } + }, + "augmentation": 220, + "size": 4, + "entryCount": 4, + "minKey": { + "key": 26, + "id": "" + }, + "maxKey": { + "key": 29, + "id": "" + } + }, + { + "ref": { + "entries": [ + [ + { + "key": 31, + "id": "" + }, + 62 + ], + [ + { + "key": 32, + "id": "" + }, + 64 + ], + [ + { + "key": 33, + "id": "" + }, + 66 + ], + [ + { + "key": 34, + "id": "" + }, + 68 + ] + ], + "children": [], + "augmentation": 260, + "size": 4, + "minKey": { + "key": 31, + "id": "" + }, + "maxKey": { + "key": 34, + "id": "" + } + }, + "augmentation": 260, + "size": 4, + "entryCount": 4, + "minKey": { + "key": 31, + "id": "" + }, + "maxKey": { + "key": 34, + "id": "" + } + }, + { + "ref": { + "entries": [ + [ + { + "key": 36, + "id": "" + }, + 72 + ], + [ + { + "key": 37, + "id": "" + }, + 74 + ], + [ + { + "key": 38, + "id": "" + }, + 76 + ], + [ + { + "key": 39, + "id": "" + }, + 78 + ], + [ + { + "key": 40, + "id": "" + }, + 80 + ] + ], + "children": [], + "augmentation": 380, + "size": 5, + "minKey": { + "key": 36, + "id": "" + }, + "maxKey": { + "key": 40, + "id": "" + } + }, + "augmentation": 380, + "size": 5, + "entryCount": 5, + "minKey": { + "key": 36, + "id": "" + }, + "maxKey": { + "key": 40, + "id": "" + } + } + ], + "augmentation": 1640, + "size": 40, + "minKey": { + "key": 1, + "id": "" + }, + "maxKey": { + "key": 40, + "id": "" + } + }, + "augmentation": 1640, + "size": 40, + "entryCount": 7, + "minKey": { + "key": 1, + "id": "" + }, + "maxKey": { + "key": 40, + "id": "" + } + } + }, + "queries": { + "size": 40, + "entriesInRange": [ + [ + 10, + 20 + ], + [ + 11, + 22 + ], + [ + 12, + 24 + ], + [ + 13, + 26 + ], + [ + 14, + 28 + ], + [ + 15, + 30 + ] + ], + "reversedLimited": [ + [ + 30, + 60 + ], + [ + 29, + 58 + ], + [ + 28, + 56 + ], + [ + 27, + 54 + ], + [ + 26, + 52 + ] + ], + "augmentationAll": 1640, + "augmentationRange": 330 + } +} diff --git a/apps/bulldozer-js/src/databases/piledriver/data-structures/__fixtures__/v1.json b/apps/bulldozer-js/src/databases/piledriver/data-structures/__fixtures__/v1.json new file mode 100644 index 000000000..83505755d --- /dev/null +++ b/apps/bulldozer-js/src/databases/piledriver/data-structures/__fixtures__/v1.json @@ -0,0 +1,659 @@ +{ + "arity": 8, + "tree": { + "type": "AugmentedTreeMap", + "root": { + "ref": { + "version": 1, + "entries": [ + [ + { + "key": 5, + "id": "" + }, + 10 + ], + [ + { + "key": 10, + "id": "" + }, + 20 + ], + [ + { + "key": 15, + "id": "" + }, + 30 + ], + [ + { + "key": 20, + "id": "" + }, + 40 + ], + [ + { + "key": 25, + "id": "" + }, + 50 + ], + [ + { + "key": 30, + "id": "" + }, + 60 + ], + [ + { + "key": 35, + "id": "" + }, + 70 + ] + ], + "entryAugmentations": [ + 10, + 20, + 30, + 40, + 50, + 60, + 70 + ], + "children": [ + { + "ref": { + "version": 1, + "entries": [ + [ + { + "key": 1, + "id": "" + }, + 2 + ], + [ + { + "key": 2, + "id": "" + }, + 4 + ], + [ + { + "key": 3, + "id": "" + }, + 6 + ], + [ + { + "key": 4, + "id": "" + }, + 8 + ] + ], + "entryAugmentations": [ + 2, + 4, + 6, + 8 + ], + "children": [], + "augmentation": 20, + "size": 4, + "minKey": { + "key": 1, + "id": "" + }, + "maxKey": { + "key": 4, + "id": "" + } + }, + "augmentation": 20, + "size": 4, + "entryCount": 4, + "minKey": { + "key": 1, + "id": "" + }, + "maxKey": { + "key": 4, + "id": "" + } + }, + { + "ref": { + "version": 1, + "entries": [ + [ + { + "key": 6, + "id": "" + }, + 12 + ], + [ + { + "key": 7, + "id": "" + }, + 14 + ], + [ + { + "key": 8, + "id": "" + }, + 16 + ], + [ + { + "key": 9, + "id": "" + }, + 18 + ] + ], + "entryAugmentations": [ + 12, + 14, + 16, + 18 + ], + "children": [], + "augmentation": 60, + "size": 4, + "minKey": { + "key": 6, + "id": "" + }, + "maxKey": { + "key": 9, + "id": "" + } + }, + "augmentation": 60, + "size": 4, + "entryCount": 4, + "minKey": { + "key": 6, + "id": "" + }, + "maxKey": { + "key": 9, + "id": "" + } + }, + { + "ref": { + "version": 1, + "entries": [ + [ + { + "key": 11, + "id": "" + }, + 22 + ], + [ + { + "key": 12, + "id": "" + }, + 24 + ], + [ + { + "key": 13, + "id": "" + }, + 26 + ], + [ + { + "key": 14, + "id": "" + }, + 28 + ] + ], + "entryAugmentations": [ + 22, + 24, + 26, + 28 + ], + "children": [], + "augmentation": 100, + "size": 4, + "minKey": { + "key": 11, + "id": "" + }, + "maxKey": { + "key": 14, + "id": "" + } + }, + "augmentation": 100, + "size": 4, + "entryCount": 4, + "minKey": { + "key": 11, + "id": "" + }, + "maxKey": { + "key": 14, + "id": "" + } + }, + { + "ref": { + "version": 1, + "entries": [ + [ + { + "key": 16, + "id": "" + }, + 32 + ], + [ + { + "key": 17, + "id": "" + }, + 34 + ], + [ + { + "key": 18, + "id": "" + }, + 36 + ], + [ + { + "key": 19, + "id": "" + }, + 38 + ] + ], + "entryAugmentations": [ + 32, + 34, + 36, + 38 + ], + "children": [], + "augmentation": 140, + "size": 4, + "minKey": { + "key": 16, + "id": "" + }, + "maxKey": { + "key": 19, + "id": "" + } + }, + "augmentation": 140, + "size": 4, + "entryCount": 4, + "minKey": { + "key": 16, + "id": "" + }, + "maxKey": { + "key": 19, + "id": "" + } + }, + { + "ref": { + "version": 1, + "entries": [ + [ + { + "key": 21, + "id": "" + }, + 42 + ], + [ + { + "key": 22, + "id": "" + }, + 44 + ], + [ + { + "key": 23, + "id": "" + }, + 46 + ], + [ + { + "key": 24, + "id": "" + }, + 48 + ] + ], + "entryAugmentations": [ + 42, + 44, + 46, + 48 + ], + "children": [], + "augmentation": 180, + "size": 4, + "minKey": { + "key": 21, + "id": "" + }, + "maxKey": { + "key": 24, + "id": "" + } + }, + "augmentation": 180, + "size": 4, + "entryCount": 4, + "minKey": { + "key": 21, + "id": "" + }, + "maxKey": { + "key": 24, + "id": "" + } + }, + { + "ref": { + "version": 1, + "entries": [ + [ + { + "key": 26, + "id": "" + }, + 52 + ], + [ + { + "key": 27, + "id": "" + }, + 54 + ], + [ + { + "key": 28, + "id": "" + }, + 56 + ], + [ + { + "key": 29, + "id": "" + }, + 58 + ] + ], + "entryAugmentations": [ + 52, + 54, + 56, + 58 + ], + "children": [], + "augmentation": 220, + "size": 4, + "minKey": { + "key": 26, + "id": "" + }, + "maxKey": { + "key": 29, + "id": "" + } + }, + "augmentation": 220, + "size": 4, + "entryCount": 4, + "minKey": { + "key": 26, + "id": "" + }, + "maxKey": { + "key": 29, + "id": "" + } + }, + { + "ref": { + "version": 1, + "entries": [ + [ + { + "key": 31, + "id": "" + }, + 62 + ], + [ + { + "key": 32, + "id": "" + }, + 64 + ], + [ + { + "key": 33, + "id": "" + }, + 66 + ], + [ + { + "key": 34, + "id": "" + }, + 68 + ] + ], + "entryAugmentations": [ + 62, + 64, + 66, + 68 + ], + "children": [], + "augmentation": 260, + "size": 4, + "minKey": { + "key": 31, + "id": "" + }, + "maxKey": { + "key": 34, + "id": "" + } + }, + "augmentation": 260, + "size": 4, + "entryCount": 4, + "minKey": { + "key": 31, + "id": "" + }, + "maxKey": { + "key": 34, + "id": "" + } + }, + { + "ref": { + "version": 1, + "entries": [ + [ + { + "key": 36, + "id": "" + }, + 72 + ], + [ + { + "key": 37, + "id": "" + }, + 74 + ], + [ + { + "key": 38, + "id": "" + }, + 76 + ], + [ + { + "key": 39, + "id": "" + }, + 78 + ], + [ + { + "key": 40, + "id": "" + }, + 80 + ] + ], + "entryAugmentations": [ + 72, + 74, + 76, + 78, + 80 + ], + "children": [], + "augmentation": 380, + "size": 5, + "minKey": { + "key": 36, + "id": "" + }, + "maxKey": { + "key": 40, + "id": "" + } + }, + "augmentation": 380, + "size": 5, + "entryCount": 5, + "minKey": { + "key": 36, + "id": "" + }, + "maxKey": { + "key": 40, + "id": "" + } + } + ], + "augmentation": 1640, + "size": 40, + "minKey": { + "key": 1, + "id": "" + }, + "maxKey": { + "key": 40, + "id": "" + } + }, + "augmentation": 1640, + "size": 40, + "entryCount": 7, + "minKey": { + "key": 1, + "id": "" + }, + "maxKey": { + "key": 40, + "id": "" + } + } + }, + "queries": { + "size": 40, + "entriesInRange": [ + [ + 10, + 20 + ], + [ + 11, + 22 + ], + [ + 12, + 24 + ], + [ + 13, + 26 + ], + [ + 14, + 28 + ], + [ + 15, + 30 + ] + ], + "reversedLimited": [ + [ + 30, + 60 + ], + [ + 29, + 58 + ], + [ + 28, + 56 + ], + [ + 27, + 54 + ], + [ + 26, + 52 + ] + ], + "augmentationAll": 1640, + "augmentationRange": 330 + } +} diff --git a/apps/bulldozer-js/src/databases/piledriver/data-structures/serialization-compat.test.ts b/apps/bulldozer-js/src/databases/piledriver/data-structures/serialization-compat.test.ts new file mode 100644 index 000000000..20442a88f --- /dev/null +++ b/apps/bulldozer-js/src/databases/piledriver/data-structures/serialization-compat.test.ts @@ -0,0 +1,36 @@ +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { SerializationDump, computeQueries, treeFromDump } from "./__fixtures__/serialization-fixture-utils.js"; + +// Golden serialization fixtures. Each file freezes the persisted shape of an AugmentedTreeMap for a +// given node format version. Loading them with the *current* code proves that data written by older +// builds (already deployed to prod) still deserialises and behaves identically. If a change to the +// node (de)serialisation format breaks reading old data, these tests fail. Regenerate/add fixtures +// with `__fixtures__/generate-serialization-fixtures.ts` — never hand-edit existing ones. +const here = dirname(fileURLToPath(import.meta.url)); + +function loadFixture(name: string): SerializationDump { + // Boundary parse of a checked-in fixture file; the shape is exercised by every assertion below. + return JSON.parse(readFileSync(join(here, "__fixtures__", `${name}.json`), "utf8")) as SerializationDump; +} + +describe("augmented tree map serialization compatibility", () => { + // v0 = pre-versioning nodes (no `version`/`entryAugmentations`); v1 = current format. + for (const version of ["v0", "v1"] as const) { + it(`reads golden fixture ${version} with the current code`, async () => { + const fixture = loadFixture(version); + // Deserialising the frozen graph and recomputing every query must reproduce exactly what the + // writer observed — including the v0 path where augmentations are recomputed, not trusted. + expect(await computeQueries(treeFromDump(fixture))).toEqual(fixture.queries); + }); + + it(`mutates a tree loaded from golden fixture ${version} into a valid current-format tree`, async () => { + const fixture = loadFixture(version); + const updated = await treeFromDump(fixture).set(10, 999); + expect(await updated.get(10)).toBe(999); + expect(await updated.getAugmentation({})).toBe(fixture.queries.augmentationAll - 20 + 999); + }); + } +});