CodeCow/stack
1
0
Fork 0
mirror of https://github.com/stack-auth/stack.git synced 2026-07-20 21:29:36 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge f8c890e472 into 0814755e88

Browse Source
This commit is contained in:
Konsti Wohlwend 2026-07-18 20:58:51 +00:00 committed by GitHub
commit 0ded99e479
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 1882 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

201
.github/workflows/bulldozer-serialization-compat.yaml vendored Normal file
View File

@ -0,0 +1,201 @@
name: Bulldozer serialization compat
# Guards the on-disk format of the bulldozer/piledriver augmented B-tree. Bulldozer data is persisted
# in production, so a change that makes the current code unable to read previously-written data (or
# makes an older build unable to read data the new code writes, which matters for rollbacks) would
# corrupt or lose data. This workflow proves both directions by having each code version read a dump
# written by the other. The per-version golden-fixture unit test (serialization-compat.test.ts) is the
# first line of defence; the two cross-version jobs below (backward-compat and forward-compat) are the
# belt-and-suspenders check. They are split into separate jobs on purpose: backward compat (upgrade)
# must always hold, whereas forward compat (rollback) may legitimately break on a non-additive format
# change — keeping them separate makes a forward-compat failure explicit without masking backward compat.
on:
push:
branches-ignore:
- main
pull_request:
# This workflow only checks out code and runs tests; it never pushes or calls the GitHub API.
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/dev' }}
# DATA_STRUCTURES_DIR is repo-relative (used for diffing/overlaying). HARNESS_REL is relative to the
# bulldozer-js package, since the harness is run via `pnpm -C apps/bulldozer-js exec tsx`.
env:
DATA_STRUCTURES_DIR: apps/bulldozer-js/src/databases/piledriver/data-structures
HARNESS_REL: src/databases/piledriver/data-structures/__fixtures__/serialization-cross-version.ts
jobs:
check-changed:
name: Check if bulldozer serialization changed
runs-on: ubuntu-latest
outputs:
changed: ${{ steps.check-diff.outputs.changed }}
base_branch: ${{ steps.check-diff.outputs.base_branch }}
steps:
- name: Checkout current branch
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
fetch-depth: 0
- name: Check for serialization-relevant changes
id: check-diff
run: |
# dev compares to main, everything else compares to dev.
if [ "${{ github.ref }}" = "refs/heads/dev" ]; then
BASE_BRANCH="main"
else
BASE_BRANCH="dev"
fi
echo "base_branch=$BASE_BRANCH" >> $GITHUB_OUTPUT
git fetch origin "$BASE_BRANCH"
MERGE_BASE=$(git merge-base HEAD "origin/$BASE_BRANCH")
# Any change under the piledriver data-structures dir could touch the persisted node format.
if git diff --quiet "$MERGE_BASE" HEAD -- "$DATA_STRUCTURES_DIR"; then
echo "No serialization-relevant changes"
echo "changed=false" >> $GITHUB_OUTPUT
else
echo "Serialization-relevant changes detected"
echo "changed=true" >> $GITHUB_OUTPUT
fi
# Backward compatibility (production UPGRADE): the new code must read data the old code wrote.
# This is the direction that must ALWAYS hold — if it fails, deploying the new code would break
# reads of already-persisted production data.
backward-compat:
name: Backward compat (new code reads old data)
needs: check-changed
if: needs.check-changed.outputs.changed == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout current branch
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
path: current
- name: Checkout base branch (${{ needs.check-changed.outputs.base_branch }})
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
ref: ${{ needs.check-changed.outputs.base_branch }}
path: base
# The harness + its utils are new, so the base checkout doesn't have them. Overlay the current
# branch's copy onto the base tree; the harness only uses the long-stable AugmentedTreeMap /
# piledriver public API, so it compiles and runs against the base branch's implementation.
- name: Overlay harness onto base checkout
run: |
rm -rf "base/$DATA_STRUCTURES_DIR/__fixtures__"
cp -r "current/$DATA_STRUCTURES_DIR/__fixtures__" "base/$DATA_STRUCTURES_DIR/__fixtures__"
- name: Setup Node.js
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
with:
node-version: 22.x
- name: Setup pnpm
uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
with:
# Both checkouts live in subdirectories, so there's no package.json at the workspace root;
# read the pnpm version from the current checkout's package.json (`packageManager`).
package_json_file: current/package.json
- name: Install and build @hexclave/shared (current)
working-directory: current
run: |
pnpm install --frozen-lockfile
pnpm exec turbo run build --filter=@hexclave/shared
- name: Install and build @hexclave/shared (base)
working-directory: base
run: |
pnpm install --frozen-lockfile
pnpm exec turbo run build --filter=@hexclave/shared
- name: Emit dump with base code
working-directory: base
run: pnpm -C apps/bulldozer-js exec tsx "$HARNESS_REL" emit "$GITHUB_WORKSPACE/old.json"
- name: Current code reads base-written dump
working-directory: current
run: pnpm -C apps/bulldozer-js exec tsx "$HARNESS_REL" verify "$GITHUB_WORKSPACE/old.json"
# Forward compatibility (production ROLLBACK): the old code must read data the new code wrote.
# This can legitimately fail when a change makes the new format unreadable by older builds (e.g. a
# non-additive format change). It runs as its OWN job so that such a failure is explicit and
# actionable, and does NOT mask the backward-compat result above.
forward-compat:
name: Forward compat (old code reads new data)
needs: check-changed
if: needs.check-changed.outputs.changed == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout current branch
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
path: current
- name: Checkout base branch (${{ needs.check-changed.outputs.base_branch }})
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
ref: ${{ needs.check-changed.outputs.base_branch }}
path: base
# See the backward-compat job: the harness is new, so overlay it onto the base checkout.
- name: Overlay harness onto base checkout
run: |
rm -rf "base/$DATA_STRUCTURES_DIR/__fixtures__"
cp -r "current/$DATA_STRUCTURES_DIR/__fixtures__" "base/$DATA_STRUCTURES_DIR/__fixtures__"
- name: Setup Node.js
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
with:
node-version: 22.x
- name: Setup pnpm
uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
with:
# Both checkouts live in subdirectories, so there's no package.json at the workspace root;
# read the pnpm version from the current checkout's package.json (`packageManager`).
package_json_file: current/package.json
- name: Install and build @hexclave/shared (current)
working-directory: current
run: |
pnpm install --frozen-lockfile
pnpm exec turbo run build --filter=@hexclave/shared
- name: Install and build @hexclave/shared (base)
working-directory: base
run: |
pnpm install --frozen-lockfile
pnpm exec turbo run build --filter=@hexclave/shared
- name: Emit dump with current code
working-directory: current
run: pnpm -C apps/bulldozer-js exec tsx "$HARNESS_REL" emit "$GITHUB_WORKSPACE/new.json"
- name: Base code reads current-written dump
working-directory: base
run: pnpm -C apps/bulldozer-js exec tsx "$HARNESS_REL" verify "$GITHUB_WORKSPACE/new.json"
# Keeps the workflow (and the aggregate `all-good` check) green when serialization wasn't touched.
skip-unchanged:
name: No serialization changes (skipped)
needs: check-changed
if: needs.check-changed.outputs.changed == 'false'
runs-on: ubuntu-latest
steps:
- name: No serialization-relevant changes detected
run: echo "No changes under $DATA_STRUCTURES_DIR detected. Skipping cross-version compat."

64
apps/bulldozer-js/src/databases/piledriver/data-structures/__fixtures__/generate-serialization-fixtures.ts Normal file
View File

@ -0,0 +1,64 @@
/**
* Regenerates the golden serialization fixtures used by `serialization-compat.test.ts`.
*
* These fixtures freeze the on-disk shape of a persisted `AugmentedTreeMap` for each node format
* version, so that if a future change alters how nodes are (de)serialized in a way that breaks
* reading previously-persisted data, the compat test fails immediately.
*
* IMPORTANT: never edit or delete an existing fixture by hand — that would defeat the purpose.
* When you introduce a new node format version, bump `nodeFormatVersion` in augmented-tree-map.ts,
* add a new `writeFixture(...)` call below, and run:
*
* pnpm -C apps/bulldozer-js exec tsx src/databases/piledriver/data-structures/__fixtures__/generate-serialization-fixtures.ts
*
* The v0 fixture (no `version`/`entryAugmentations` fields) reproduces the exact node shape that
* builds before format versioning wrote, so we keep proving that legacy production data still reads.
*/
import { mkdirSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import {
FIXTURE_ARITY,
FIXTURE_ROW_COUNT,
Json,
SerializationDump,
buildSampleTree,
computeQueries,
resolveHeap,
} from "./serialization-fixture-utils.js";
const here = dirname(fileURLToPath(import.meta.url));
// Produces the pre-versioning node shape by dropping the fields v0 nodes never had, everywhere they
// appear in the graph.
function stripToV0(value: Json): Json {
if (Array.isArray(value)) return value.map(stripToV0);
if (value === null || typeof value !== "object") return value;
const out: { [key: string]: Json } = {};
for (const [key, child] of Object.entries(value)) {
if (key === "version" || key === "entryAugmentations") continue;
out[key] = stripToV0(child);
}
return out;
}
function writeFixture(name: string, body: SerializationDump) {
const path = join(here, `${name}.json`);
writeFileSync(path, `${JSON.stringify(body, null, 2)}\n`);
process.stdout.write(`wrote ${path}\n`);
}
async function main() {
mkdirSync(here, { recursive: true });
const tree = await buildSampleTree(FIXTURE_ARITY, FIXTURE_ROW_COUNT);
const treeObject = await resolveHeap(tree.toPiledriverObject());
const queries = await computeQueries(tree);
writeFixture("v1", { arity: FIXTURE_ARITY, tree: treeObject, queries });
writeFixture("v0", { arity: FIXTURE_ARITY, tree: stripToV0(treeObject), queries });
}
main().catch((error) => {
process.stderr.write(`${error?.stack ?? error}\n`);
process.exit(1);
});

65
apps/bulldozer-js/src/databases/piledriver/data-structures/__fixtures__/serialization-cross-version.ts Normal file
View File

@ -0,0 +1,65 @@
/**
* Cross-version serialization compatibility harness, used by the `serialization-compat` GitHub
* Actions workflow. Unlike the golden fixtures (which pin a frozen shape), this proves that two
* *different code versions* interoperate on the wire:
*
* emit <file> build the deterministic sample tree, persist it, and write a self-describing dump
* verify <file> read a dump written by (possibly) another code version and assert it reproduces
* the exact query results its writer observed
*
* The workflow runs `emit` under one checkout and `verify` under the other, in both directions:
* - new code reads a dump written by base code (backward compat: prod upgrade)
* - base code reads a dump written by new code (forward compat: prod rollback)
*
* This file (and serialization-fixture-utils.ts) is copied into the base-branch checkout by the
* workflow, so it must only rely on the long-stable AugmentedTreeMap / piledriver public API.
*/
import { readFileSync, writeFileSync } from "node:fs";
import {
FIXTURE_ARITY,
FIXTURE_ROW_COUNT,
SerializationDump,
buildSampleTree,
computeQueries,
resolveHeap,
treeFromDump,
} from "./serialization-fixture-utils.js";
async function emit(path: string) {
const tree = await buildSampleTree(FIXTURE_ARITY, FIXTURE_ROW_COUNT);
const dump: SerializationDump = {
arity: FIXTURE_ARITY,
tree: await resolveHeap(tree.toPiledriverObject()),
queries: await computeQueries(tree),
};
writeFileSync(path, `${JSON.stringify(dump, null, 2)}\n`);
process.stdout.write(`emitted serialization dump to ${path}\n`);
}
async function verify(path: string) {
// Boundary parse of a dump written by this or another code version; mismatches surface below.
const dump = JSON.parse(readFileSync(path, "utf8")) as SerializationDump;
const actual = await computeQueries(treeFromDump(dump));
const expected = JSON.stringify(dump.queries);
if (JSON.stringify(actual) !== expected) {
process.stderr.write(`serialization cross-version MISMATCH reading ${path}\n expected: ${expected}\n actual: ${JSON.stringify(actual)}\n`);
process.exit(1);
}
process.stdout.write(`verified serialization dump ${path} (queries reproduced exactly)\n`);
}
async function main() {
const [mode, path] = process.argv.slice(2);
if ((mode !== "emit" && mode !== "verify") || !path) {
process.stderr.write("usage: serialization-cross-version.ts <emit|verify> <file>\n");
process.exit(2);
return;
}
if (mode === "emit") await emit(path);
else await verify(path);
}
main().catch((error) => {
process.stderr.write(`${error?.stack ?? error}\n`);
process.exit(1);
});

111
apps/bulldozer-js/src/databases/piledriver/data-structures/__fixtures__/serialization-fixture-utils.ts Normal file
View File

@ -0,0 +1,111 @@
/**
* Shared helpers for the augmented-tree-map serialization compatibility checks:
* - the golden fixtures + `serialization-compat.test.ts` (checked-in, per-version regression), and
* - `serialization-cross-version.ts` (CI harness that has one code version write a dump and another
* read it back).
*
* Everything here depends only on the long-stable `AugmentedTreeMap` / piledriver public API, so this
* module also imports cleanly under older checkouts of the repo (which the cross-version CI job relies
* on when it runs the harness against the base branch's code).
*/
import { AugmentedTreeMap } from "../augmented-tree-map.js";
import { PiledriverHeapObject, PiledriverObject, isPiledriverHeapObjectSymbol } from "../../index.js";
export type Json = null | boolean | number | string | Json[] | { [key: string]: Json };
export type SerializationQueries = {
size: number,
entriesInRange: [number, number][],
reversedLimited: [number, number][],
augmentationAll: number,
augmentationRange: number,
};
// A self-describing serialized tree: the persisted object graph plus the query results the writer
// observed. A reader passes the check iff, after deserialising `tree`, it reproduces `queries`.
export type SerializationDump = {
arity: number,
tree: Json,
queries: SerializationQueries,
};
export const FIXTURE_ARITY = 8;
export const FIXTURE_ROW_COUNT = 40;
export function optionsFor(arity: number) {
return {
arity,
comparator: (a: number, b: number) => a - b,
initialAugmentation: 0,
extractAugmentation: (value: number) => value,
mergeAugmentations: (...values: number[]) => values.reduce((sum, value) => sum + value, 0),
};
}
// The deterministic dataset every writer produces, so dumps from different code versions are directly
// comparable: keys 1..FIXTURE_ROW_COUNT mapped to key*2.
export async function buildSampleTree(arity: number, rowCount: number): Promise<AugmentedTreeMap<number, number, number>> {
let tree = new AugmentedTreeMap<number, number, number>(optionsFor(arity));
for (let key = 1; key <= rowCount; key++) tree = await tree.set(key, key * 2);
return tree;
}
async function arrayFrom<T>(iterable: AsyncIterable<T>): Promise<T[]> {
const result: T[] = [];
for await (const item of iterable) result.push(item);
return result;
}
// The fixed set of queries used to attest that a deserialised tree behaves identically regardless of
// which code version wrote it. Covers forward/reverse iteration, limits, and full/partial aggregation.
export async function computeQueries(tree: AugmentedTreeMap<number, number, number>): Promise<SerializationQueries> {
return {
size: await tree.size(),
entriesInRange: await arrayFrom(tree.entries({ gte: 10, lte: 15 })),
reversedLimited: await arrayFrom(tree.entries({ lte: 30, reverse: true, limit: 5 })),
augmentationAll: await tree.getAugmentation({}),
augmentationRange: await tree.getAugmentation({ gte: 10, lte: 20 }),
};
}
// Recursively resolves every heap object into its inline content, producing a plain-JSON snapshot of
// the persisted object graph (heap references inlined where they point). Mirrors the real on-disk node
// shape without needing the internal Node/Child types.
export async function resolveHeap(value: PiledriverObject): Promise<Json> {
if (value === null || typeof value !== "object") return value;
if (isPiledriverHeapObjectSymbol in value) return await resolveHeap(await value.get());
if (Array.isArray(value)) return await Promise.all(value.map(resolveHeap));
const out: { [key: string]: Json } = {};
for (const [key, child] of Object.entries(value)) out[key] = await resolveHeap(child);
return out;
}
// Wraps a plain object as a heap object whose `get()` resolves to it, mirroring how piledriver hands
// deserialised heap objects back to the tree.
function asFrozenHeapObject(object: PiledriverObject): PiledriverHeapObject {
return {
async get() {
return object;
},
[isPiledriverHeapObjectSymbol]: true,
};
}
// Reconstructs the persisted object graph, wrapping every `ref` field back into a heap object (the
// only place the tree stores heap references). Keying on the real field name is intentional: if a
// future format renames/relocates it, the fixture's `ref` becomes unreadable and the check fails —
// which is exactly the regression we want to surface.
export function inflate(value: Json): PiledriverObject {
if (value === null || typeof value !== "object") return value;
if (Array.isArray(value)) return value.map(inflate);
const out: { [key: string]: PiledriverObject } = {};
for (const [key, child] of Object.entries(value)) {
const inflated = inflate(child);
out[key] = key === "ref" ? asFrozenHeapObject(inflated) : inflated;
}
return out;
}
export function treeFromDump(dump: SerializationDump): AugmentedTreeMap<number, number, number> {
return AugmentedTreeMap.fromPiledriverObject<number, number, number>(inflate(dump.tree), optionsFor(dump.arity));
}

592
apps/bulldozer-js/src/databases/piledriver/data-structures/__fixtures__/v0.json Normal file
View File

@ -0,0 +1,592 @@
{
"arity": 8,
"tree": {
"type": "AugmentedTreeMap",
"root": {
"ref": {
"entries": [
[
{
"key": 5,
"id": ""
},
10
],
[
{
"key": 10,
"id": ""
},
20
],
[
{
"key": 15,
"id": ""
},
30
],
[
{
"key": 20,
"id": ""
},
40
],
[
{
"key": 25,
"id": ""
},
50
],
[
{
"key": 30,
"id": ""
},
60
],
[
{
"key": 35,
"id": ""
},
70
]
],
"children": [
{
"ref": {
"entries": [
[
{
"key": 1,
"id": ""
},
2
],
[
{
"key": 2,
"id": ""
},
4
],
[
{
"key": 3,
"id": ""
},
6
],
[
{
"key": 4,
"id": ""
},
8
]
],
"children": [],
"augmentation": 20,
"size": 4,
"minKey": {
"key": 1,
"id": ""
},
"maxKey": {
"key": 4,
"id": ""
}
},
"augmentation": 20,
"size": 4,
"entryCount": 4,
"minKey": {
"key": 1,
"id": ""
},
"maxKey": {
"key": 4,
"id": ""
}
},
{
"ref": {
"entries": [
[
{
"key": 6,
"id": ""
},
12
],
[
{
"key": 7,
"id": ""
},
14
],
[
{
"key": 8,
"id": ""
},
16
],
[
{
"key": 9,
"id": ""
},
18
]
],
"children": [],
"augmentation": 60,
"size": 4,
"minKey": {
"key": 6,
"id": ""
},
"maxKey": {
"key": 9,
"id": ""
}
},
"augmentation": 60,
"size": 4,
"entryCount": 4,
"minKey": {
"key": 6,
"id": ""
},
"maxKey": {
"key": 9,
"id": ""
}
},
{
"ref": {
"entries": [
[
{
"key": 11,
"id": ""
},
22
],
[
{
"key": 12,
"id": ""
},
24
],
[
{
"key": 13,
"id": ""
},
26
],
[
{
"key": 14,
"id": ""
},
28
]
],
"children": [],
"augmentation": 100,
"size": 4,
"minKey": {
"key": 11,
"id": ""
},
"maxKey": {
"key": 14,
"id": ""
}
},
"augmentation": 100,
"size": 4,
"entryCount": 4,
"minKey": {
"key": 11,
"id": ""
},
"maxKey": {
"key": 14,
"id": ""
}
},
{
"ref": {
"entries": [
[
{
"key": 16,
"id": ""
},
32
],
[
{
"key": 17,
"id": ""
},
34
],
[
{
"key": 18,
"id": ""
},
36
],
[
{
"key": 19,
"id": ""
},
38
]
],
"children": [],
"augmentation": 140,
"size": 4,
"minKey": {
"key": 16,
"id": ""
},
"maxKey": {
"key": 19,
"id": ""
}
},
"augmentation": 140,
"size": 4,
"entryCount": 4,
"minKey": {
"key": 16,
"id": ""
},
"maxKey": {
"key": 19,
"id": ""
}
},
{
"ref": {
"entries": [
[
{
"key": 21,
"id": ""
},
42
],
[
{
"key": 22,
"id": ""
},
44
],
[
{
"key": 23,
"id": ""
},
46
],
[
{
"key": 24,
"id": ""
},
48
]
],
"children": [],
"augmentation": 180,
"size": 4,
"minKey": {
"key": 21,
"id": ""
},
"maxKey": {
"key": 24,
"id": ""
}
},
"augmentation": 180,
"size": 4,
"entryCount": 4,
"minKey": {
"key": 21,
"id": ""
},
"maxKey": {
"key": 24,
"id": ""
}
},
{
"ref": {
"entries": [
[
{
"key": 26,
"id": ""
},
52
],
[
{
"key": 27,
"id": ""
},
54
],
[
{
"key": 28,
"id": ""
},
56
],
[
{
"key": 29,
"id": ""
},
58
]
],
"children": [],
"augmentation": 220,
"size": 4,
"minKey": {
"key": 26,
"id": ""
},
"maxKey": {
"key": 29,
"id": ""
}
},
"augmentation": 220,
"size": 4,
"entryCount": 4,
"minKey": {
"key": 26,
"id": ""
},
"maxKey": {
"key": 29,
"id": ""
}
},
{
"ref": {
"entries": [
[
{
"key": 31,
"id": ""
},
62
],
[
{
"key": 32,
"id": ""
},
64
],
[
{
"key": 33,
"id": ""
},
66
],
[
{
"key": 34,
"id": ""
},
68
]
],
"children": [],
"augmentation": 260,
"size": 4,
"minKey": {
"key": 31,
"id": ""
},
"maxKey": {
"key": 34,
"id": ""
}
},
"augmentation": 260,
"size": 4,
"entryCount": 4,
"minKey": {
"key": 31,
"id": ""
},
"maxKey": {
"key": 34,
"id": ""
}
},
{
"ref": {
"entries": [
[
{
"key": 36,
"id": ""
},
72
],
[
{
"key": 37,
"id": ""
},
74
],
[
{
"key": 38,
"id": ""
},
76
],
[
{
"key": 39,
"id": ""
},
78
],
[
{
"key": 40,
"id": ""
},
80
]
],
"children": [],
"augmentation": 380,
"size": 5,
"minKey": {
"key": 36,
"id": ""
},
"maxKey": {
"key": 40,
"id": ""
}
},
"augmentation": 380,
"size": 5,
"entryCount": 5,
"minKey": {
"key": 36,
"id": ""
},
"maxKey": {
"key": 40,
"id": ""
}
}
],
"augmentation": 1640,
"size": 40,
"minKey": {
"key": 1,
"id": ""
},
"maxKey": {
"key": 40,
"id": ""
}
},
"augmentation": 1640,
"size": 40,
"entryCount": 7,
"minKey": {
"key": 1,
"id": ""
},
"maxKey": {
"key": 40,
"id": ""
}
}
},
"queries": {
"size": 40,
"entriesInRange": [
[
10,
20
],
[
11,
22
],
[
12,
24
],
[
13,
26
],
[
14,
28
],
[
15,
30
]
],
"reversedLimited": [
[
30,
60
],
[
29,
58
],
[
28,
56
],
[
27,
54
],
[
26,
52
]
],
"augmentationAll": 1640,
"augmentationRange": 330
}
}

659
apps/bulldozer-js/src/databases/piledriver/data-structures/__fixtures__/v1.json Normal file
View File

@ -0,0 +1,659 @@
{
"arity": 8,
"tree": {
"type": "AugmentedTreeMap",
"root": {
"ref": {
"version": 1,
"entries": [
[
{
"key": 5,
"id": ""
},
10
],
[
{
"key": 10,
"id": ""
},
20
],
[
{
"key": 15,
"id": ""
},
30
],
[
{
"key": 20,
"id": ""
},
40
],
[
{
"key": 25,
"id": ""
},
50
],
[
{
"key": 30,
"id": ""
},
60
],
[
{
"key": 35,
"id": ""
},
70
]
],
"entryAugmentations": [
10,
20,
30,
40,
50,
60,
70
],
"children": [
{
"ref": {
"version": 1,
"entries": [
[
{
"key": 1,
"id": ""
},
2
],
[
{
"key": 2,
"id": ""
},
4
],
[
{
"key": 3,
"id": ""
},
6
],
[
{
"key": 4,
"id": ""
},
8
]
],
"entryAugmentations": [
2,
4,
6,
8
],
"children": [],
"augmentation": 20,
"size": 4,
"minKey": {
"key": 1,
"id": ""
},
"maxKey": {
"key": 4,
"id": ""
}
},
"augmentation": 20,
"size": 4,
"entryCount": 4,
"minKey": {
"key": 1,
"id": ""
},
"maxKey": {
"key": 4,
"id": ""
}
},
{
"ref": {
"version": 1,
"entries": [
[
{
"key": 6,
"id": ""
},
12
],
[
{
"key": 7,
"id": ""
},
14
],
[
{
"key": 8,
"id": ""
},
16
],
[
{
"key": 9,
"id": ""
},
18
]
],
"entryAugmentations": [
12,
14,
16,
18
],
"children": [],
"augmentation": 60,
"size": 4,
"minKey": {
"key": 6,
"id": ""
},
"maxKey": {
"key": 9,
"id": ""
}
},
"augmentation": 60,
"size": 4,
"entryCount": 4,
"minKey": {
"key": 6,
"id": ""
},
"maxKey": {
"key": 9,
"id": ""
}
},
{
"ref": {
"version": 1,
"entries": [
[
{
"key": 11,
"id": ""
},
22
],
[
{
"key": 12,
"id": ""
},
24
],
[
{
"key": 13,
"id": ""
},
26
],
[
{
"key": 14,
"id": ""
},
28
]
],
"entryAugmentations": [
22,
24,
26,
28
],
"children": [],
"augmentation": 100,
"size": 4,
"minKey": {
"key": 11,
"id": ""
},
"maxKey": {
"key": 14,
"id": ""
}
},
"augmentation": 100,
"size": 4,
"entryCount": 4,
"minKey": {
"key": 11,
"id": ""
},
"maxKey": {
"key": 14,
"id": ""
}
},
{
"ref": {
"version": 1,
"entries": [
[
{
"key": 16,
"id": ""
},
32
],
[
{
"key": 17,
"id": ""
},
34
],
[
{
"key": 18,
"id": ""
},
36
],
[
{
"key": 19,
"id": ""
},
38
]
],
"entryAugmentations": [
32,
34,
36,
38
],
"children": [],
"augmentation": 140,
"size": 4,
"minKey": {
"key": 16,
"id": ""
},
"maxKey": {
"key": 19,
"id": ""
}
},
"augmentation": 140,
"size": 4,
"entryCount": 4,
"minKey": {
"key": 16,
"id": ""
},
"maxKey": {
"key": 19,
"id": ""
}
},
{
"ref": {
"version": 1,
"entries": [
[
{
"key": 21,
"id": ""
},
42
],
[
{
"key": 22,
"id": ""
},
44
],
[
{
"key": 23,
"id": ""
},
46
],
[
{
"key": 24,
"id": ""
},
48
]
],
"entryAugmentations": [
42,
44,
46,
48
],
"children": [],
"augmentation": 180,
"size": 4,
"minKey": {
"key": 21,
"id": ""
},
"maxKey": {
"key": 24,
"id": ""
}
},
"augmentation": 180,
"size": 4,
"entryCount": 4,
"minKey": {
"key": 21,
"id": ""
},
"maxKey": {
"key": 24,
"id": ""
}
},
{
"ref": {
"version": 1,
"entries": [
[
{
"key": 26,
"id": ""
},
52
],
[
{
"key": 27,
"id": ""
},
54
],
[
{
"key": 28,
"id": ""
},
56
],
[
{
"key": 29,
"id": ""
},
58
]
],
"entryAugmentations": [
52,
54,
56,
58
],
"children": [],
"augmentation": 220,
"size": 4,
"minKey": {
"key": 26,
"id": ""
},
"maxKey": {
"key": 29,
"id": ""
}
},
"augmentation": 220,
"size": 4,
"entryCount": 4,
"minKey": {
"key": 26,
"id": ""
},
"maxKey": {
"key": 29,
"id": ""
}
},
{
"ref": {
"version": 1,
"entries": [
[
{
"key": 31,
"id": ""
},
62
],
[
{
"key": 32,
"id": ""
},
64
],
[
{
"key": 33,
"id": ""
},
66
],
[
{
"key": 34,
"id": ""
},
68
]
],
"entryAugmentations": [
62,
64,
66,
68
],
"children": [],
"augmentation": 260,
"size": 4,
"minKey": {
"key": 31,
"id": ""
},
"maxKey": {
"key": 34,
"id": ""
}
},
"augmentation": 260,
"size": 4,
"entryCount": 4,
"minKey": {
"key": 31,
"id": ""
},
"maxKey": {
"key": 34,
"id": ""
}
},
{
"ref": {
"version": 1,
"entries": [
[
{
"key": 36,
"id": ""
},
72
],
[
{
"key": 37,
"id": ""
},
74
],
[
{
"key": 38,
"id": ""
},
76
],
[
{
"key": 39,
"id": ""
},
78
],
[
{
"key": 40,
"id": ""
},
80
]
],
"entryAugmentations": [
72,
74,
76,
78,
80
],
"children": [],
"augmentation": 380,
"size": 5,
"minKey": {
"key": 36,
"id": ""
},
"maxKey": {
"key": 40,
"id": ""
}
},
"augmentation": 380,
"size": 5,
"entryCount": 5,
"minKey": {
"key": 36,
"id": ""
},
"maxKey": {
"key": 40,
"id": ""
}
}
],
"augmentation": 1640,
"size": 40,
"minKey": {
"key": 1,
"id": ""
},
"maxKey": {
"key": 40,
"id": ""
}
},
"augmentation": 1640,
"size": 40,
"entryCount": 7,
"minKey": {
"key": 1,
"id": ""
},
"maxKey": {
"key": 40,
"id": ""
}
}
},
"queries": {
"size": 40,
"entriesInRange": [
[
10,
20
],
[
11,
22
],
[
12,
24
],
[
13,
26
],
[
14,
28
],
[
15,
30
]
],
"reversedLimited": [
[
30,
60
],
[
29,
58
],
[
28,
56
],
[
27,
54
],
[
26,
52
]
],
"augmentationAll": 1640,
"augmentationRange": 330
}
}

31
apps/bulldozer-js/src/databases/piledriver/data-structures/augmented-tree-map.test.ts
View File

@ -288,6 +288,37 @@ describe("AugmentedTreeMap", () => {
expect(await tree.getAugmentation({ gte: 20, lte: 25 })).toBe(135);
});
it("reads legacy nodes persisted before format versioning", async () => {
// Nodes written by an older build have no `version` and no `entryAugmentations`. Simulate such
// an on-disk tree by stripping those fields on read, then confirm reads still work (the tree
// must fall back to recomputing augmentations rather than trusting absent cached ones).
const legacyRef = (ref: PiledriverHeapObject): PiledriverHeapObject => ({
async get() {
const node: any = await ref.get();
if (!isNode(node)) return node;
const { version: _version, entryAugmentations: _entryAugmentations, ...legacy } = node as any;
return { ...legacy, children: node.children.map((child: any) => child ? { ...child, ref: legacyRef(child.ref) } : child) };
},
[isPiledriverHeapObjectSymbol]: true as const,
} as PiledriverHeapObject);
const built = await build(100, 8);
const serialized = built.toPiledriverObject();
const legacy = AugmentedTreeMap.fromPiledriverObject(
{ ...serialized, root: serialized.root ? { ...serialized.root, ref: legacyRef(serialized.root.ref) } : null },
options(8),
);
expect(await arrayFrom(legacy.entries({ gte: 20, lt: 25 }))).toEqual([[20, 20], [21, 21], [22, 22], [23, 23], [24, 24]]);
expect(await legacy.getAugmentation({ gte: 20, lte: 25 })).toBe(135);
expect(await legacy.getAugmentation({})).toBe(5050);
// A mutation over a legacy tree must produce a valid current-format tree.
const updated = await legacy.set(50, -1);
expect(await updated.get(50)).toBe(-1);
expect(await updated.getAugmentation({})).toBe(5050 - 50 + -1);
});
it("defaults arity when omitted", async () => {
let tree = new AugmentedTreeMap({
comparator: (a: number, b: number) => a - b,

161
apps/bulldozer-js/src/databases/piledriver/data-structures/augmented-tree-map.ts
View File

@ -30,7 +30,18 @@ type StoredValue<V extends PiledriverObject> = V | PiledriverHeapObject;
type Split<K, V extends PiledriverObject, A> = { entry: Entry<K, StoredValue<V>>, right: Child<K, A> };
// Persisted B-tree node. Children are heap objects, so unchanged subtrees are shared across versions.
type Node<K, V extends PiledriverObject, A> = {
// On-disk format version for this node. Absent (undefined) on nodes persisted before versioning
// was introduced — those are treated as version 0. Bump `nodeFormatVersion` whenever a persisted
// field is added/changed in a way that makes previously-written data unsafe to trust, and gate
// the reads of that field on the version so old nodes keep deserialising correctly.
version?: number,
entries: Entry<K, StoredValue<V>>[],
// Cached per-entry augmentations, parallel to `entries`. When a node is path-copied during a
// mutation, unchanged entries (same tuple identity) can reuse their cached augmentation instead
// of re-calling extractAugmentation. This avoids creating duplicate heap objects for the same
// data, letting the downstream serializer recognise them as already-written (heap cache hits).
// Only trusted when `version >= 1` (see `nodeFormatVersion`).
entryAugmentations?: A[],
children: Child<K, A>[],
augmentation: A,
size: number,
@ -66,6 +77,10 @@ const lowerEntryId = Symbol("lower-entry-id");
const upperEntryId = Symbol("upper-entry-id");
const mapEntryId = "";
const defaultArity = 32;
// Current on-disk B-tree node format version. Version 1 introduced cached per-entry augmentations
// (`entryAugmentations`). Nodes without a `version` field were written before versioning (version 0)
// and have their augmentations recomputed on read. See the `version` field on `Node`.
const nodeFormatVersion = 1;
// Code-unit comparisons; localeCompare would persist a tree whose order depends on the runtime locale.
function compareStrings(a: string, b: string) {
@ -291,11 +306,16 @@ export class AugmentedTreeMultiMap<Key extends PiledriverObject, Value extends P
}
// Recomputes all cached metadata for a freshly path-copied node.
private async make(entries: Entry<MultiKey<Key, EntryId>, StoredValue<Value>>[], children: Child<MultiKey<Key, EntryId>, Augmentation>[] = []) {
// cachedEntryAugs: when provided, maps entry tuple references to their previously computed
// augmentations. Unchanged entries (same tuple identity) get a cache hit, avoiding a redundant
// extractAugmentation call and — critically — reusing the same PiledriverHeapObjects so that
// downstream serialization sees them as already-written (heap object cache hits).
private async make(entries: Entry<MultiKey<Key, EntryId>, StoredValue<Value>>[], children: Child<MultiKey<Key, EntryId>, Augmentation>[] = [], cachedEntryAugs?: ReadonlyMap<object, Augmentation>) {
if (!entries.length && children.length !== 1) throw new Error("Invalid empty B-tree node");
if (!entries.length) {
const [onlyChild] = children;
const node = {
version: nodeFormatVersion,
entries,
children,
augmentation: onlyChild.augmentation,
@ -307,6 +327,7 @@ export class AugmentedTreeMultiMap<Key extends PiledriverObject, Value extends P
}
const augmentations: Augmentation[] = [];
const entryAugmentations: Augmentation[] = [];
let size = entries.length;
for (let i = 0; i < entries.length; i++) {
@ -315,7 +336,12 @@ export class AugmentedTreeMultiMap<Key extends PiledriverObject, Value extends P
augmentations.push(child.augmentation);
size += child.size;
}
augmentations.push(await this.options.extractAugmentation(await this.loadValue(entries[i][1]), entries[i][0].key, entries[i][0].id));
const cached = cachedEntryAugs?.get(entries[i]);
const entryAug = cached !== undefined
? cached
: await this.options.extractAugmentation(await this.loadValue(entries[i][1]), entries[i][0].key, entries[i][0].id);
entryAugmentations.push(entryAug);
augmentations.push(entryAug);
}
const lastChild = children.at(entries.length);
if (lastChild) {
@ -324,7 +350,9 @@ export class AugmentedTreeMultiMap<Key extends PiledriverObject, Value extends P
}
const node = {
version: nodeFormatVersion,
entries,
entryAugmentations,
children,
augmentation: await this.options.mergeAugmentations(...augmentations),
size,
@ -334,6 +362,34 @@ export class AugmentedTreeMultiMap<Key extends PiledriverObject, Value extends P
return this.child(asHeapObject(node as PiledriverObject), node);
}
// Builds a Map from entry tuple references to their cached augmentations, enabling O(1) lookups
// during make(). Returns undefined if the node has no cached augmentations (e.g. old format).
private buildEntryAugCache(node: Node<MultiKey<Key, EntryId>, Value, Augmentation>): ReadonlyMap<object, Augmentation> | undefined {
if ((node.version ?? 0) < 1) return undefined;
if (!node.entryAugmentations || node.entryAugmentations.length !== node.entries.length) return undefined;
const cache = new Map<object, Augmentation>();
for (let i = 0; i < node.entries.length; i++) {
cache.set(node.entries[i], node.entryAugmentations[i]);
}
return cache;
}
// Combines a parent entry-aug cache (for the separator entry) with a child node's cache into a
// single map for make(). Used by borrow paths where the receiving node is built from a parent
// separator plus a child's entries.
private combineCaches(
parentCache: ReadonlyMap<object, Augmentation> | undefined,
separator: Entry<MultiKey<Key, EntryId>, StoredValue<Value>>,
childCache: ReadonlyMap<object, Augmentation> | undefined,
): ReadonlyMap<object, Augmentation> | undefined {
const sepAug = parentCache?.get(separator);
if (sepAug === undefined && !childCache) return undefined;
const combined = new Map<object, Augmentation>();
if (childCache) for (const [e, a] of childCache) combined.set(e, a);
if (sepAug !== undefined) combined.set(separator, sepAug);
return combined;
}
private arity() {
return Math.max(3, this.options.arity ?? defaultArity);
}
@ -377,15 +433,15 @@ export class AugmentedTreeMultiMap<Key extends PiledriverObject, Value extends P
return result.split ? await this.make([result.split.entry], [result.root, result.split.right]) : result.root;
}
private async split(entries: Entry<MultiKey<Key, EntryId>, StoredValue<Value>>[], children: Child<MultiKey<Key, EntryId>, Augmentation>[]) {
private async split(entries: Entry<MultiKey<Key, EntryId>, StoredValue<Value>>[], children: Child<MultiKey<Key, EntryId>, Augmentation>[], cachedEntryAugs?: ReadonlyMap<object, Augmentation>) {
const middle = entries.length >> 1;
const left = await this.make(entries.slice(0, middle), children.length ? children.slice(0, middle + 1) : []);
const right = await this.make(entries.slice(middle + 1), children.length ? children.slice(middle + 1) : []);
const left = await this.make(entries.slice(0, middle), children.length ? children.slice(0, middle + 1) : [], cachedEntryAugs);
const right = await this.make(entries.slice(middle + 1), children.length ? children.slice(middle + 1) : [], cachedEntryAugs);
return { root: left, split: { entry: entries[middle], right } };
}
private async done(entries: Entry<MultiKey<Key, EntryId>, StoredValue<Value>>[], children: Child<MultiKey<Key, EntryId>, Augmentation>[], added: boolean) {
const result = entries.length > this.maxEntries() ? await this.split(entries, children) : { root: await this.make(entries, children) };
private async done(entries: Entry<MultiKey<Key, EntryId>, StoredValue<Value>>[], children: Child<MultiKey<Key, EntryId>, Augmentation>[], added: boolean, cachedEntryAugs?: ReadonlyMap<object, Augmentation>) {
const result = entries.length > this.maxEntries() ? await this.split(entries, children, cachedEntryAugs) : { root: await this.make(entries, children, cachedEntryAugs) };
return { ...result, added };
}
@ -396,16 +452,17 @@ export class AugmentedTreeMultiMap<Key extends PiledriverObject, Value extends P
const { index, found } = this.search(node.entries, key);
const entries = [...node.entries];
const children = [...node.children];
const cachedEntryAugs = this.buildEntryAugCache(node);
if (found) {
if (!replace) throw new Error("Key already exists");
entries[index] = [key, this.storeValue(value)];
return await this.done(entries, children, false);
return await this.done(entries, children, false, cachedEntryAugs);
}
if (!children.length) {
entries.splice(index, 0, [key, this.storeValue(value)]);
return await this.done(entries, children, true);
return await this.done(entries, children, true, cachedEntryAugs);
}
const child = await this.upsert(children[index].ref, key, value, replace);
@ -414,7 +471,7 @@ export class AugmentedTreeMultiMap<Key extends PiledriverObject, Value extends P
entries.splice(index, 0, child.split.entry);
children.splice(index + 1, 0, child.split.right);
}
return await this.done(entries, children, child.added);
return await this.done(entries, children, child.added, cachedEntryAugs);
}
private async deleteFrom(child: Child<MultiKey<Key, EntryId>, Augmentation> | null, key: MultiKey<Key, EntryId>, isRoot = false): Promise<{ root: Child<MultiKey<Key, EntryId>, Augmentation> | null, deleted: boolean }> {
@ -424,11 +481,12 @@ export class AugmentedTreeMultiMap<Key extends PiledriverObject, Value extends P
const { index, found } = this.search(node.entries, key);
const entries = [...node.entries];
const children = [...node.children] as (Child<MultiKey<Key, EntryId>, Augmentation> | null)[];
const cachedEntryAugs = this.buildEntryAugCache(node);
if (found) {
if (!children.length) {
entries.splice(index, 1);
return await this.afterDelete(entries, children, isRoot, true);
return await this.afterDelete(entries, children, isRoot, true, cachedEntryAugs);
}
// Replace the separator with its predecessor or successor (from whichever side has more
@ -441,12 +499,12 @@ export class AugmentedTreeMultiMap<Key extends PiledriverObject, Value extends P
const { root, entry } = await this.deleteMin(right);
entries[index] = entry;
children[index + 1] = root;
return await this.fixChild(entries, children, index + 1, isRoot);
return await this.fixChild(entries, children, index + 1, isRoot, cachedEntryAugs);
}
const { root, entry } = await this.deleteMax(left);
entries[index] = entry;
children[index] = root;
return await this.fixChild(entries, children, index, isRoot);
return await this.fixChild(entries, children, index, isRoot, cachedEntryAugs);
}
if (!children.length) return { root: child, deleted: false };
@ -454,89 +512,110 @@ export class AugmentedTreeMultiMap<Key extends PiledriverObject, Value extends P
const deletedChild = await this.deleteFrom(children[index], key);
if (!deletedChild.deleted) return { root: child, deleted: false };
children[index] = deletedChild.root;
return await this.fixChild(entries, children, index, isRoot);
return await this.fixChild(entries, children, index, isRoot, cachedEntryAugs);
}
private async deleteMin(child: Child<MultiKey<Key, EntryId>, Augmentation>): Promise<{ root: Child<MultiKey<Key, EntryId>, Augmentation> | null, entry: Entry<MultiKey<Key, EntryId>, StoredValue<Value>> }> {
const node = (await this.node(child.ref))!;
const entries = [...node.entries];
const children = [...node.children] as (Child<MultiKey<Key, EntryId>, Augmentation> | null)[];
const cachedEntryAugs = this.buildEntryAugCache(node);
if (!children.length) {
const [entry] = entries.splice(0, 1);
return { root: entries.length ? await this.make(entries) : null, entry };
return { root: entries.length ? await this.make(entries, [], cachedEntryAugs) : null, entry };
}
const result = await this.deleteMin(children[0]!);
children[0] = result.root;
return { root: (await this.fixChild(entries, children, 0, false)).root, entry: result.entry };
return { root: (await this.fixChild(entries, children, 0, false, cachedEntryAugs)).root, entry: result.entry };
}
private async deleteMax(child: Child<MultiKey<Key, EntryId>, Augmentation>): Promise<{ root: Child<MultiKey<Key, EntryId>, Augmentation> | null, entry: Entry<MultiKey<Key, EntryId>, StoredValue<Value>> }> {
const node = (await this.node(child.ref))!;
const entries = [...node.entries];
const children = [...node.children] as (Child<MultiKey<Key, EntryId>, Augmentation> | null)[];
const cachedEntryAugs = this.buildEntryAugCache(node);
if (!children.length) {
const entry = entries.pop()!;
return { root: entries.length ? await this.make(entries) : null, entry };
return { root: entries.length ? await this.make(entries, [], cachedEntryAugs) : null, entry };
}
const index = children.length - 1;
const result = await this.deleteMax(children[index]!);
children[index] = result.root;
return { root: (await this.fixChild(entries, children, index, false)).root, entry: result.entry };
return { root: (await this.fixChild(entries, children, index, false, cachedEntryAugs)).root, entry: result.entry };
}
private async afterDelete(entries: Entry<MultiKey<Key, EntryId>, StoredValue<Value>>[], children: (Child<MultiKey<Key, EntryId>, Augmentation> | null)[], isRoot: boolean, deleted: boolean) {
private async afterDelete(entries: Entry<MultiKey<Key, EntryId>, StoredValue<Value>>[], children: (Child<MultiKey<Key, EntryId>, Augmentation> | null)[], isRoot: boolean, deleted: boolean, cachedEntryAugs?: ReadonlyMap<object, Augmentation>) {
const liveChildren = children.filter(child => child !== null);
if (!entries.length) return { root: isRoot ? liveChildren[0] ?? null : liveChildren[0] ? await this.make([], [liveChildren[0]]) : null, deleted };
return { root: await this.make(entries, liveChildren), deleted };
return { root: await this.make(entries, liveChildren, cachedEntryAugs), deleted };
}
private async fixChild(entries: Entry<MultiKey<Key, EntryId>, StoredValue<Value>>[], children: (Child<MultiKey<Key, EntryId>, Augmentation> | null)[], index: number, isRoot: boolean): Promise<{ root: Child<MultiKey<Key, EntryId>, Augmentation> | null, deleted: boolean }> {
private async fixChild(entries: Entry<MultiKey<Key, EntryId>, StoredValue<Value>>[], children: (Child<MultiKey<Key, EntryId>, Augmentation> | null)[], index: number, isRoot: boolean, cachedEntryAugs?: ReadonlyMap<object, Augmentation>): Promise<{ root: Child<MultiKey<Key, EntryId>, Augmentation> | null, deleted: boolean }> {
const child = children[index];
if (child && child.entryCount >= this.minEntries()) return await this.afterDelete(entries, children, isRoot, true);
if (child && child.entryCount >= this.minEntries()) return await this.afterDelete(entries, children, isRoot, true, cachedEntryAugs);
const left = children[index - 1];
if (left && left.entryCount > this.minEntries()) {
const leftNode = (await this.node(left.ref))!;
const childNode = child ? (await this.node(child.ref))! : { entries: [], children: [] };
const loadedChild = child ? (await this.node(child.ref))! : undefined;
const childNode = loadedChild ?? { entries: [] as Entry<MultiKey<Key, EntryId>, StoredValue<Value>>[], children: [] as Child<MultiKey<Key, EntryId>, Augmentation>[] };
const separator = entries[index - 1];
const borrowedEntry = leftNode.entries[leftNode.entries.length - 1];
const borrowedChild = leftNode.children.at(-1);
entries[index - 1] = borrowedEntry;
children[index - 1] = await this.make(leftNode.entries.slice(0, -1), leftNode.children.slice(0, -1));
children[index] = await this.make([separator, ...childNode.entries], borrowedChild === undefined ? childNode.children : [borrowedChild, ...childNode.children]);
return await this.afterDelete(entries, children, isRoot, true);
children[index - 1] = await this.make(leftNode.entries.slice(0, -1), leftNode.children.slice(0, -1), this.buildEntryAugCache(leftNode));
const borrowRecvCache = this.combineCaches(cachedEntryAugs, separator, loadedChild ? this.buildEntryAugCache(loadedChild) : undefined);
children[index] = await this.make([separator, ...childNode.entries], borrowedChild === undefined ? childNode.children : [borrowedChild, ...childNode.children], borrowRecvCache);
return await this.afterDelete(entries, children, isRoot, true, cachedEntryAugs);
}
const right = children[index + 1];
if (right && right.entryCount > this.minEntries()) {
const childNode = child ? (await this.node(child.ref))! : { entries: [], children: [] };
const loadedChild = child ? (await this.node(child.ref))! : undefined;
const childNode = loadedChild ?? { entries: [] as Entry<MultiKey<Key, EntryId>, StoredValue<Value>>[], children: [] as Child<MultiKey<Key, EntryId>, Augmentation>[] };
const rightNode = (await this.node(right.ref))!;
const separator = entries[index];
const borrowedEntry = rightNode.entries[0];
const borrowedChild = rightNode.children.at(0);
entries[index] = borrowedEntry;
children[index] = await this.make([...childNode.entries, separator], borrowedChild === undefined ? childNode.children : [...childNode.children, borrowedChild]);
children[index + 1] = await this.make(rightNode.entries.slice(1), rightNode.children.slice(1));
return await this.afterDelete(entries, children, isRoot, true);
const borrowRecvCache = this.combineCaches(cachedEntryAugs, separator, loadedChild ? this.buildEntryAugCache(loadedChild) : undefined);
children[index] = await this.make([...childNode.entries, separator], borrowedChild === undefined ? childNode.children : [...childNode.children, borrowedChild], borrowRecvCache);
children[index + 1] = await this.make(rightNode.entries.slice(1), rightNode.children.slice(1), this.buildEntryAugCache(rightNode));
return await this.afterDelete(entries, children, isRoot, true, cachedEntryAugs);
}
const mergeIndex = left ? index - 1 : index;
const merged = await this.mergeChildren(children[mergeIndex], entries[mergeIndex], children[mergeIndex + 1]);
const merged = await this.mergeChildren(children[mergeIndex], entries[mergeIndex], children[mergeIndex + 1], cachedEntryAugs);
entries.splice(mergeIndex, 1);
children.splice(mergeIndex, 2, merged);
return await this.afterDelete(entries, children, isRoot, true);
return await this.afterDelete(entries, children, isRoot, true, cachedEntryAugs);
}
private async mergeChildren(left: Child<MultiKey<Key, EntryId>, Augmentation> | null, entry: Entry<MultiKey<Key, EntryId>, StoredValue<Value>>, right: Child<MultiKey<Key, EntryId>, Augmentation> | null) {
const leftNode = left ? (await this.node(left.ref))! : { entries: [], children: [] };
const rightNode = right ? (await this.node(right.ref))! : { entries: [], children: [] };
private async mergeChildren(left: Child<MultiKey<Key, EntryId>, Augmentation> | null, entry: Entry<MultiKey<Key, EntryId>, StoredValue<Value>>, right: Child<MultiKey<Key, EntryId>, Augmentation> | null, parentEntryAugs?: ReadonlyMap<object, Augmentation>) {
const leftNode = left ? (await this.node(left.ref))! : null;
const rightNode = right ? (await this.node(right.ref))! : null;
// Build a combined entry augmentation cache from both child nodes and the parent separator
const cache = new Map<object, Augmentation>();
if (leftNode) {
const lc = this.buildEntryAugCache(leftNode);
if (lc) for (const [e, a] of lc) cache.set(e, a);
}
if (rightNode) {
const rc = this.buildEntryAugCache(rightNode);
if (rc) for (const [e, a] of rc) cache.set(e, a);
}
const sepAug = parentEntryAugs?.get(entry);
if (sepAug !== undefined) cache.set(entry, sepAug);
return await this.make(
[...leftNode.entries, entry, ...rightNode.entries],
[...leftNode.children, ...rightNode.children],
[...(leftNode?.entries ?? []), entry, ...(rightNode?.entries ?? [])],
[...(leftNode?.children ?? []), ...(rightNode?.children ?? [])],
cache.size ? cache : undefined,
);
}
@ -558,7 +637,13 @@ export class AugmentedTreeMultiMap<Key extends PiledriverObject, Value extends P
for (let i = 0; i < node.entries.length; i++) {
result = await this.merge(result, await this.augmentation(node.children[i] ?? null, range));
if (aboveLowerBound(node.entries[i][0]) && belowUpperBound(node.entries[i][0])) {
result = await this.merge(result, await this.options.extractAugmentation(await this.loadValue(node.entries[i][1]), node.entries[i][0].key, node.entries[i][0].id));
// Trust the cached augmentation only for versioned nodes, and check `!== undefined` (not
// `??`) so a legitimately-cached `null` augmentation is still a hit rather than re-extracted.
const cachedEntryAug = (node.version ?? 0) >= 1 ? node.entryAugmentations?.[i] : undefined;
const entryAug = cachedEntryAug !== undefined
? cachedEntryAug
: await this.options.extractAugmentation(await this.loadValue(node.entries[i][1]), node.entries[i][0].key, node.entries[i][0].id);
result = await this.merge(result, entryAug);
}
}
return await this.merge(result, await this.augmentation(node.children[node.entries.length] ?? null, range));

36
apps/bulldozer-js/src/databases/piledriver/data-structures/serialization-compat.test.ts Normal file
View File

@ -0,0 +1,36 @@
import { readFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { describe, expect, it } from "vitest";
import { SerializationDump, computeQueries, treeFromDump } from "./__fixtures__/serialization-fixture-utils.js";
// Golden serialization fixtures. Each file freezes the persisted shape of an AugmentedTreeMap for a
// given node format version. Loading them with the *current* code proves that data written by older
// builds (already deployed to prod) still deserialises and behaves identically. If a change to the
// node (de)serialisation format breaks reading old data, these tests fail. Regenerate/add fixtures
// with `__fixtures__/generate-serialization-fixtures.ts` — never hand-edit existing ones.
const here = dirname(fileURLToPath(import.meta.url));
function loadFixture(name: string): SerializationDump {
// Boundary parse of a checked-in fixture file; the shape is exercised by every assertion below.
return JSON.parse(readFileSync(join(here, "__fixtures__", `${name}.json`), "utf8")) as SerializationDump;
}
describe("augmented tree map serialization compatibility", () => {
// v0 = pre-versioning nodes (no `version`/`entryAugmentations`); v1 = current format.
for (const version of ["v0", "v1"] as const) {
it(`reads golden fixture ${version} with the current code`, async () => {
const fixture = loadFixture(version);
// Deserialising the frozen graph and recomputing every query must reproduce exactly what the
// writer observed — including the v0 path where augmentations are recomputed, not trusted.
expect(await computeQueries(treeFromDump(fixture))).toEqual(fixture.queries);
});
it(`mutates a tree loaded from golden fixture ${version} into a valid current-format tree`, async () => {
const fixture = loadFixture(version);
const updated = await treeFromDump(fixture).set(10, 999);
expect(await updated.get(10)).toBe(999);
expect(await updated.getAugmentation({})).toBe(fixture.queries.augmentationAll - 20 + 999);
});
}
});