refactor(bs): unwrap lmdb errors

This helps observability

apps/bulldozer-js/src/databases/low-level/implementations/instant-availability.ts

@@ -121,7 +121,7 @@ export function declareInstantAvailabilityLowLevelDatabase(wrapped: LowLevelData
     // reconciled. We deliberately don't block any caller on this (no fsync on the
     // hot path); other awaiters of `underlyingAvailable` handle the rejection
     // themselves, this handler exists purely to report + avoid an unhandled
-    // rejection.
+    // rejection. (LMDB unwraps opaque commit failures in the low-level layer.)
     underlyingAvailable.catch(error => captureError("bulldozer-js:instant-availability-durable-commit", error));
     const record = { underlyingSeq, underlyingAvailable, isSettled: false };
     createdSeqRecords++;

apps/bulldozer-js/src/databases/low-level/implementations/lmdb.ts

@@ -5,6 +5,7 @@ import { shouldSuppressPeriodicBulldozerLogs } from "../../../logging.js";
 import { traceSpanHot } from "../../../otel.js";
 import { DatabaseSeq } from "../../index.js";
 import { LowLevelDatabase, LowLevelDatabaseDebugEntry, LowLevelKvDump, LowLevelKvStore } from "../index.js";
+import { unwrapLmdbCommitError } from "../unwrap-commit-error.js";
 
 type LmdbSeq = readonly [dbId: string, seqId: string] & { __brand: "hexclave-low-level-kv-store-seq" };
 type BinaryDatabase = lmdb.Database<Buffer, Uint8Array>;
@@ -206,44 +207,54 @@ export function declareLmdbLowLevelDatabase(options: { path: string, dbId?: stri
   const getDurabilityPromise = (seqId: string) => {
     return seqToDurability.get(seqId) ?? combinedSeqToDurability.get(seqId) ?? Promise.resolve();
   };
+  // LMDB may reject with an opaque "Commit failed" wrapper whose real status
+  // lives on `.commitError` (a Promise). Unwrap before surfacing to callers.
+  const awaitLmdbPromise = (promise: Promise<unknown>) => promise.catch(async (error) => {
+    throw await unwrapLmdbCommitError(error);
+  });
   const rememberAvailability = (seqId: string, promise: Promise<unknown>) => {
     const insertedAt = performance.now();
-    const availability = traceSpanHot({ description: "bulldozer-js.low-level.lmdb.availability", attributes: { "bulldozer.low_level.backend": "lmdb" } }, async () => await promise.then(() => {
+    const availability = traceSpanHot({ description: "bulldozer-js.low-level.lmdb.availability", attributes: { "bulldozer.low_level.backend": "lmdb" } }, async () => {
+      await awaitLmdbPromise(promise);
       activityStats.waitUntilAvailableResolveTotalMs += performance.now() - insertedAt;
       activityStats.waitUntilAvailableResolves++;
       seqToAvailability.delete(seqId);
-    }));
+    });
     availability.catch(() => {});
     seqToAvailability.set(seqId, availability);
   };
   const rememberDurability = (seqId: string, promise: Promise<unknown>) => {
     const insertedAt = performance.now();
-    const durability = traceSpanHot({ description: "bulldozer-js.low-level.lmdb.durability", attributes: { "bulldozer.low_level.backend": "lmdb" } }, async () => await promise.then(async () => await root.flushed).then(() => {
+    const durability = traceSpanHot({ description: "bulldozer-js.low-level.lmdb.durability", attributes: { "bulldozer.low_level.backend": "lmdb" } }, async () => {
+      await awaitLmdbPromise(promise);
+      await root.flushed;
       activityStats.waitUntilDurableResolveTotalMs += performance.now() - insertedAt;
       activityStats.waitUntilDurableResolves++;
       seqToDurability.delete(seqId);
-    }));
+    });
     durability.catch(() => {});
     seqToDurability.set(seqId, durability);
   };
   const rememberCombinedAvailability = (seqId: string, promise: Promise<unknown>) => {
     const insertedAt = performance.now();
-    const availability = traceSpanHot({ description: "bulldozer-js.low-level.lmdb.combinedAvailability", attributes: { "bulldozer.low_level.backend": "lmdb" } }, async () => await promise.then(() => {
+    const availability = traceSpanHot({ description: "bulldozer-js.low-level.lmdb.combinedAvailability", attributes: { "bulldozer.low_level.backend": "lmdb" } }, async () => {
+      await awaitLmdbPromise(promise);
       activityStats.combinedSeqAvailabilityResolveTotalMs += performance.now() - insertedAt;
       activityStats.combinedSeqAvailabilityResolves++;
       combinedSeqToAvailability.delete(seqId);
       combinedSeqDependencies.delete(seqId);
-    }));
+    });
     availability.catch(() => {});
     combinedSeqToAvailability.set(seqId, availability);
   };
   const rememberCombinedDurability = (seqId: string, promise: Promise<unknown>) => {
     const insertedAt = performance.now();
-    const durability = traceSpanHot({ description: "bulldozer-js.low-level.lmdb.combinedDurability", attributes: { "bulldozer.low_level.backend": "lmdb" } }, async () => await promise.then(() => {
+    const durability = traceSpanHot({ description: "bulldozer-js.low-level.lmdb.combinedDurability", attributes: { "bulldozer.low_level.backend": "lmdb" } }, async () => {
+      await awaitLmdbPromise(promise);
       activityStats.combinedSeqDurabilityResolveTotalMs += performance.now() - insertedAt;
       activityStats.combinedSeqDurabilityResolves++;
       combinedSeqToDurability.delete(seqId);
-    }));
+    });
     durability.catch(() => {});
     combinedSeqToDurability.set(seqId, durability);
   };
@@ -285,8 +296,9 @@ export function declareLmdbLowLevelDatabase(options: { path: string, dbId?: stri
       });
       for (const operation of operations) operation.resolve();
     } catch (error) {
-      for (const operation of operations) operation.reject(error);
-      throw error;
+      const unwrapped = await unwrapLmdbCommitError(error);
+      for (const operation of operations) operation.reject(unwrapped);
+      throw unwrapped;
     }
   };
   const flushPendingCommits = async () => {

apps/bulldozer-js/src/databases/low-level/unwrap-commit-error.test.ts

@@ -0,0 +1,36 @@
+import { describe, expect, it } from "vitest";
+import { unwrapLmdbCommitError } from "./unwrap-commit-error.js";
+
+describe("unwrapLmdbCommitError", () => {
+  it("passes through values that are not LMDB commit wrappers", async () => {
+    expect(await unwrapLmdbCommitError("plain")).toBe("plain");
+    const ordinary = new Error("ordinary failure");
+    expect(await unwrapLmdbCommitError(ordinary)).toBe(ordinary);
+  });
+
+  it("returns the underlying error after LMDB's rejectCommit → commitRejectPromise.reject sequence", async () => {
+    // Mirrors lmdb/write.js: rejectCommit() attaches an unsettled Promise and rejects
+    // the write; then the same sync callback settles that Promise with lmdbError(status).
+    let rejectCommitError!: (error: Error) => void;
+    const commitError = new Promise<never>((_resolve, reject) => {
+      rejectCommitError = reject;
+    });
+    commitError.catch(() => {});
+
+    const wrapper = Object.assign(new Error("Commit failed (see commitError for details)"), { commitError });
+    const underlying = new Error("ENOSPC: no space left on device");
+
+    const unwrappedPromise = unwrapLmdbCommitError(wrapper);
+    rejectCommitError(underlying);
+
+    expect(await unwrappedPromise).toBe(underlying);
+  });
+
+  it("returns HexclaveAssertionError when commitError does not settle to an Error", async () => {
+    const wrapper = Object.assign(new Error("Commit failed (see commitError for details)"), {
+      commitError: Promise.resolve("not-an-error"),
+    });
+    const result = await unwrapLmdbCommitError(wrapper);
+    expect(result).toMatchObject({ name: "HexclaveAssertionError" });
+  });
+});

apps/bulldozer-js/src/databases/low-level/unwrap-commit-error.ts

@@ -0,0 +1,26 @@
+import { HexclaveAssertionError } from "@hexclave/shared/dist/utils/errors";
+
+/**
+ * LMDB rejects failed writes as `Error("Commit failed (see commitError for details)")`
+ * with `.commitError` set to a Promise that later rejects with the real native status
+ * (ENOSPC, MDB_MAP_FULL, …). Await that before rethrowing/logging, or you only see the
+ * opaque wrapper.
+ *
+ * Why awaiting works: in lmdb/write.js, `rejectCommit()` attaches the Promise and rejects
+ * the write; then in the *same* sync callback, `commitRejectPromise.reject(lmdbError(status))`
+ * settles it. By the time our microtask runs, `.commitError` already has the real error.
+ */
+export async function unwrapLmdbCommitError(error: unknown): Promise<unknown> {
+  if (!(error instanceof Error) || !("commitError" in error)) return error;
+
+  const settled = error.commitError instanceof Promise
+    ? await error.commitError.then(value => value, rejected => rejected)
+    : error.commitError;
+
+  if (settled instanceof Error) return settled;
+
+  return new HexclaveAssertionError(
+    "LMDB commit failed but commitError did not settle to an Error",
+    { cause: error, commitError: settled },
+  );
+}