mirror of
https://github.com/stack-auth/stack.git
synced 2026-07-20 21:29:36 +08:00
feat(analytics): spans telemetry surface + event parent_span_ids
Add a generic Spans telemetry surface alongside events, written directly to ClickHouse (spans are siblings of events, not routed through ext-db-sync). - analytics_internal.spans table (ReplacingMergeTree, versioned) + default.spans view, isolated by a row policy and granted to limited_user exactly like default.events. - Span types: $session-replay and $session-replay-segment (per browser tab), written directly from the session-replays batch route and re-written per batch so their end advances; $refresh-token, derived in the view from the synced refresh_tokens dimension. - events.parent_span_ids: additive column holding the root-first ancestor span-id list ([rti, sri, srsi]), stamped on each event insert. Typed-id prefixes live only on span ids / parent_span_ids; scalar id columns stay raw so existing customer SQL keeps working (additive, backwards compatible). - SDK: EventTracker and SessionRecorder now share one per-tab session_replay_segment_id (each previously minted its own, so events could not be correlated to a replay recording). - Dashboard: Spans tab on the analytics tables page. - Tests: spans unit tests + e2e for session replays, analytics events batch, and the analytics query surface.
This commit is contained in:
parent
db94ec2d94
commit
8d124a4517
@ -33,12 +33,14 @@ export async function runClickhouseMigrations() {
|
||||
client.command({ query: REFRESH_TOKENS_TABLE_BASE_SQL }),
|
||||
client.command({ query: CONNECTED_ACCOUNTS_TABLE_BASE_SQL }),
|
||||
client.command({ query: CLICKMAP_EVENTS_TABLE_SQL }),
|
||||
client.command({ query: SPANS_TABLE_BASE_SQL }),
|
||||
]);
|
||||
|
||||
await client.command({ query: CLICKMAP_EVENTS_ADD_DEAD_COLUMN_SQL });
|
||||
|
||||
// Alter events table (must come before views that reference new columns)
|
||||
await client.command({ query: EVENTS_ADD_REPLAY_COLUMNS_SQL });
|
||||
await client.command({ query: EVENTS_ADD_SPAN_COLUMNS_SQL });
|
||||
|
||||
// Clickmap materialized view depends on the events table existing; create after the ALTER above
|
||||
// so the view sees the replay columns. IF NOT EXISTS makes this idempotent across reboots.
|
||||
@ -59,6 +61,7 @@ export async function runClickhouseMigrations() {
|
||||
client.command({ query: NOTIFICATION_PREFERENCES_VIEW_SQL }),
|
||||
client.command({ query: REFRESH_TOKENS_VIEW_SQL }),
|
||||
client.command({ query: CONNECTED_ACCOUNTS_VIEW_SQL }),
|
||||
client.command({ query: SPANS_VIEW_SQL }),
|
||||
]);
|
||||
|
||||
// Data migrations (mutations)
|
||||
@ -73,6 +76,7 @@ export async function runClickhouseMigrations() {
|
||||
"events", "users", "contact_channels", "teams", "team_member_profiles",
|
||||
"team_permissions", "team_invitations", "email_outboxes",
|
||||
"project_permissions", "notification_preferences", "refresh_tokens", "connected_accounts",
|
||||
"spans",
|
||||
];
|
||||
await Promise.all(tables.map(table =>
|
||||
client.command({
|
||||
@ -248,6 +252,94 @@ WHERE event_type = '$token-refresh'
|
||||
AND data.refresh_token_id::Nullable(String) IS NOT NULL;
|
||||
`;
|
||||
|
||||
// Span-graph column on events (additive, backwards compatible — default.events is
|
||||
// SELECT * so it surfaces automatically). parent_span_ids holds the deduped, root-first
|
||||
// list of ancestor span ids as PREFIXED ids (e.g. [rti-<rt>, sri-<replay>, srsi-<segment>]),
|
||||
// NOT raw uuids. The per-tab id is already carried in the legacy `session_replay_segment_id`
|
||||
// column, so no separate tab-id column is added.
|
||||
const EVENTS_ADD_SPAN_COLUMNS_SQL = `
|
||||
ALTER TABLE analytics_internal.events
|
||||
ADD COLUMN IF NOT EXISTS parent_span_ids Array(String) DEFAULT [];
|
||||
`;
|
||||
|
||||
// Spans: telemetry siblings of events, written DIRECTLY to ClickHouse (never through
|
||||
// ext-db-sync). Interval facts whose end advances over time, so ReplacingMergeTree(version)
|
||||
// keeps the highest-version row per id (writers re-insert the full row with a bumped
|
||||
// version + latest span_ended_at). `default.spans` reads FINAL, which collapses versions
|
||||
// correctly even in the rare case a long-lived span straddles two monthly partitions.
|
||||
// Scalar identity columns stay raw (String/UUID-as-String); prefixes live only on `id`
|
||||
// and `parent_span_ids`. Session-replay/segment spans are written from the replay batch route;
|
||||
// $refresh-token spans are derived in the view from the synced refresh_tokens dimension.
|
||||
const SPANS_TABLE_BASE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS analytics_internal.spans (
|
||||
id String,
|
||||
span_type LowCardinality(String),
|
||||
span_started_at DateTime64(3, 'UTC'),
|
||||
span_ended_at Nullable(DateTime64(3, 'UTC')),
|
||||
parent_span_ids Array(String) DEFAULT [],
|
||||
data String DEFAULT '{}',
|
||||
project_id String,
|
||||
branch_id String,
|
||||
user_id Nullable(String),
|
||||
team_id Nullable(String),
|
||||
refresh_token_id Nullable(String),
|
||||
session_replay_id Nullable(String),
|
||||
session_replay_segment_id Nullable(String),
|
||||
created_at DateTime64(3, 'UTC') DEFAULT now64(3),
|
||||
version UInt64
|
||||
)
|
||||
ENGINE ReplacingMergeTree(version)
|
||||
PARTITION BY toYYYYMM(created_at)
|
||||
ORDER BY (project_id, branch_id, id);
|
||||
`;
|
||||
|
||||
// Customer-facing spans surface. UNION ALL of: (1) the physical spans table
|
||||
// ($session-replay + $session-replay-segment + any future custom spans), and (2) the
|
||||
// $refresh-token spans derived from the already-synced refresh_tokens dimension (no
|
||||
// separate write). Every branch casts to one identical column set. SECURITY DEFINER +
|
||||
// the default.spans row policy give the same per-project isolation as default.events.
|
||||
const SPANS_VIEW_SQL = `
|
||||
CREATE OR REPLACE VIEW default.spans
|
||||
SQL SECURITY DEFINER
|
||||
AS
|
||||
SELECT
|
||||
id,
|
||||
span_type,
|
||||
span_started_at,
|
||||
span_ended_at,
|
||||
parent_span_ids,
|
||||
data,
|
||||
project_id,
|
||||
branch_id,
|
||||
user_id,
|
||||
team_id,
|
||||
refresh_token_id,
|
||||
session_replay_id,
|
||||
session_replay_segment_id,
|
||||
created_at
|
||||
FROM analytics_internal.spans FINAL
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
concat('rti-', toString(id)) AS id,
|
||||
CAST('$refresh-token', 'LowCardinality(String)') AS span_type,
|
||||
created_at AS span_started_at,
|
||||
expires_at AS span_ended_at,
|
||||
CAST([], 'Array(String)') AS parent_span_ids,
|
||||
CAST('{}', 'String') AS data,
|
||||
project_id,
|
||||
branch_id,
|
||||
CAST(toString(user_id), 'Nullable(String)') AS user_id,
|
||||
CAST(NULL, 'Nullable(String)') AS team_id,
|
||||
CAST(toString(id), 'Nullable(String)') AS refresh_token_id,
|
||||
CAST(NULL, 'Nullable(String)') AS session_replay_id,
|
||||
CAST(NULL, 'Nullable(String)') AS session_replay_segment_id,
|
||||
sync_created_at AS created_at
|
||||
FROM analytics_internal.refresh_tokens FINAL
|
||||
WHERE sync_is_deleted = 0;
|
||||
`;
|
||||
|
||||
const CONTACT_CHANNELS_TABLE_BASE_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS analytics_internal.contact_channels (
|
||||
project_id String,
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import { getClickhouseAdminClient } from "@/lib/clickhouse";
|
||||
import { arePlanLimitsEnforced, getBillingTeamId } from "@/lib/plan-entitlements";
|
||||
import { findRecentSessionReplay } from "@/lib/session-replays";
|
||||
import { buildEventSpanFields } from "@/lib/spans";
|
||||
import { getHexclaveServerApp } from "@/hexclave";
|
||||
import { getPrismaClientForTenancy } from "@/prisma-client";
|
||||
import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler";
|
||||
@ -134,6 +135,15 @@ export const POST = createSmartRouteHandler({
|
||||
|
||||
const clickhouseClient = getClickhouseAdminClient();
|
||||
|
||||
// Point each event at its ancestor spans (root-first: refresh-token, replay,
|
||||
// then the per-tab span when a replay exists). The per-tab id itself already
|
||||
// lives in the session_replay_segment_id column — see lib/spans.tsx.
|
||||
const eventSpanFields = buildEventSpanFields({
|
||||
sessionReplayId: recentSession?.id ?? null,
|
||||
sessionReplaySegmentId: body.session_replay_segment_id,
|
||||
refreshTokenId,
|
||||
});
|
||||
|
||||
const rows = body.events.map((event) => ({
|
||||
event_type: event.event_type,
|
||||
event_at: new Date(event.event_at_ms),
|
||||
@ -145,6 +155,7 @@ export const POST = createSmartRouteHandler({
|
||||
refresh_token_id: refreshTokenId,
|
||||
session_replay_id: recentSession?.id ?? null,
|
||||
session_replay_segment_id: body.session_replay_segment_id,
|
||||
...eventSpanFields,
|
||||
}));
|
||||
|
||||
await clickhouseClient.insert({
|
||||
|
||||
@ -2,13 +2,15 @@ import { getPrismaClientForTenancy } from "@/prisma-client";
|
||||
import { uploadBytes } from "@/s3";
|
||||
import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler";
|
||||
import { Prisma } from "@/generated/prisma/client";
|
||||
import { getClickhouseAdminClient } from "@/lib/clickhouse";
|
||||
import { arePlanLimitsEnforced, getBillingTeamId } from "@/lib/plan-entitlements";
|
||||
import { findRecentSessionReplay } from "@/lib/session-replays";
|
||||
import { insertSessionReplaySpans } from "@/lib/spans";
|
||||
import { getHexclaveServerApp } from "@/hexclave";
|
||||
import { KnownErrors } from "@hexclave/shared";
|
||||
import { ITEM_IDS } from "@hexclave/shared/dist/plans";
|
||||
import { adaptSchema, clientOrHigherAuthTypeSchema, yupArray, yupMixed, yupNumber, yupObject, yupString } from "@hexclave/shared/dist/schema-fields";
|
||||
import { StatusError } from "@hexclave/shared/dist/utils/errors";
|
||||
import { captureError, StatusError } from "@hexclave/shared/dist/utils/errors";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { promisify } from "node:util";
|
||||
import { gzip as gzipCb, gunzipSync } from "node:zlib";
|
||||
@ -248,6 +250,35 @@ export const POST = createSmartRouteHandler({
|
||||
data: { shouldUpdateSequenceId: true },
|
||||
});
|
||||
|
||||
// Additionally mirror this replay into the spans telemetry surface, written
|
||||
// directly to ClickHouse (the same way events are). Purely additive to the
|
||||
// Postgres/S3 replay storage above — a $session-replay span for the whole
|
||||
// replay and a $session-replay-segment span for this recording segment (per
|
||||
// tab), re-written on every non-deduped batch so their end advances. Never fail
|
||||
// the upload if ClickHouse is unavailable.
|
||||
try {
|
||||
const segmentBounds = await prisma.sessionReplayChunk.aggregate({
|
||||
where: { tenancyId, sessionReplayId: replayId, sessionReplaySegmentId },
|
||||
_min: { firstEventAt: true },
|
||||
_max: { lastEventAt: true },
|
||||
});
|
||||
await insertSessionReplaySpans(getClickhouseAdminClient(), {
|
||||
projectId,
|
||||
branchId,
|
||||
replayId,
|
||||
sessionReplaySegmentId,
|
||||
projectUserId,
|
||||
refreshTokenId,
|
||||
replayStartedAt: new Date(newStartedAtMs),
|
||||
replayLastEventAt: new Date(newLastEventAtMs),
|
||||
segmentStartedAt: segmentBounds._min.firstEventAt ?? new Date(firstMs),
|
||||
segmentLastEventAt: segmentBounds._max.lastEventAt ?? new Date(lastMs),
|
||||
version: Date.now(),
|
||||
});
|
||||
} catch (error) {
|
||||
captureError("session-replay-spans-insert", error);
|
||||
}
|
||||
|
||||
return {
|
||||
statusCode: 200,
|
||||
bodyType: "json",
|
||||
|
||||
@ -14,6 +14,7 @@ import { generateUuid } from "@hexclave/shared/dist/utils/uuids";
|
||||
import * as yup from "yup";
|
||||
import { getClickhouseAdminClient } from "./clickhouse";
|
||||
import { getEndUserInfo } from "./end-users";
|
||||
import { buildEventSpanFields } from "./spans";
|
||||
import { DEFAULT_BRANCH_ID } from "./tenancies";
|
||||
|
||||
export const endUserIpInfoSchema = yupObject({
|
||||
@ -390,6 +391,11 @@ export async function logEvent<T extends EventType[]>(
|
||||
refresh_token_id: resolvedRefreshTokenId ?? null,
|
||||
session_replay_id: options.sessionReplayId ?? null,
|
||||
session_replay_segment_id: options.sessionReplaySegmentId ?? null,
|
||||
...buildEventSpanFields({
|
||||
sessionReplayId: options.sessionReplayId ?? null,
|
||||
sessionReplaySegmentId: options.sessionReplaySegmentId ?? null,
|
||||
refreshTokenId: resolvedRefreshTokenId ?? null,
|
||||
}),
|
||||
}],
|
||||
format: "JSONEachRow",
|
||||
clickhouse_settings: {
|
||||
|
||||
104
apps/backend/src/lib/spans.test.ts
Normal file
104
apps/backend/src/lib/spans.test.ts
Normal file
@ -0,0 +1,104 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { SPAN_ID_PREFIXES, buildEventSpanFields, insertSessionReplaySpans, toSpanId } from "./spans";
|
||||
|
||||
describe("toSpanId", () => {
|
||||
it("prefixes raw ids without touching the raw value", () => {
|
||||
expect(toSpanId(SPAN_ID_PREFIXES.sessionReplay, "abc")).toBe("sri-abc");
|
||||
expect(toSpanId(SPAN_ID_PREFIXES.sessionReplaySegment, "seg")).toBe("srsi-seg");
|
||||
expect(toSpanId(SPAN_ID_PREFIXES.refreshToken, "rt1")).toBe("rti-rt1");
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildEventSpanFields", () => {
|
||||
it("lists all known ancestors root-first: refresh-token, replay, segment", () => {
|
||||
expect(buildEventSpanFields({ sessionReplayId: "s1", sessionReplaySegmentId: "seg1", refreshTokenId: "r1" })).toEqual({
|
||||
parent_span_ids: ["rti-r1", "sri-s1", "srsi-seg1"],
|
||||
});
|
||||
});
|
||||
|
||||
it("omits the segment parent when there is no replay (segment spans only exist under a replay)", () => {
|
||||
expect(buildEventSpanFields({ sessionReplayId: null, sessionReplaySegmentId: "seg1", refreshTokenId: "r1" })).toEqual({
|
||||
parent_span_ids: ["rti-r1"],
|
||||
});
|
||||
});
|
||||
|
||||
it("includes refresh-token and replay when there is a replay but no segment", () => {
|
||||
expect(buildEventSpanFields({ sessionReplayId: "s1", refreshTokenId: "r1" })).toEqual({
|
||||
parent_span_ids: ["rti-r1", "sri-s1"],
|
||||
});
|
||||
});
|
||||
|
||||
it("emits an empty parent list when nothing is known", () => {
|
||||
expect(buildEventSpanFields({})).toEqual({ parent_span_ids: [] });
|
||||
});
|
||||
});
|
||||
|
||||
describe("insertSessionReplaySpans", () => {
|
||||
it("emits a replay span and a segment span in one insert with the right ids and parents", async () => {
|
||||
const captured: any[] = [];
|
||||
const client = {
|
||||
insert: vi.fn(async (args: any) => {
|
||||
captured.push(args);
|
||||
}),
|
||||
} as any;
|
||||
|
||||
const replayStartedAt = new Date("2026-01-01T00:00:00.000Z");
|
||||
const replayLastEventAt = new Date("2026-01-01T00:05:00.000Z");
|
||||
const segmentStartedAt = new Date("2026-01-01T00:01:00.000Z");
|
||||
const segmentLastEventAt = new Date("2026-01-01T00:04:00.000Z");
|
||||
|
||||
await insertSessionReplaySpans(client, {
|
||||
projectId: "p1",
|
||||
branchId: "b1",
|
||||
replayId: "replay1",
|
||||
sessionReplaySegmentId: "seg1",
|
||||
projectUserId: "user1",
|
||||
refreshTokenId: "rt1",
|
||||
replayStartedAt,
|
||||
replayLastEventAt,
|
||||
segmentStartedAt,
|
||||
segmentLastEventAt,
|
||||
version: 123,
|
||||
});
|
||||
|
||||
expect(client.insert).toHaveBeenCalledTimes(1);
|
||||
expect(captured[0].table).toBe("analytics_internal.spans");
|
||||
const rows = captured[0].values;
|
||||
expect(rows).toHaveLength(2);
|
||||
|
||||
const [replaySpan, segmentSpan] = rows;
|
||||
|
||||
expect(replaySpan).toMatchObject({
|
||||
id: "sri-replay1",
|
||||
span_type: "$session-replay",
|
||||
parent_span_ids: ["rti-rt1"],
|
||||
project_id: "p1",
|
||||
branch_id: "b1",
|
||||
user_id: "user1",
|
||||
refresh_token_id: "rt1",
|
||||
session_replay_id: "replay1",
|
||||
session_replay_segment_id: null,
|
||||
version: 123,
|
||||
});
|
||||
expect(replaySpan.span_started_at).toBe(replayStartedAt);
|
||||
expect(replaySpan.span_ended_at).toBe(replayLastEventAt);
|
||||
|
||||
expect(segmentSpan).toMatchObject({
|
||||
id: "srsi-seg1",
|
||||
span_type: "$session-replay-segment",
|
||||
parent_span_ids: ["rti-rt1", "sri-replay1"],
|
||||
session_replay_id: "replay1",
|
||||
session_replay_segment_id: "seg1",
|
||||
version: 123,
|
||||
});
|
||||
expect(segmentSpan.span_started_at).toBe(segmentStartedAt);
|
||||
expect(segmentSpan.span_ended_at).toBe(segmentLastEventAt);
|
||||
});
|
||||
|
||||
it("does not call insert when given an empty row list (insertSpans guard)", async () => {
|
||||
const client = { insert: vi.fn() } as any;
|
||||
const { insertSpans } = await import("./spans");
|
||||
await insertSpans(client, []);
|
||||
expect(client.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
160
apps/backend/src/lib/spans.tsx
Normal file
160
apps/backend/src/lib/spans.tsx
Normal file
@ -0,0 +1,160 @@
|
||||
import type { ClickHouseClient } from "./clickhouse";
|
||||
|
||||
/**
|
||||
* Spans are telemetry facts (time intervals) — the sibling of events. They are
|
||||
* written DIRECTLY to ClickHouse (`analytics_internal.spans`), the same way
|
||||
* events are, and never go through the ext-db-sync (that is a dimension
|
||||
* replicator, not a telemetry pipe). See the plan/notes in
|
||||
* `apps/backend/scripts/clickhouse-migrations.ts` for the table + `default.spans` view.
|
||||
*/
|
||||
|
||||
// Typed-id prefixes. Applied ONLY to a span's own `id` and to the values inside
|
||||
// `parent_span_ids` — so a heterogeneous parent array is self-describing. They are
|
||||
// NEVER applied to scalar identity columns (project_id, user_id, session_replay_id, …),
|
||||
// which stay raw so existing customer SQL keeps working.
|
||||
export const SPAN_ID_PREFIXES = {
|
||||
sessionReplay: "sri-",
|
||||
// The per-tab id. The SDK mints exactly one `session_replay_segment_id` per
|
||||
// browser tab, so despite the "segment" name it IS the per-tab id — there is
|
||||
// only this one level (there is no separate "tab"). Named "segment" to stay
|
||||
// consistent with the pre-existing `session_replay_segment_id` column + SDK field.
|
||||
sessionReplaySegment: "srsi-",
|
||||
refreshToken: "rti-",
|
||||
user: "ui-",
|
||||
team: "ti-",
|
||||
project: "pi-",
|
||||
branch: "bi-",
|
||||
} as const;
|
||||
|
||||
export const SPAN_TYPES = {
|
||||
sessionReplay: "$session-replay",
|
||||
sessionReplaySegment: "$session-replay-segment",
|
||||
refreshToken: "$refresh-token",
|
||||
} as const;
|
||||
|
||||
export function toSpanId(prefix: string, rawId: string): string {
|
||||
return `${prefix}${rawId}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* The `parent_span_ids` an event insert stamps on each row — the full, deduped
|
||||
* list of ancestor spans the server can name, root-first: refresh-token, then the
|
||||
* session-replay span (via the server-resolved `session_replay_id`), then the
|
||||
* per-tab `$session-replay-segment` span (only when a replay exists, since that span
|
||||
* is written under a replay). Parent links are logical ancestry, not FK-guaranteed
|
||||
* rows; the list always contains a higher-level ancestor even if the segment span was
|
||||
* never written (recording off). The per-tab id is carried in `session_replay_segment_id`.
|
||||
*/
|
||||
export function buildEventSpanFields(opts: {
|
||||
sessionReplayId?: string | null,
|
||||
sessionReplaySegmentId?: string | null,
|
||||
refreshTokenId?: string | null,
|
||||
}): { parent_span_ids: string[] } {
|
||||
const parentSpanIds: string[] = [];
|
||||
if (opts.refreshTokenId) {
|
||||
parentSpanIds.push(toSpanId(SPAN_ID_PREFIXES.refreshToken, opts.refreshTokenId));
|
||||
}
|
||||
if (opts.sessionReplayId) {
|
||||
parentSpanIds.push(toSpanId(SPAN_ID_PREFIXES.sessionReplay, opts.sessionReplayId));
|
||||
}
|
||||
if (opts.sessionReplayId && opts.sessionReplaySegmentId) {
|
||||
parentSpanIds.push(toSpanId(SPAN_ID_PREFIXES.sessionReplaySegment, opts.sessionReplaySegmentId));
|
||||
}
|
||||
return { parent_span_ids: parentSpanIds };
|
||||
}
|
||||
|
||||
/**
|
||||
* One row of `analytics_internal.spans`. `created_at` is omitted (the table
|
||||
* defaults it to now64(3) = ingested-at). `version` is a server-monotonic value
|
||||
* per span id (write-time epoch ms): the ReplacingMergeTree keeps the highest
|
||||
* version, so re-writing a span with a later `span_ended_at` advances its end.
|
||||
*/
|
||||
export type SpanInsertRow = {
|
||||
id: string,
|
||||
span_type: string,
|
||||
span_started_at: Date,
|
||||
span_ended_at: Date | null,
|
||||
parent_span_ids: string[],
|
||||
data: string,
|
||||
project_id: string,
|
||||
branch_id: string,
|
||||
user_id: string | null,
|
||||
team_id: string | null,
|
||||
refresh_token_id: string | null,
|
||||
session_replay_id: string | null,
|
||||
session_replay_segment_id: string | null,
|
||||
version: number,
|
||||
};
|
||||
|
||||
export async function insertSpans(client: ClickHouseClient, rows: SpanInsertRow[]): Promise<void> {
|
||||
if (rows.length === 0) return;
|
||||
await client.insert({
|
||||
table: "analytics_internal.spans",
|
||||
values: rows,
|
||||
format: "JSONEachRow",
|
||||
clickhouse_settings: {
|
||||
date_time_input_format: "best_effort",
|
||||
async_insert: 1,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Emits the two spans that describe a session replay from the replay batch route:
|
||||
* the replay-level `$session-replay` span and the per-tab `$session-replay-segment`
|
||||
* span. Re-written on every batch (with a fresh `version` and the latest bounds)
|
||||
* so their `span_ended_at` advances as recording continues. The segment span uses
|
||||
* the RECORDING's `sessionReplaySegmentId` — the per-tab id — as its identity.
|
||||
*/
|
||||
export async function insertSessionReplaySpans(
|
||||
client: ClickHouseClient,
|
||||
opts: {
|
||||
projectId: string,
|
||||
branchId: string,
|
||||
replayId: string,
|
||||
sessionReplaySegmentId: string,
|
||||
projectUserId: string,
|
||||
refreshTokenId: string,
|
||||
replayStartedAt: Date,
|
||||
replayLastEventAt: Date,
|
||||
segmentStartedAt: Date,
|
||||
segmentLastEventAt: Date,
|
||||
version: number,
|
||||
},
|
||||
): Promise<void> {
|
||||
const base = {
|
||||
data: "{}",
|
||||
project_id: opts.projectId,
|
||||
branch_id: opts.branchId,
|
||||
user_id: opts.projectUserId,
|
||||
team_id: null,
|
||||
refresh_token_id: opts.refreshTokenId,
|
||||
session_replay_id: opts.replayId,
|
||||
version: opts.version,
|
||||
} as const;
|
||||
|
||||
const replaySpan: SpanInsertRow = {
|
||||
...base,
|
||||
id: toSpanId(SPAN_ID_PREFIXES.sessionReplay, opts.replayId),
|
||||
span_type: SPAN_TYPES.sessionReplay,
|
||||
span_started_at: opts.replayStartedAt,
|
||||
span_ended_at: opts.replayLastEventAt,
|
||||
parent_span_ids: [toSpanId(SPAN_ID_PREFIXES.refreshToken, opts.refreshTokenId)],
|
||||
session_replay_segment_id: null,
|
||||
};
|
||||
|
||||
const segmentSpan: SpanInsertRow = {
|
||||
...base,
|
||||
id: toSpanId(SPAN_ID_PREFIXES.sessionReplaySegment, opts.sessionReplaySegmentId),
|
||||
span_type: SPAN_TYPES.sessionReplaySegment,
|
||||
span_started_at: opts.segmentStartedAt,
|
||||
span_ended_at: opts.segmentLastEventAt,
|
||||
parent_span_ids: [
|
||||
toSpanId(SPAN_ID_PREFIXES.refreshToken, opts.refreshTokenId),
|
||||
toSpanId(SPAN_ID_PREFIXES.sessionReplay, opts.replayId),
|
||||
],
|
||||
session_replay_segment_id: opts.sessionReplaySegmentId,
|
||||
};
|
||||
|
||||
await insertSpans(client, [replaySpan, segmentSpan]);
|
||||
}
|
||||
@ -40,6 +40,15 @@ const AVAILABLE_TABLES = new Map<TableId, TableConfig>([
|
||||
defaultOrderDir: "desc",
|
||||
},
|
||||
],
|
||||
[
|
||||
"spans",
|
||||
{
|
||||
displayName: "Spans",
|
||||
baseQuery: "SELECT * FROM default.spans",
|
||||
defaultOrderBy: "span_started_at",
|
||||
defaultOrderDir: "desc",
|
||||
},
|
||||
],
|
||||
[
|
||||
"users",
|
||||
{
|
||||
|
||||
@ -592,6 +592,52 @@ it("inserted events are queryable via analytics query endpoint", async ({ expect
|
||||
`);
|
||||
});
|
||||
|
||||
it("sets parent_span_ids on inserted events", async ({ expect }) => {
|
||||
await Project.createAndSwitch({ config: { magic_link_enabled: true } });
|
||||
await Project.updateConfig({ apps: { installed: { analytics: { enabled: true } } } });
|
||||
await Auth.Otp.signIn();
|
||||
|
||||
const sessionReplaySegmentId = randomUUID();
|
||||
const now = Date.now();
|
||||
|
||||
await uploadEventBatch({
|
||||
sessionReplaySegmentId,
|
||||
batchId: randomUUID(),
|
||||
sentAtMs: now,
|
||||
events: [
|
||||
{
|
||||
event_type: "$page-view",
|
||||
event_at_ms: now - 100,
|
||||
data: { url: "https://example.com/spans", path: "/spans" },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
let queryRes;
|
||||
for (let attempt = 0; attempt < 15; attempt++) {
|
||||
await wait(500);
|
||||
queryRes = await niceBackendFetch("/api/v1/internal/analytics/query", {
|
||||
method: "POST",
|
||||
accessType: "admin",
|
||||
body: {
|
||||
query: "SELECT parent_span_ids, session_replay_id FROM events WHERE session_replay_segment_id = {segId:String}",
|
||||
params: { segId: sessionReplaySegmentId },
|
||||
},
|
||||
});
|
||||
if (queryRes.status === 200 && queryRes.body?.result?.length === 1) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
expect(queryRes?.status).toBe(200);
|
||||
const row = (queryRes?.body as any).result[0];
|
||||
// No active session replay for this user, so the only ancestor span is the
|
||||
// refresh-token span. The per-tab id itself lives in session_replay_segment_id.
|
||||
expect(row.session_replay_id).toBeNull();
|
||||
expect(row.parent_span_ids).toHaveLength(1);
|
||||
expect(row.parent_span_ids[0]).toMatch(/^rti-/);
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Analytics event limit enforcement tests
|
||||
// ============================================================================
|
||||
|
||||
@ -595,6 +595,7 @@ it("has limited grants", async ({ expect }) => {
|
||||
{ "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.notification_preferences TO limited_user" },
|
||||
{ "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.project_permissions TO limited_user" },
|
||||
{ "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.refresh_tokens TO limited_user" },
|
||||
{ "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.spans TO limited_user" },
|
||||
{ "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.team_invitations TO limited_user" },
|
||||
{ "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.team_member_profiles TO limited_user" },
|
||||
{ "GRANTS WITH IMPLICIT FINAL FORMAT JSONEachRow": "GRANT SHOW TABLES, SHOW COLUMNS, SELECT ON default.team_permissions TO limited_user" },
|
||||
@ -665,6 +666,10 @@ it("can see only some tables", async ({ expect }) => {
|
||||
"database": "default",
|
||||
"name": "refresh_tokens",
|
||||
},
|
||||
{
|
||||
"database": "default",
|
||||
"name": "spans",
|
||||
},
|
||||
{
|
||||
"database": "default",
|
||||
"name": "team_invitations",
|
||||
@ -709,6 +714,7 @@ it("SHOW TABLES should have the correct tables", async ({ expect }) => {
|
||||
{ "name": "notification_preferences" },
|
||||
{ "name": "project_permissions" },
|
||||
{ "name": "refresh_tokens" },
|
||||
{ "name": "spans" },
|
||||
{ "name": "team_invitations" },
|
||||
{ "name": "team_member_profiles" },
|
||||
{ "name": "team_permissions" },
|
||||
@ -1201,6 +1207,7 @@ it("shows grants", async ({ expect }) => {
|
||||
{ "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.notification_preferences TO limited_user" },
|
||||
{ "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.project_permissions TO limited_user" },
|
||||
{ "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.refresh_tokens TO limited_user" },
|
||||
{ "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.spans TO limited_user" },
|
||||
{ "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.team_invitations TO limited_user" },
|
||||
{ "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.team_member_profiles TO limited_user" },
|
||||
{ "GRANTS FORMAT JSONEachRow": "GRANT SELECT ON default.team_permissions TO limited_user" },
|
||||
|
||||
@ -163,6 +163,66 @@ it("stores session replay batch metadata and dedupes by (session_replay_id, batc
|
||||
expect(second.body?.session_replay_id).toBe(recordingId);
|
||||
});
|
||||
|
||||
it("emits $session-replay and $session-replay-segment spans queryable via the spans surface", async ({ expect }) => {
|
||||
await Project.createAndSwitch({ config: { magic_link_enabled: true } });
|
||||
await Project.updateConfig({ apps: { installed: { analytics: { enabled: true } } } });
|
||||
await Auth.Otp.signIn();
|
||||
|
||||
const now = Date.now();
|
||||
const sessionReplaySegmentId = randomUUID();
|
||||
|
||||
const uploadRes = await uploadBatch({
|
||||
browserSessionId: randomUUID(),
|
||||
batchId: randomUUID(),
|
||||
sessionReplaySegmentId,
|
||||
startedAtMs: now,
|
||||
sentAtMs: now + 500,
|
||||
events: [
|
||||
{ timestamp: now + 100, type: 2 },
|
||||
{ timestamp: now + 200, type: 3 },
|
||||
],
|
||||
});
|
||||
expect(uploadRes.status).toBe(200);
|
||||
const replayId = uploadRes.body?.session_replay_id as string;
|
||||
|
||||
let queryRes;
|
||||
for (let attempt = 0; attempt < 15; attempt++) {
|
||||
await wait(500);
|
||||
queryRes = await niceBackendFetch("/api/v1/internal/analytics/query", {
|
||||
method: "POST",
|
||||
accessType: "admin",
|
||||
body: {
|
||||
query: "SELECT span_type, id, parent_span_ids, session_replay_id, session_replay_segment_id FROM spans WHERE session_replay_id = {replayId:String} ORDER BY span_type",
|
||||
params: { replayId },
|
||||
},
|
||||
});
|
||||
if (queryRes.status === 200 && queryRes.body?.result?.length === 2) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
expect(queryRes?.status).toBe(200);
|
||||
const rows = (queryRes?.body as any).result;
|
||||
expect(rows).toHaveLength(2);
|
||||
|
||||
const replaySpan = rows.find((r: any) => r.span_type === "$session-replay");
|
||||
const segmentSpan = rows.find((r: any) => r.span_type === "$session-replay-segment");
|
||||
|
||||
// Replay-level span: id is prefixed, parented to the refresh-token span, no segment.
|
||||
expect(replaySpan.id).toBe(`sri-${replayId}`);
|
||||
expect(replaySpan.parent_span_ids).toHaveLength(1);
|
||||
expect(replaySpan.parent_span_ids[0]).toMatch(/^rti-/);
|
||||
expect(replaySpan.session_replay_segment_id).toBeNull();
|
||||
|
||||
// Segment span (per tab): ancestor list is root-first [refresh-token, replay];
|
||||
// its identity is the recording's per-tab session_replay_segment_id.
|
||||
expect(segmentSpan.id).toBe(`srsi-${sessionReplaySegmentId}`);
|
||||
expect(segmentSpan.parent_span_ids).toHaveLength(2);
|
||||
expect(segmentSpan.parent_span_ids[0]).toMatch(/^rti-/);
|
||||
expect(segmentSpan.parent_span_ids[1]).toBe(`sri-${replayId}`);
|
||||
expect(segmentSpan.session_replay_segment_id).toBe(sessionReplaySegmentId);
|
||||
});
|
||||
|
||||
it("accepts a gzipped binary body (compressed large-payload encoding)", async ({ expect }) => {
|
||||
await Project.createAndSwitch({ config: { magic_link_enabled: true } });
|
||||
await Project.updateConfig({ apps: { installed: { analytics: { enabled: true } } } });
|
||||
|
||||
@ -720,12 +720,18 @@ export class _HexclaveClientAppImplIncomplete<HasTokenStore extends boolean, Pro
|
||||
const analyticsEnabled = this._analyticsOptions?.enabled !== false;
|
||||
|
||||
const sessionReplayOptions = getSessionReplayOptions(this._analyticsOptions);
|
||||
// One per-tab id shared by the SessionRecorder and EventTracker, so replay
|
||||
// chunks and analytics events from the same tab report the same
|
||||
// session_replay_segment_id (they used to each mint their own, which made
|
||||
// event rows impossible to correlate to a replay tab).
|
||||
const sessionReplaySegmentId = generateUuid();
|
||||
if (analyticsEnabled && isBrowserLike() && this._hasPersistentTokenStore() && sessionReplayOptions.enabled) {
|
||||
this._sessionRecorder = new SessionRecorder({
|
||||
projectId: this.projectId,
|
||||
sendBatch: async (body, opts) => {
|
||||
return await this._interface.sendSessionReplayBatch(body, await getAnalyticsSession(), opts);
|
||||
},
|
||||
sessionReplaySegmentId,
|
||||
}, sessionReplayOptions);
|
||||
this._sessionRecorder.start();
|
||||
}
|
||||
@ -736,6 +742,7 @@ export class _HexclaveClientAppImplIncomplete<HasTokenStore extends boolean, Pro
|
||||
sendBatch: async (body, opts) => {
|
||||
return await this._interface.sendAnalyticsEventBatch(body, await getAnalyticsSession(), opts);
|
||||
},
|
||||
sessionReplaySegmentId,
|
||||
});
|
||||
this._eventTracker.start();
|
||||
}
|
||||
|
||||
@ -101,6 +101,10 @@ function isInsideHexclaveUiNode(node: Node | null): boolean {
|
||||
export type EventTrackerDeps = {
|
||||
projectId: string,
|
||||
sendBatch: (body: string, options: { keepalive: boolean }) => Promise<Result<Response, Error>>,
|
||||
// Per-tab id shared with the SessionRecorder so analytics events and replay
|
||||
// chunks from the same tab carry the same session_replay_segment_id. Falls
|
||||
// back to a fresh uuid when constructed standalone (e.g. in tests).
|
||||
sessionReplaySegmentId?: string,
|
||||
};
|
||||
|
||||
type TrackedEvent = {
|
||||
@ -136,7 +140,7 @@ export class EventTracker {
|
||||
|
||||
constructor(deps: EventTrackerDeps) {
|
||||
this._deps = deps;
|
||||
this._sessionReplaySegmentId = generateUuid();
|
||||
this._sessionReplaySegmentId = deps.sessionReplaySegmentId ?? generateUuid();
|
||||
}
|
||||
|
||||
start() {
|
||||
|
||||
@ -173,6 +173,10 @@ export function getOrRotateSession(options: { key: string, legacyKey?: string, n
|
||||
export type SessionRecorderDeps = {
|
||||
projectId: string,
|
||||
sendBatch: (body: string, options: { keepalive: boolean }) => Promise<Result<Response, Error>>,
|
||||
// Per-tab id shared with the EventTracker so replay chunks and analytics
|
||||
// events from the same tab carry the same session_replay_segment_id. Falls
|
||||
// back to a fresh uuid when constructed standalone (e.g. in tests).
|
||||
sessionReplaySegmentId?: string,
|
||||
};
|
||||
|
||||
export function isAnalyticsNotEnabledError(error: unknown): boolean {
|
||||
@ -220,7 +224,7 @@ export class SessionRecorder {
|
||||
constructor(deps: SessionRecorderDeps, replayOptions: AnalyticsReplayOptions) {
|
||||
this._deps = deps;
|
||||
this._replayOptions = replayOptions;
|
||||
this._sessionReplaySegmentId = generateUuid();
|
||||
this._sessionReplaySegmentId = deps.sessionReplaySegmentId ?? generateUuid();
|
||||
this._storageKey = makeStorageKey(deps.projectId);
|
||||
this._legacyStorageKey = makeLegacyStorageKey(deps.projectId);
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user