From 723d63210a7d708e3e78a5f7a06ec40c656038c1 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Fri, 21 Aug 2026 08:32:10 -0400 Subject: [PATCH 1/4] eng-2129-slots-and-slotDefinitions-to-ref_content --- packages/database/src/crossAppContracts.ts | 2 + .../__tests__/dbToCrossAppConverters.test.ts | 17 +++++---- .../database/src/lib/crossAppConverters.ts | 5 +++ .../src/lib/dbToCrossAppConverters.ts | 38 +++++++++++++++---- 4 files changed, 48 insertions(+), 14 deletions(-) diff --git a/packages/database/src/crossAppContracts.ts b/packages/database/src/crossAppContracts.ts index e60090690..fe97fd6c2 100644 --- a/packages/database/src/crossAppContracts.ts +++ b/packages/database/src/crossAppContracts.ts @@ -25,6 +25,7 @@ export type CrossAppNodeSchema = CrossAppSchemaBase & { label: string; template?: string; templateTitle?: string; + slotDefinitions?: Record; }; // A relation type schema @@ -75,6 +76,7 @@ type InlineCrossAppTypedContent = InlineCrossAppContent & { // A node instance export type CrossAppNode = CrossAppBase & { nodeType: LocalId; + slots?: Record; content: { direct: InlineCrossAppContent; full?: InlineCrossAppTypedContent; diff --git a/packages/database/src/lib/__tests__/dbToCrossAppConverters.test.ts b/packages/database/src/lib/__tests__/dbToCrossAppConverters.test.ts index 86ee73ba7..f52141a7c 100644 --- a/packages/database/src/lib/__tests__/dbToCrossAppConverters.test.ts +++ b/packages/database/src/lib/__tests__/dbToCrossAppConverters.test.ts @@ -51,13 +51,16 @@ describe("dbNodeSchemaToCrossApp", () => { extra: "kept", }, }); - expect(dbNodeSchemaToCrossApp(schema, spaceMap, accountMap)).toEqual({ + expect( + dbNodeSchemaToCrossApp({ schema, spaceMap, accountMap, schemaMap: {} }), + ).toEqual({ rid: "orn:obsidian.schema:vault-a/concept-1", localId: "concept-1", createdAt: new Date("2026-06-14T11:00:00Z"), modifiedAt: new Date("2026-06-14T13:00:00Z"), label: "Some concept", metadata: { extra: "kept" }, + slotDefinitions: {}, template: "template body", templateTitle: "Template Title", authorId: "account-local-1", @@ -66,16 +69,16 @@ describe("dbNodeSchemaToCrossApp", () => { it("throws when the author is unknown", () => { const schema = baseConcept({ author_id: 999 }); - expect(() => dbNodeSchemaToCrossApp(schema, spaceMap, accountMap)).toThrow( - "Missing author", - ); + expect(() => + dbNodeSchemaToCrossApp({ schema, spaceMap, accountMap, schemaMap: {} }), + ).toThrow("Missing author"); }); it("throws when the space is unknown", () => { const schema = baseConcept({ space_id: 999 }); - expect(() => dbNodeSchemaToCrossApp(schema, spaceMap, accountMap)).toThrow( - "Missing space", - ); + expect(() => + dbNodeSchemaToCrossApp({ schema, spaceMap, accountMap, schemaMap: {} }), + ).toThrow("Missing space"); }); }); diff --git a/packages/database/src/lib/crossAppConverters.ts b/packages/database/src/lib/crossAppConverters.ts index 3199f299c..1ff856f7f 100644 --- a/packages/database/src/lib/crossAppConverters.ts +++ b/packages/database/src/lib/crossAppConverters.ts @@ -86,16 +86,20 @@ export const crossAppNodeToDbConcept = ( ]), created: node.createdAt?.toISOString(), last_modified: node.modifiedAt?.toISOString(), + local_reference_content: node.slots, }); }; export const crossAppNodeSchemaToDbConcept = ( node: CrossAppNodeSchema, ): LocalConceptDataInput => { + const slots = Object.keys(node.slotDefinitions ?? {}); const literalInfo = filterUndefined({ template: node.templateTitle, template_content: node.template, + roles: slots.length > 0 ? slots : undefined, }); + const referenceContent = slots.length ? node.slotDefinitions! : undefined; const spaceUri = node.rid ? ridToSpaceUriAndLocalId(node.rid).spaceUri : undefined; @@ -107,6 +111,7 @@ export const crossAppNodeSchemaToDbConcept = ( is_schema: true, literal_content: Object.keys(literalInfo).length > 0 ? literalInfo : undefined, + local_reference_content: referenceContent, created: node.createdAt?.toISOString(), last_modified: node.modifiedAt?.toISOString(), }); diff --git a/packages/database/src/lib/dbToCrossAppConverters.ts b/packages/database/src/lib/dbToCrossAppConverters.ts index 4fbfbede8..07839536f 100644 --- a/packages/database/src/lib/dbToCrossAppConverters.ts +++ b/packages/database/src/lib/dbToCrossAppConverters.ts @@ -15,6 +15,7 @@ const getConceptMap = async ( conceptIds: number[], spaceMap: Record, ): Promise> => { + if (conceptIds.length === 0) return {}; const request = await client .from("my_concepts") .select("id, space_id, source_local_id") @@ -79,12 +80,22 @@ const asSimpleLocalId = ( return rid; }; -export const dbNodeSchemaToCrossApp = ( - schema: Concept, - spaceMap: Record, - accountMap: Record, -): CrossAppNodeSchema => { - const { template, template_content, ...other } = +export const dbNodeSchemaToCrossApp = ({ + schema, + spaceMap, + accountMap, + schemaMap, +}: { + schema: Concept; + spaceMap: Record; + accountMap: Record; + schemaMap: Record; +}): CrossAppNodeSchema => { + const referenceContent = (schema.reference_content ?? {}) as Record< + string, + number + >; + const { template, template_content, roles, ...other } = schema.literal_content as Record; const authorId = accountMap[schema.author_id || 0]; if (authorId === undefined) throw new Error("Missing author"); @@ -95,6 +106,14 @@ export const dbNodeSchemaToCrossApp = ( schema.source_local_id!, "schema", ); + const slotDefinitions: Record = Object.fromEntries( + ((roles as string[] | undefined) ?? []) + .map((r) => [ + r, + asSimpleLocalId(schemaMap[referenceContent[r] ?? 0], spaceUrl), + ]) + .filter(([, s]) => s !== undefined) as [string, string][], + ); return { rid, localId: schema.source_local_id!, @@ -105,6 +124,7 @@ export const dbNodeSchemaToCrossApp = ( template: template_content as string | undefined, templateTitle: template as string | undefined, authorId, + slotDefinitions, }; }; @@ -126,7 +146,11 @@ export const dbNodeSchemasToCrossApp = async ({ ); accountMap = await getAccountMap(client, [...authorIds]); } - return schemas.map((r) => dbNodeSchemaToCrossApp(r, spaceMap, accountMap)); + const referredSchemaIds = schemas.flatMap((schema) => schema.refs); + const schemaMap = await getConceptMap(client, referredSchemaIds, spaceMap); + return schemas.map((schema) => + dbNodeSchemaToCrossApp({ schema, spaceMap, accountMap, schemaMap }), + ); }; export const dbRelationTypeSchemaToCrossApp = ( From dfdb98e80a79f67c2aaa5de992998b525a0426d4 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Fri, 21 Aug 2026 16:30:07 -0400 Subject: [PATCH 2/4] add slots to SharedNode --- .../src/lib/__tests__/sharedNodes.test.ts | 2 + packages/database/src/lib/sharedNodes.ts | 43 ++++++++++++++++--- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/packages/database/src/lib/__tests__/sharedNodes.test.ts b/packages/database/src/lib/__tests__/sharedNodes.test.ts index 37b422299..af73fb196 100644 --- a/packages/database/src/lib/__tests__/sharedNodes.test.ts +++ b/packages/database/src/lib/__tests__/sharedNodes.test.ts @@ -19,6 +19,8 @@ const nodes: BuildArgs["nodes"] = [ schema_id: 200, source_local_id: "node-1", space_id: 20, + reference_content: {}, + concepts_of_relation: [], }, ]; const directContents: BuildArgs["directContents"] = [ diff --git a/packages/database/src/lib/sharedNodes.ts b/packages/database/src/lib/sharedNodes.ts index 36dd023e7..c5c7c549a 100644 --- a/packages/database/src/lib/sharedNodes.ts +++ b/packages/database/src/lib/sharedNodes.ts @@ -4,8 +4,19 @@ import type { Enums, Json, Tables } from "../dbTypes"; type SharedConcept = Pick< Tables<"my_concepts">, - "is_schema" | "last_modified" | "schema_id" | "source_local_id" | "space_id" ->; + | "is_schema" + | "last_modified" + | "schema_id" + | "source_local_id" + | "space_id" + | "reference_content" +> & { + concepts_of_relation: { + id: number | null; + space_id: number | null; + source_local_id: string | null; + }[]; +}; type SharedContent = Pick< Tables<"my_contents">, | "author_id" @@ -45,6 +56,7 @@ export type SharedNode = { lastModified: string; authorId?: number; directMetadata: Json; + slots?: Record; }; export type SharedNodeRows = { @@ -54,8 +66,8 @@ export type SharedNodeRows = { spaces: SharedSpace[]; }; -const CONCEPT_COLUMNS = - "is_schema, last_modified, schema_id, source_local_id, space_id"; +const CONCEPT_COLUMNS_WITH_SLOTS = + "is_schema, last_modified, schema_id, source_local_id, space_id, reference_content, concepts_of_relation(id, space_id, source_local_id)"; const DIRECT_CONTENT_COLUMNS = "author_id, created, last_modified, metadata, source_local_id, space_id, text, variant"; const FULL_CONTENT_SUMMARY_COLUMNS = "last_modified, source_local_id, space_id"; @@ -184,6 +196,24 @@ export const buildSharedNodes = ({ return []; } + const nodeRidById = Object.fromEntries( + node.concepts_of_relation.map((c) => { + if (c.space_id === node.space_id) return [c.id, c.source_local_id]; + const space = spacesById.get(c.space_id || 0); + if (!space || !c.source_local_id || !c.id) return [c.id, undefined]; + return [c.id, spaceUriAndLocalIdToRid(space.url, c.source_local_id)]; + }) as [number, string | undefined][], + ); + const referenceContent = (node.reference_content ?? {}) as Record< + string, + number + >; + const slots = Object.fromEntries( + Object.entries(referenceContent ?? {}) + .map(([k, v]) => [k, nodeRidById[v]]) + .filter(([, v]) => v !== undefined) as [string, string][], + ); + return [ { rid, @@ -197,6 +227,7 @@ export const buildSharedNodes = ({ lastModified, authorId: direct.author_id ?? undefined, directMetadata: direct.metadata, + slots: Object.keys(slots).length > 0 ? slots : undefined, }, ]; }) @@ -218,7 +249,7 @@ const getSharedNodeRows = async ({ await Promise.all([ client .from("my_concepts") - .select(CONCEPT_COLUMNS) + .select(CONCEPT_COLUMNS_WITH_SLOTS) .neq("space_id", currentSpaceId) .eq("is_schema", false) .eq("is_relation", false), @@ -278,7 +309,7 @@ export const getSharedNodeByRid = async ({ const [conceptsResponse, directResponse, fullResponse] = await Promise.all([ client .from("my_concepts") - .select(CONCEPT_COLUMNS) + .select(CONCEPT_COLUMNS_WITH_SLOTS) .eq("space_id", space.id) .eq("source_local_id", sourceLocalId) .eq("is_schema", false) From 03486e78998072756266fac11e980b7c42cd3438 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Sat, 22 Aug 2026 12:39:39 -0400 Subject: [PATCH 3/4] tests --- .../__tests__/dbToCrossAppConverters.test.ts | 53 +++++++++++++++++++ .../src/lib/__tests__/sharedNodes.test.ts | 30 +++++++++++ 2 files changed, 83 insertions(+) diff --git a/packages/database/src/lib/__tests__/dbToCrossAppConverters.test.ts b/packages/database/src/lib/__tests__/dbToCrossAppConverters.test.ts index f52141a7c..1251cb5e2 100644 --- a/packages/database/src/lib/__tests__/dbToCrossAppConverters.test.ts +++ b/packages/database/src/lib/__tests__/dbToCrossAppConverters.test.ts @@ -67,6 +67,59 @@ describe("dbNodeSchemaToCrossApp", () => { }); }); + it("resolves slot definitions from roles and reference content", () => { + const schema = baseConcept({ + literal_content: { roles: ["evidence", "claim"], extra: "kept" }, + reference_content: { evidence: 10, claim: 20 }, + }); + const result = dbNodeSchemaToCrossApp({ + schema, + spaceMap, + accountMap, + schemaMap: { + 10: "orn:obsidian.schema:vault-a/evidence-type", + 20: "orn:obsidian.schema:vault-a/claim-type", + }, + }); + // schemas are always local, so slots hold plain source local ids + expect(result.slotDefinitions).toEqual({ + evidence: "evidence-type", + claim: "claim-type", + }); + // roles drive the slot definitions, they are not kept as plain metadata + expect(result.metadata).toEqual({ extra: "kept" }); + }); + + it("throws when a slot points at a schema in another space", () => { + const schema = baseConcept({ + literal_content: { roles: ["evidence"] }, + reference_content: { evidence: 10 }, + }); + expect(() => + dbNodeSchemaToCrossApp({ + schema, + spaceMap, + accountMap, + schemaMap: { 10: "orn:obsidian.schema:vault-b/evidence-type" }, + }), + ).toThrow("Unexpected spaceUri"); + }); + + it("omits slots whose referenced schema cannot be resolved", () => { + const schema = baseConcept({ + literal_content: { roles: ["evidence", "claim"] }, + reference_content: { evidence: 10 }, + }); + expect( + dbNodeSchemaToCrossApp({ + schema, + spaceMap, + accountMap, + schemaMap: { 10: "orn:obsidian.schema:vault-a/evidence-type" }, + }).slotDefinitions, + ).toEqual({ evidence: "evidence-type" }); + }); + it("throws when the author is unknown", () => { const schema = baseConcept({ author_id: 999 }); expect(() => diff --git a/packages/database/src/lib/__tests__/sharedNodes.test.ts b/packages/database/src/lib/__tests__/sharedNodes.test.ts index af73fb196..2ef057cd8 100644 --- a/packages/database/src/lib/__tests__/sharedNodes.test.ts +++ b/packages/database/src/lib/__tests__/sharedNodes.test.ts @@ -169,6 +169,36 @@ describe("buildSharedNodes", () => { expect(build({ nodesOverride, directOverride })).toEqual([]); }); + it("resolves slots to local ids in the same space and rids elsewhere", () => { + const otherSpace: BuildArgs["spaces"][number] = { + id: 21, + name: "Other vault", + platform: "Obsidian", + url: "obsidian:vault-b", + }; + const nodeWithSlots: BuildArgs["nodes"][number] = { + ...nodes[0]!, + reference_content: { evidence: 5, claim: 6, dangling: 7 }, + concepts_of_relation: [ + { id: 5, space_id: 20, source_local_id: "node-5" }, + { id: 6, space_id: 21, source_local_id: "node-6" }, + ], + }; + expect( + build({ + nodesOverride: [nodeWithSlots], + spacesOverride: [...spaces, otherSpace], + })[0]?.slots, + ).toEqual({ + evidence: "node-5", + claim: "orn:obsidian:vault-b/node-6", + }); + }); + + it("leaves slots undefined when the node references nothing", () => { + expect(build()[0]?.slots).toBeUndefined(); + }); + it("sorts newest nodes first", () => { const olderNode = { ...nodes[0]!, From 9ca6cdebf97b99f29b88f16aa68aa4ddf2ab9453 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Sun, 23 Aug 2026 08:53:55 -0400 Subject: [PATCH 4/4] ENG-2165-Refactor Roam Sync to use crossAppNodes --- .../settings/DiscourseNodeSuggestiveRules.tsx | 9 +- .../roamToCrossAppConverters.test.ts | 143 ++++++++++- apps/roam/src/utils/conceptConversion.ts | 65 ----- .../src/utils/convertRoamNodeToFullContent.ts | 32 --- .../src/utils/getAllDiscourseNodesSince.ts | 7 +- .../src/utils/roamToCrossAppConverters.ts | 70 ++++-- apps/roam/src/utils/syncDgNodesToSupabase.ts | 231 +++++++----------- apps/roam/src/utils/templateToText.ts | 20 ++ .../upsertNodesAsContentWithEmbeddings.ts | 133 +++------- packages/database/src/crossAppContracts.ts | 4 + .../database/src/lib/crossAppConverters.ts | 10 +- 11 files changed, 345 insertions(+), 379 deletions(-) delete mode 100644 apps/roam/src/utils/convertRoamNodeToFullContent.ts create mode 100644 apps/roam/src/utils/templateToText.ts diff --git a/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx b/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx index 67a546c92..a3dec1dcf 100644 --- a/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx +++ b/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx @@ -5,7 +5,7 @@ import getSubTree from "roamjs-components/util/getSubTree"; import { DiscourseNode } from "~/utils/getDiscourseNodes"; import extractRef from "roamjs-components/util/extractRef"; import { getAllDiscourseNodesSince } from "~/utils/getAllDiscourseNodesSince"; -import { upsertNodesToSupabaseAsContentWithEmbeddings } from "~/utils/syncDgNodesToSupabase"; +import { upsertNodesWithEmbeddings } from "~/utils/syncDgNodesToSupabase"; import { getLoggedInClient, getSupabaseContext } from "~/utils/supabaseContext"; import { DiscourseNodeFlagPanel, @@ -56,11 +56,12 @@ const DiscourseNodeSuggestiveRules = ({ const context = await getSupabaseContext(); if (context && blockNodesSince) { - await upsertNodesToSupabaseAsContentWithEmbeddings( - blockNodesSince, + await upsertNodesWithEmbeddings({ + nodes: blockNodesSince, + nodeTypes: [node], supabaseClient, context, - ); + }); } } finally { setIsUpdating(false); diff --git a/apps/roam/src/utils/__tests__/roamToCrossAppConverters.test.ts b/apps/roam/src/utils/__tests__/roamToCrossAppConverters.test.ts index 5596eb602..f17518a0d 100644 --- a/apps/roam/src/utils/__tests__/roamToCrossAppConverters.test.ts +++ b/apps/roam/src/utils/__tests__/roamToCrossAppConverters.test.ts @@ -9,7 +9,17 @@ vi.mock("roamjs-components/queries/getPageViewType", () => ({ })); vi.mock("~/utils/pageToMarkdown", () => ({ toMarkdown: () => "" })); -import { nodeUidsWithTypeToCrossApp } from "~/utils/roamToCrossAppConverters"; +import { + contentNodeToCrossApp, + nodeSchemaToCrossApp, + nodeUidsWithTypeToCrossApp, +} from "~/utils/roamToCrossAppConverters"; +import { + crossAppNodeSchemaToDbConcept, + crossAppNodeToDbConcept, + crossAppNodeToDbContent, +} from "@repo/database/lib/crossAppConverters"; +import type { DiscourseNode } from "~/utils/getDiscourseNodes"; const USER_ROW = { ":db/id": 5, ":user/uid": "user-1" }; @@ -60,3 +70,134 @@ describe("nodeUidsWithTypeToCrossApp timestamps", () => { expect(node.modifiedAt).toEqual(new Date(1000)); }); }); + +const NODE_ROW = { + author_local_id: "user-1", + source_local_id: "node-1", + created: 1000, + last_modified: 2000, + text: "EVD - some evidence", + type: "schema-1", +}; + +describe("contentNodeToCrossApp", () => { + it("names a page-backed node by its title, as a plain direct", () => { + const node = contentNodeToCrossApp(NODE_ROW); + expect(node.content.direct.value).toEqual("EVD - some evidence"); + expect(node.content.direct.variant).toEqual("direct"); + expect(node.content.full).toBeUndefined(); + expect(node.nodeType).toEqual("schema-1"); + expect(node.createdAt).toEqual(new Date(1000)); + expect(node.modifiedAt).toEqual(new Date(2000)); + }); + + it("names a block-backed node by its page title and its text", () => { + const node = contentNodeToCrossApp({ + ...NODE_ROW, + node_title: "EVD - some evidence", + text: "the block text", + }); + expect(node.content.direct.value).toEqual( + "EVD - some evidence the block text", + ); + expect(node.content.direct.variant).toEqual("direct_and_description"); + }); + + it("falls back to a plain direct when the first child block is missing", () => { + const node = contentNodeToCrossApp({ + ...NODE_ROW, + node_title: "EVD - some evidence", + text: "", + }); + expect(node.content.direct.value).toEqual("EVD - some evidence"); + expect(node.content.direct.variant).toEqual("direct"); + }); + + it("carries the variant of the direct slot into the db content", () => { + const node = contentNodeToCrossApp({ + ...NODE_ROW, + node_title: "EVD - some evidence", + text: "the block text", + }); + expect(crossAppNodeToDbContent(node, "direct")).toMatchObject({ + source_local_id: "node-1", + text: "EVD - some evidence the block text", + variant: "direct_and_description", + scale: "document", + author_local_id: "user-1", + }); + }); + + it("carries the node's direct content into the db concept", () => { + const concept = crossAppNodeToDbConcept(contentNodeToCrossApp(NODE_ROW)); + expect(concept.contents_inline).toEqual([ + expect.objectContaining({ + source_local_id: "node-1", + text: "EVD - some evidence", + variant: "direct", + }), + ]); + expect(concept).toMatchObject({ + source_local_id: "node-1", + name: "EVD - some evidence", + author_local_id: "user-1", + schema_represented_by_local_id: "schema-1", + }); + }); +}); + +const schemaPull = () => ({ + ":create/time": 1000, + ":edit/time": 2000, + ":create/user": { ":user/uid": "user-1" }, +}); + +const convertSchema = (node: DiscourseNode) => { + (globalThis as { window: unknown }).window = { + roamAlphaAPI: { pull: schemaPull }, + }; + return nodeSchemaToCrossApp(node); +}; + +const schemaNode = (overrides: Partial): DiscourseNode => ({ + text: "Evidence", + type: "_EVD-node", + shortcut: "e", + format: "[[EVD]] - {content}", + specification: [], + backedBy: "user", + canvasSettings: {}, + ...overrides, +}); + +describe("nodeSchemaToCrossApp", () => { + it("carries the modification time, so that the concept can be inserted", () => { + const schema = convertSchema(schemaNode({})); + expect(schema?.createdAt).toEqual(new Date(1000)); + expect(schema?.modifiedAt).toEqual(new Date(2000)); + }); + + it("renders the template as text, under template_content in the db concept", () => { + const schema = convertSchema( + schemaNode({ + template: [ + { text: "Question:", children: [{ text: "why?" }] }, + { text: "{{roam/render}}" }, + ], + }), + ); + expect(schema?.template).toEqual("* Question:\n * why?\n \n"); + const concept = crossAppNodeSchemaToDbConcept(schema!); + expect(concept.literal_content).toEqual({ + template_content: "* Question:\n * why?\n \n", + }); + }); + + it("omits the template when the node type has none", () => { + const schema = convertSchema(schemaNode({ template: [] })); + expect(schema?.template).toBeUndefined(); + expect( + crossAppNodeSchemaToDbConcept(schema!).literal_content, + ).toBeUndefined(); + }); +}); diff --git a/apps/roam/src/utils/conceptConversion.ts b/apps/roam/src/utils/conceptConversion.ts index 83abd1a14..a5086d147 100644 --- a/apps/roam/src/utils/conceptConversion.ts +++ b/apps/roam/src/utils/conceptConversion.ts @@ -1,6 +1,4 @@ -import { InputTextNode } from "roamjs-components/types"; import getBlockProps from "./getBlockProps"; -import { DiscourseNode } from "./getDiscourseNodes"; import getDiscourseRelations from "./getDiscourseRelations"; import type { DiscourseRelation } from "./getDiscourseRelations"; import type { SupabaseContext } from "~/utils/supabaseContext"; @@ -58,69 +56,6 @@ const getNodeExtraData = ( /* eslint-enable @typescript-eslint/naming-convention */ }; -const indent = (s: string): string => - s - .split("\n") - .map((l) => " " + l) - .join("\n") + "\n"; - -const templateToText = (template: InputTextNode[]): string => - template - .filter((itn) => !itn.text.startsWith("{{")) - .map( - (itn) => - `* ${itn.text}\n${itn.children?.length ? indent(templateToText(itn.children)) : ""}`, - ) - .join(""); - -export const discourseNodeSchemaToLocalConcept = ( - context: SupabaseContext, - node: DiscourseNode, -): LocalConceptDataInput => { - const titleParts = node.text.split("/"); - const label = titleParts[titleParts.length - 1] ?? node.text; - const result: LocalConceptDataInput = { - space_id: context.spaceId, - name: node.text, - source_local_id: node.type, - is_schema: true, - literal_content: { - label, - }, - /* eslint-enable @typescript-eslint/naming-convention */ - ...getNodeExtraData(node.type), - }; - if (node.template !== undefined) - result.literal_content = { - label, - template: templateToText(node.template), - }; - return result; -}; - -export const discourseNodeBlockToLocalConcept = ( - context: SupabaseContext, - { - nodeUid, - schemaUid, - text, - }: { - nodeUid: string; - schemaUid: string; - text: string; - }, -): LocalConceptDataInput => { - return { - space_id: context.spaceId, - name: text, - source_local_id: nodeUid, - schema_represented_by_local_id: schemaUid, - is_schema: false, - /* eslint-enable @typescript-eslint/naming-convention */ - ...getNodeExtraData(nodeUid), - }; -}; - const STANDARD_ROLES = ["source", "destination"]; export const discourseRelationSchemaToLocalConcept = ( diff --git a/apps/roam/src/utils/convertRoamNodeToFullContent.ts b/apps/roam/src/utils/convertRoamNodeToFullContent.ts deleted file mode 100644 index d1f24cba1..000000000 --- a/apps/roam/src/utils/convertRoamNodeToFullContent.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { crossAppNodeToDbContent } from "@repo/database/lib/crossAppConverters"; -import { fullContentNodeToCrossApp } from "./roamToCrossAppConverters"; -import type { LocalContentDataInput } from "@repo/database/inputTypes"; - -export type RoamFullContentNode = { - author_local_id: string; - source_local_id: string; - created: string | number; - last_modified: string | number; - text: string; - node_type_id: string; - node_title?: string; -}; - -export const convertRoamNodeToFullContent = ({ - nodes, -}: { - nodes: RoamFullContentNode[]; -}): LocalContentDataInput[] => - nodes.flatMap((node) => { - try { - const crossAppNode = fullContentNodeToCrossApp(node); - const fullContent = crossAppNodeToDbContent(crossAppNode, "full"); - return fullContent === undefined ? [] : [fullContent]; - } catch (error) { - console.error( - `convertRoamNodeToFullContent: failed to build full markdown for ${node.source_local_id}:`, - error, - ); - return []; - } - }); diff --git a/apps/roam/src/utils/getAllDiscourseNodesSince.ts b/apps/roam/src/utils/getAllDiscourseNodesSince.ts index 53046c40e..bafe5511c 100644 --- a/apps/roam/src/utils/getAllDiscourseNodesSince.ts +++ b/apps/roam/src/utils/getAllDiscourseNodesSince.ts @@ -6,10 +6,11 @@ const DEFAULT_TIME = new Date("1970-01-01").getTime(); export type RoamDiscourseNodeData = { author_local_id: string; - author_name: string; + author_name?: string; source_local_id: string; - created: string; - last_modified: string; + // Roam returns :create/time and :edit/time as epoch milliseconds + created: number; + last_modified: number; text: string; type: string; node_title?: string; diff --git a/apps/roam/src/utils/roamToCrossAppConverters.ts b/apps/roam/src/utils/roamToCrossAppConverters.ts index a3c33f399..09e6cc412 100644 --- a/apps/roam/src/utils/roamToCrossAppConverters.ts +++ b/apps/roam/src/utils/roamToCrossAppConverters.ts @@ -4,8 +4,8 @@ import type { CrossAppRelation, CrossAppRelationTripleSchema, } from "@repo/database/crossAppContracts"; -import type { RoamFullContentNode } from "./convertRoamNodeToFullContent"; import type { DiscourseNode } from "./getDiscourseNodes"; +import type { RoamDiscourseNodeData } from "./getAllDiscourseNodesSince"; import type { TreeNode, ViewType } from "roamjs-components/types"; import type { NodeUidWithType } from "~/utils/publishNodesToGroups"; import type { Json } from "@repo/database/dbTypes"; @@ -15,6 +15,7 @@ import { toMarkdown } from "./pageToMarkdown"; import getFullTreeByParentUid from "roamjs-components/queries/getFullTreeByParentUid"; import getPageViewType from "roamjs-components/queries/getPageViewType"; import { contentTypes } from "@repo/content-model"; +import { templateToText } from "./templateToText"; const FULL_MARKDOWN_OPTS = { refs: true, @@ -62,25 +63,53 @@ const buildFullInlineContent = ({ }; }; +// A node whose text lives in a first child block below its page (the isFirstChild +// node type setting) is known by its page title followed by that block's text, and +// its direct content is a direct_and_description. A node that is a page holds a +// plain direct: its title. When the first child block is missing, there is no +// description to speak of, and the title alone is left: a plain direct again. +// Known gap: a published isFirstChild node should arguably carry both, so that its +// plain direct is kept up to date alongside the direct_and_description. +const directContent = ( + node: RoamDiscourseNodeData, +): CrossAppNode["content"]["direct"] => + node.node_title && node.text + ? { + localId: node.source_local_id, + value: `${node.node_title} ${node.text}`, + variant: "direct_and_description", + } + : { + localId: node.source_local_id, + value: node.node_title || node.text, + variant: "direct", + }; + +// The node without its full content: building that content walks the whole page, +// so callers that do not need it should not pay for it. +export const contentNodeToCrossApp = ( + node: RoamDiscourseNodeData, +): CrossAppNode => ({ + authorId: node.author_local_id, + localId: node.source_local_id, + createdAt: new Date(node.created || Date.now()), + modifiedAt: new Date(node.last_modified || Date.now()), + nodeType: node.type, + content: { + direct: directContent(node), + }, +}); + export const fullContentNodeToCrossApp = ( - node: RoamFullContentNode, + node: RoamDiscourseNodeData, ): CrossAppNode => { - const title = node.node_title ?? node.text; - - return { - authorId: node.author_local_id, - localId: node.source_local_id, - createdAt: new Date(node.created || Date.now()), - modifiedAt: new Date(node.last_modified || Date.now()), - nodeType: node.node_type_id, - content: { - direct: { - localId: node.source_local_id, - value: title, - }, - full: buildFullInlineContent({ uid: node.source_local_id, title }), - }, - }; + const crossAppNode = contentNodeToCrossApp(node); + // the full content is a rendering of the node's page, so it is titled by the page + crossAppNode.content.full = buildFullInlineContent({ + uid: node.source_local_id, + title: node.node_title ?? node.text, + }); + return crossAppNode; }; export const nodeUidsWithTypeToCrossApp = async ( @@ -195,10 +224,13 @@ export const nodeSchemaToCrossApp = ( if (!relData) return null; const userUid = (relData[":create/user"] ?? {})[":user/uid"]; if (!userUid) return null; + const createdTime = relData[":create/time"] || Date.now(); return { localId: s.type, label: s.text, authorId: userUid, - createdAt: new Date(relData[":create/time"] || Date.now()), + createdAt: new Date(createdTime), + modifiedAt: new Date(relData[":edit/time"] || createdTime), + ...(s.template?.length ? { template: templateToText(s.template) } : {}), }; }; diff --git a/apps/roam/src/utils/syncDgNodesToSupabase.ts b/apps/roam/src/utils/syncDgNodesToSupabase.ts index af20f4566..164f7043e 100644 --- a/apps/roam/src/utils/syncDgNodesToSupabase.ts +++ b/apps/roam/src/utils/syncDgNodesToSupabase.ts @@ -12,17 +12,19 @@ import { } from "./supabaseContext"; import { type RoamDiscourseNodeData } from "./getAllDiscourseNodesSince"; import getDiscourseNodes, { type DiscourseNode } from "./getDiscourseNodes"; +import { orderConceptsByDependency } from "./conceptConversion"; import { - discourseNodeBlockToLocalConcept, - discourseNodeSchemaToLocalConcept, - orderConceptsByDependency, -} from "./conceptConversion"; -import { fetchEmbeddingsForNodes } from "./upsertNodesAsContentWithEmbeddings"; -import { convertRoamNodeToLocalContent } from "./upsertNodesAsContentWithEmbeddings"; + contentNodeToCrossApp, + fullContentNodeToCrossApp, + nodeSchemaToCrossApp, +} from "./roamToCrossAppConverters"; +import type { CrossAppNode } from "@repo/database/crossAppContracts"; import { - convertRoamNodeToFullContent, - type RoamFullContentNode, -} from "./convertRoamNodeToFullContent"; + crossAppNodeSchemaToDbConcept, + crossAppNodeToDbConcept, +} from "@repo/database/lib/crossAppConverters"; +import { attachEmbeddingsToNodes } from "./upsertNodesAsContentWithEmbeddings"; +import { convertRoamNodeToLocalContent } from "./upsertNodesAsContentWithEmbeddings"; import type { DGSupabaseClient } from "@repo/database/lib/client"; import { intersection } from "@repo/utils/setOperations"; import type { Json, Enums } from "@repo/database/dbTypes"; @@ -45,8 +47,9 @@ const SYNC_INTERVAL = "130s"; // Interval between syncs for each client individually const BASE_SYNC_INTERVAL = 5 * 60 * 1000; // 5 minutes const SYNC_TIMEOUT = "60s"; // must be less than half the SYNC_INTERVAL. -const BATCH_SIZE = 200; -const CONCEPT_BATCH_SIZE = 200; +// Concepts carry their content, and a published node's full content is the markdown +// of its whole page, so these batches are kept small. +const CONCEPT_BATCH_SIZE = 50; const END_SYNC_TASK_RESULT_VERSION = 1; const DEFAULT_SYNC_TIME = new Date("1970-01-01").getTime(); @@ -592,6 +595,7 @@ const upsertNodeSchemaToContent = async ({ userId: number; supabaseClient: DGSupabaseClient; }) => { + if (nodeTypesUids.length === 0) return; const query = `[ :find ?uid ?create-time ?edit-time ?user-uuid ?title ?author-name :keys source_local_id created last_modified author_local_id text author_name @@ -629,8 +633,13 @@ const upsertNodeSchemaToContent = async ({ } }; +// A node is sent as a single concept carrying its content: its direct content +// (with an embedding, unless the caller opts out) and, for published nodes, the +// full markdown of its page. export const convertDgToSupabaseConcepts = async ({ nodesSince, + fullContentNodes = [], + withEmbeddings = true, since, allNodeTypes, sharedNodeTypeIds = new Set(), @@ -638,6 +647,8 @@ export const convertDgToSupabaseConcepts = async ({ context, }: { nodesSince: RoamDiscourseNodeData[]; + fullContentNodes?: CrossAppNode[]; + withEmbeddings?: boolean; since: number | undefined; allNodeTypes: DiscourseNode[]; sharedNodeTypeIds?: ReadonlySet; @@ -663,18 +674,50 @@ export const convertDgToSupabaseConcepts = async ({ supabaseClient, }); - const nodesTypesToLocalConcepts = nodeTypes.map((node) => { - return discourseNodeSchemaToLocalConcept(context, node); + const crossAppNodeSchemas = nodeTypes.map((node) => ({ + node, + schema: nodeSchemaToCrossApp(node), + })); + const unconvertibleTypes = crossAppNodeSchemas + .filter(({ schema }) => schema === null) + .map(({ node }) => node.type); + if (unconvertibleTypes.length > 0) { + console.warn( + "Some node types have no author and were not synced:", + unconvertibleTypes, + ); + } + const nodesTypesToLocalConcepts = crossAppNodeSchemas + .map(({ schema }) => schema) + .filter((schema) => schema !== null) + .map((schema) => crossAppNodeSchemaToDbConcept(schema)); + + const crossAppNodes = new Map( + nodesSince.map((node) => [ + node.source_local_id, + contentNodeToCrossApp(node), + ]), + ); + // Nodes whose embedding is worth (re)computing: the ones that changed. A node + // present only for a full content backfill has not changed, and keeps its own. + const nodesToEmbed = [...crossAppNodes.values()]; + + fullContentNodes.forEach((withFullContent) => { + const changed = crossAppNodes.get(withFullContent.localId); + if (changed === undefined) { + crossAppNodes.set(withFullContent.localId, withFullContent); + } else { + changed.content.full = withFullContent.content.full; + } }); - const nodeBlockToLocalConcepts = nodesSince.map((node) => { - const localConcept = discourseNodeBlockToLocalConcept(context, { - nodeUid: node.source_local_id, - schemaUid: node.type, - text: node.node_title ? `${node.node_title} ${node.text}` : node.text, - }); - return localConcept; - }); + if (withEmbeddings) { + await attachEmbeddingsToNodes(nodesToEmbed); + } + + const nodeBlockToLocalConcepts = [...crossAppNodes.values()].map((node) => + crossAppNodeToDbConcept(node), + ); const conceptsToUpsert = [ ...nodesTypesToLocalConcepts, @@ -696,108 +739,24 @@ export const convertDgToSupabaseConcepts = async ({ }); }; -const uploadContentBatches = async ({ - content, - supabaseClient, - context, -}: { - content: LocalContentDataInput[]; - supabaseClient: DGSupabaseClient; - context: SupabaseContext; -}): Promise => { - if (content.length === 0) { - return; - } - - const batches = chunk(content, BATCH_SIZE); - - for (let idx = 0; idx < batches.length; idx++) { - const batch = batches[idx]; - - const { error } = await supabaseClient.rpc("upsert_content", { - data: batch as Json, - v_space_id: context.spaceId, - v_creator_id: context.userId, - content_as_document: true, - }); - - if (error) { - throw new Error(`upsert_content failed for batch ${idx + 1}`, { - cause: error, - }); - } - } -}; - -export const upsertNodesToSupabaseAsContentWithEmbeddings = async ( - roamNodes: RoamDiscourseNodeData[], - supabaseClient: DGSupabaseClient, - context: SupabaseContext, -): Promise => { - if (roamNodes.length === 0) { - return; - } - const allNodeInstancesAsLocalContent = convertRoamNodeToLocalContent({ - nodes: roamNodes, - }); - - let nodesWithEmbeddings: LocalContentDataInput[]; - try { - nodesWithEmbeddings = await fetchEmbeddingsForNodes( - allNodeInstancesAsLocalContent, - ); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - console.error( - `upsertNodesToSupabaseAsContentWithEmbeddings: Embedding service failed – ${message}`, - ); - throw new Error(message); - } - - if (nodesWithEmbeddings.length !== allNodeInstancesAsLocalContent.length) { - console.error( - "upsertNodesToSupabaseAsContentWithEmbeddings: Mismatch between node and embedding counts.", - ); - throw new Error( - "upsertNodesToSupabaseAsContentWithEmbeddings: Mismatch between node and embedding counts.", - ); - } - - await uploadContentBatches({ - content: nodesWithEmbeddings, - supabaseClient, - context, - }); -}; - -const upsertNodesToSupabaseAsContent = async ( - roamNodes: RoamDiscourseNodeData[], - supabaseClient: DGSupabaseClient, - context: SupabaseContext, -): Promise => { - if (roamNodes.length === 0) { - return; - } - const content = convertRoamNodeToLocalContent({ nodes: roamNodes }); - await uploadContentBatches({ content, supabaseClient, context }); -}; - -const upsertRoamNodesToSupabaseAsFullContent = async ({ +// Recomputes the embeddings of a node type's nodes, on demand from the settings panel. +export const upsertNodesWithEmbeddings = async ({ nodes, + nodeTypes, supabaseClient, context, }: { - nodes: RoamFullContentNode[]; + nodes: RoamDiscourseNodeData[]; + nodeTypes: DiscourseNode[]; supabaseClient: DGSupabaseClient; context: SupabaseContext; }): Promise => { - if (nodes.length === 0) { - return; - } - - const fullContent = convertRoamNodeToFullContent({ nodes }); - await uploadContentBatches({ - content: fullContent, + if (nodes.length === 0) return; + await convertDgToSupabaseConcepts({ + nodesSince: nodes, + since: undefined, + allNodeTypes: nodeTypes, + sharedNodeTypeIds: new Set(nodeTypes.map((nodeType) => nodeType.type)), supabaseClient, context, }); @@ -998,7 +957,7 @@ type SharedFullContentUpdateRow = { }; type SharedFullContentUpdate = { - fullContentNode: RoamFullContentNode; + node: CrossAppNode; nodeTypeId: string; }; @@ -1055,14 +1014,14 @@ const getSharedRoamNodesWithFullContentUpdatesSince = async ({ return [ { - fullContentNode: { + node: fullContentNodeToCrossApp({ author_local_id: row.author_local_id, source_local_id: row.source_local_id, created: row.created, last_modified: Math.max(row.node_edit_time, row.page_edit_time), text: row.text, - node_type_id: matchingNodeType.type, - }, + type: matchingNodeType.type, + }), nodeTypeId: matchingNodeType.type, }, ]; @@ -1266,7 +1225,7 @@ export const createOrUpdateDiscourseEmbedding = async ( }, }); const sharedFullContentNodes = sharedFullContentUpdates.map( - (update) => update.fullContentNode, + (update) => update.node, ); const sharedNodeTypeIds = new Set( sharedFullContentUpdates.map((update) => update.nodeTypeId), @@ -1278,38 +1237,16 @@ export const createOrUpdateDiscourseEmbedding = async ( operation: () => upsertUsers(allUsers, activeSupabaseClient, activeContext), }); - await measureSyncPhase({ - phase: "upsertNodes", - phases, - operation: () => - sharedNodesOnlySync - ? upsertNodesToSupabaseAsContent( - nodeInstancesToSync, - activeSupabaseClient, - activeContext, - ) - : upsertNodesToSupabaseAsContentWithEmbeddings( - nodeInstancesToSync, - activeSupabaseClient, - activeContext, - ), - }); - await measureSyncPhase({ - phase: "upsertFullContent", - phases, - operation: () => - upsertRoamNodesToSupabaseAsFullContent({ - nodes: sharedFullContentNodes, - supabaseClient: activeSupabaseClient, - context: activeContext, - }), - }); + // Nodes are upserted as concepts carrying their content: the direct content of + // every changed node, and the full page content of the published ones. await measureSyncPhase({ phase: "convertConcepts", phases, operation: () => convertDgToSupabaseConcepts({ nodesSince: nodeInstancesToSync, + fullContentNodes: sharedFullContentNodes, + withEmbeddings: !sharedNodesOnlySync, since: sinceTime, allNodeTypes: allDgNodeTypes, sharedNodeTypeIds, diff --git a/apps/roam/src/utils/templateToText.ts b/apps/roam/src/utils/templateToText.ts new file mode 100644 index 000000000..d4f143028 --- /dev/null +++ b/apps/roam/src/utils/templateToText.ts @@ -0,0 +1,20 @@ +import { InputTextNode } from "roamjs-components/types"; + +const indent = (s: string): string => + s + .split("\n") + .map((l) => " " + l) + .join("\n") + "\n"; + +// A discourse node's template, as a markdown bullet list. Roam's own template +// macros ({{...}}) are dropped, as they mean nothing outside Roam. +export const templateToText = (template: InputTextNode[]): string => + template + .filter((itn) => !itn.text.startsWith("{{")) + .map( + (itn) => + `* ${itn.text}\n${itn.children?.length ? indent(templateToText(itn.children)) : ""}`, + ) + .join(""); + +export default templateToText; diff --git a/apps/roam/src/utils/upsertNodesAsContentWithEmbeddings.ts b/apps/roam/src/utils/upsertNodesAsContentWithEmbeddings.ts index 88ec44f7f..6dbb8479a 100644 --- a/apps/roam/src/utils/upsertNodesAsContentWithEmbeddings.ts +++ b/apps/roam/src/utils/upsertNodesAsContentWithEmbeddings.ts @@ -1,9 +1,9 @@ import { type RoamDiscourseNodeData } from "./getAllDiscourseNodesSince"; -import { type SupabaseContext } from "./supabaseContext"; import { nextApiRoot } from "@repo/utils/execContext"; -import type { DGSupabaseClient } from "@repo/database/lib/client"; -import type { Json } from "@repo/database/dbTypes"; import type { LocalContentDataInput } from "@repo/database/inputTypes"; +import { crossAppNodeToDbContent } from "@repo/database/lib/crossAppConverters"; +import type { CrossAppNode } from "@repo/database/crossAppContracts"; +import { contentNodeToCrossApp } from "./roamToCrossAppConverters"; const EMBEDDING_BATCH_SIZE = 200; const EMBEDDING_MODEL = "openai_text_embedding_3_small_1536"; @@ -14,34 +14,24 @@ type EmbeddingApiResponse = { }[]; }; +// The direct content of a node, as the cross-app converters define it. The embedding +// is added afterwards, by fetchEmbeddingsForNodes: it is a concern of the sync only. export const convertRoamNodeToLocalContent = ({ nodes, }: { nodes: RoamDiscourseNodeData[]; -}): LocalContentDataInput[] => { - return nodes.map((node) => { - const variant = node.node_title ? "direct_and_description" : "direct"; - const text = node.node_title - ? `${node.node_title} ${node.text}` - : node.text; - return { - author_local_id: node.author_local_id, - source_local_id: node.source_local_id, - created: new Date(node.created || Date.now()).toISOString(), - last_modified: new Date(node.last_modified || Date.now()).toISOString(), - text: text, - variant: variant, - scale: "document", - // use the default text/plain content type - }; +}): LocalContentDataInput[] => + nodes.flatMap((node) => { + const content = crossAppNodeToDbContent( + contentNodeToCrossApp(node), + "direct", + ); + return content === undefined ? [] : [content]; }); -}; -export const fetchEmbeddingsForNodes = async ( - nodes: LocalContentDataInput[], -): Promise => { +const fetchEmbeddingVectors = async (texts: string[]): Promise => { const allEmbeddings: number[][] = []; - const allNodesTexts = nodes.map((node) => node.text || ""); + const allNodesTexts = texts; for (let i = 0; i < allNodesTexts.length; i += EMBEDDING_BATCH_SIZE) { const batch = allNodesTexts.slice(i, i + EMBEDDING_BATCH_SIZE); @@ -83,91 +73,28 @@ export const fetchEmbeddingsForNodes = async ( const batchEmbeddings = data.data.map((item) => item.embedding); allEmbeddings.push(...batchEmbeddings); } - if (nodes.length !== allEmbeddings.length) { + if (texts.length !== allEmbeddings.length) { throw new Error( - `Mismatch between nodes (${nodes.length}) and embeddings (${allEmbeddings.length})`, + `Mismatch between nodes (${texts.length}) and embeddings (${allEmbeddings.length})`, ); } - return nodes.map((node, i) => ({ - ...node, - embedding_inline: { - model: EMBEDDING_MODEL, - vector: allEmbeddings[i], - }, - })); + return allEmbeddings; }; -const uploadBatches = async ( - batches: LocalContentDataInput[][], - supabaseClient: DGSupabaseClient, - context: SupabaseContext, -) => { - const { spaceId, userId } = context; - for (let idx = 0; idx < batches.length; idx++) { - const batch = batches[idx]; - const { error } = await supabaseClient.rpc("upsert_content", { - data: batch as unknown as Json, - v_space_id: spaceId, - v_creator_id: userId, - content_as_document: true, - }); - - if (error) { - console.error(`upsert_content failed for batch ${idx + 1}:`, error); - throw error; - } - } -}; - -export const upsertNodesToSupabaseAsContentWithEmbeddings = async ( - roamNodes: RoamDiscourseNodeData[], - supabaseClient: DGSupabaseClient, - context: SupabaseContext, +// Embeddings are computed for a node's direct content, and only by the sync: +// nothing else has a reason to pay for them. +export const attachEmbeddingsToNodes = async ( + nodes: CrossAppNode[], ): Promise => { - if (!context?.userId) { - console.error("No Supabase context found."); - return; - } - - if (roamNodes.length === 0) { - return; - } - const localContentNodes = convertRoamNodeToLocalContent({ - nodes: roamNodes, - }); - - let nodesWithEmbeddings: LocalContentDataInput[]; - try { - nodesWithEmbeddings = await fetchEmbeddingsForNodes(localContentNodes); - } catch (error: unknown) { - const errorMessage = error instanceof Error ? error.message : String(error); - console.error( - `upsertNodesToSupabaseAsContentWithEmbeddings: Embedding service failed – ${errorMessage}`, - ); - return; - } - - if (nodesWithEmbeddings.length !== roamNodes.length) { - console.error( - "upsertNodesToSupabaseAsContentWithEmbeddings: Mismatch between node and embedding counts.", - ); - return; - } - - const batchSize = 200; - - const chunk = (array: T[], size: number): T[][] => { - const chunks: T[][] = []; - for (let i = 0; i < array.length; i += size) { - chunks.push(array.slice(i, i + size)); - } - return chunks; - }; - - await uploadBatches( - chunk(nodesWithEmbeddings, batchSize), - supabaseClient, - context, + if (nodes.length === 0) return; + const vectors = await fetchEmbeddingVectors( + nodes.map((node) => node.content.direct.value), ); + nodes.forEach((node, i) => { + node.content.direct.embedding = { + value: vectors[i], + embedding: EMBEDDING_MODEL, + }; + }); }; diff --git a/packages/database/src/crossAppContracts.ts b/packages/database/src/crossAppContracts.ts index fe97fd6c2..616f5fafa 100644 --- a/packages/database/src/crossAppContracts.ts +++ b/packages/database/src/crossAppContracts.ts @@ -66,6 +66,10 @@ export type InlineCrossAppContent = Partial & { embedding?: CrossAppEmbedding; scale?: Enums<"Scale">; contentType?: ContentType; + // Which variant of the node's content this is. Defaults to the key it is held + // under: a node whose text lives in a block below its page holds a + // direct_and_description under direct, for instance. + variant?: Enums<"ContentVariant">; }; // An inline Content with obligatory typing diff --git a/packages/database/src/lib/crossAppConverters.ts b/packages/database/src/lib/crossAppConverters.ts index 1ff856f7f..2f96f0ab7 100644 --- a/packages/database/src/lib/crossAppConverters.ts +++ b/packages/database/src/lib/crossAppConverters.ts @@ -34,7 +34,7 @@ const filterUndefinedArray = (data: (T | undefined)[]): T[] => const inlineCrossAppContentToDbContent = ( content: InlineCrossAppContent | undefined, - variant: Enums<"ContentVariant">, + defaultVariant: Enums<"ContentVariant">, ): LocalContentDataInput | undefined => { if (content === undefined) return undefined; return filterUndefined({ @@ -42,7 +42,7 @@ const inlineCrossAppContentToDbContent = ( text: content.value, scale: content.scale || "document", content_type: content.contentType || "text/plain", - variant, + variant: content.variant || defaultVariant, created: content.createdAt?.toISOString(), last_modified: content.modifiedAt?.toISOString(), author_local_id: content.authorId, @@ -52,10 +52,10 @@ const inlineCrossAppContentToDbContent = ( export const crossAppNodeToDbContent = ( node: CrossAppNode | undefined, - variant: "full" | "direct", + defaultVariant: "full" | "direct", ): LocalContentDataInput | undefined => { if (node === undefined) return undefined; - const content = node.content[variant]; + const content = node.content[defaultVariant]; if (content === undefined) return undefined; return inlineCrossAppContentToDbContent( { @@ -64,7 +64,7 @@ export const crossAppNodeToDbContent = ( modifiedAt: content.modifiedAt || node.modifiedAt, authorId: content.authorId || node.authorId, }, - variant, + defaultVariant, ); };