ENG-2165 Refactor Roam Sync to use crossAppNodes

apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx

@@ -5,7 +5,7 @@ import getSubTree from "roamjs-components/util/getSubTree";
 import { DiscourseNode } from "~/utils/getDiscourseNodes";
 import extractRef from "roamjs-components/util/extractRef";
 import { getAllDiscourseNodesSince } from "~/utils/getAllDiscourseNodesSince";
-import { upsertNodesToSupabaseAsContentWithEmbeddings } from "~/utils/syncDgNodesToSupabase";
+import { upsertNodesWithEmbeddings } from "~/utils/syncDgNodesToSupabase";
 import { getLoggedInClient, getSupabaseContext } from "~/utils/supabaseContext";
 import {
   DiscourseNodeFlagPanel,
@@ -56,11 +56,12 @@ const DiscourseNodeSuggestiveRules = ({
 
       const context = await getSupabaseContext();
       if (context && blockNodesSince) {
-        await upsertNodesToSupabaseAsContentWithEmbeddings(
-          blockNodesSince,
+        await upsertNodesWithEmbeddings({
+          nodes: blockNodesSince,
+          nodeTypes: [node],
           supabaseClient,
           context,
-        );
+        });
       }
     } finally {
       setIsUpdating(false);

apps/roam/src/utils/__tests__/roamToCrossAppConverters.test.ts

@@ -9,7 +9,17 @@ vi.mock("roamjs-components/queries/getPageViewType", () => ({
 }));
 vi.mock("~/utils/pageToMarkdown", () => ({ toMarkdown: () => "" }));
 
-import { nodeUidsWithTypeToCrossApp } from "~/utils/roamToCrossAppConverters";
+import {
+  contentNodeToCrossApp,
+  nodeSchemaToCrossApp,
+  nodeUidsWithTypeToCrossApp,
+} from "~/utils/roamToCrossAppConverters";
+import {
+  crossAppNodeSchemaToDbConcept,
+  crossAppNodeToDbConcept,
+  crossAppNodeToDbContent,
+} from "@repo/database/lib/crossAppConverters";
+import type { DiscourseNode } from "~/utils/getDiscourseNodes";
 
 const USER_ROW = { ":db/id": 5, ":user/uid": "user-1" };
 
@@ -60,3 +70,134 @@ describe("nodeUidsWithTypeToCrossApp timestamps", () => {
     expect(node.modifiedAt).toEqual(new Date(1000));
   });
 });
+
+const NODE_ROW = {
+  author_local_id: "user-1",
+  source_local_id: "node-1",
+  created: 1000,
+  last_modified: 2000,
+  text: "EVD - some evidence",
+  type: "schema-1",
+};
+
+describe("contentNodeToCrossApp", () => {
+  it("names a page-backed node by its title, as a plain direct", () => {
+    const node = contentNodeToCrossApp(NODE_ROW);
+    expect(node.content.direct.value).toEqual("EVD - some evidence");
+    expect(node.content.direct.variant).toEqual("direct");
+    expect(node.content.full).toBeUndefined();
+    expect(node.nodeType).toEqual("schema-1");
+    expect(node.createdAt).toEqual(new Date(1000));
+    expect(node.modifiedAt).toEqual(new Date(2000));
+  });
+
+  it("names a block-backed node by its page title and its text", () => {
+    const node = contentNodeToCrossApp({
+      ...NODE_ROW,
+      node_title: "EVD - some evidence",
+      text: "the block text",
+    });
+    expect(node.content.direct.value).toEqual(
+      "EVD - some evidence the block text",
+    );
+    expect(node.content.direct.variant).toEqual("direct_and_description");
+  });
+
+  it("falls back to a plain direct when the first child block is missing", () => {
+    const node = contentNodeToCrossApp({
+      ...NODE_ROW,
+      node_title: "EVD - some evidence",
+      text: "",
+    });
+    expect(node.content.direct.value).toEqual("EVD - some evidence");
+    expect(node.content.direct.variant).toEqual("direct");
+  });
+
+  it("carries the variant of the direct slot into the db content", () => {
+    const node = contentNodeToCrossApp({
+      ...NODE_ROW,
+      node_title: "EVD - some evidence",
+      text: "the block text",
+    });
+    expect(crossAppNodeToDbContent(node, "direct")).toMatchObject({
+      source_local_id: "node-1",
+      text: "EVD - some evidence the block text",
+      variant: "direct_and_description",
+      scale: "document",
+      author_local_id: "user-1",
+    });
+  });
+
+  it("carries the node's direct content into the db concept", () => {
+    const concept = crossAppNodeToDbConcept(contentNodeToCrossApp(NODE_ROW));
+    expect(concept.contents_inline).toEqual([
+      expect.objectContaining({
+        source_local_id: "node-1",
+        text: "EVD - some evidence",
+        variant: "direct",
+      }),
+    ]);
+    expect(concept).toMatchObject({
+      source_local_id: "node-1",
+      name: "EVD - some evidence",
+      author_local_id: "user-1",
+      schema_represented_by_local_id: "schema-1",
+    });
+  });
+});
+
+const schemaPull = () => ({
+  ":create/time": 1000,
+  ":edit/time": 2000,
+  ":create/user": { ":user/uid": "user-1" },
+});
+
+const convertSchema = (node: DiscourseNode) => {
+  (globalThis as { window: unknown }).window = {
+    roamAlphaAPI: { pull: schemaPull },
+  };
+  return nodeSchemaToCrossApp(node);
+};
+
+const schemaNode = (overrides: Partial<DiscourseNode>): DiscourseNode => ({
+  text: "Evidence",
+  type: "_EVD-node",
+  shortcut: "e",
+  format: "[[EVD]] - {content}",
+  specification: [],
+  backedBy: "user",
+  canvasSettings: {},
+  ...overrides,
+});
+
+describe("nodeSchemaToCrossApp", () => {
+  it("carries the modification time, so that the concept can be inserted", () => {
+    const schema = convertSchema(schemaNode({}));
+    expect(schema?.createdAt).toEqual(new Date(1000));
+    expect(schema?.modifiedAt).toEqual(new Date(2000));
+  });
+
+  it("renders the template as text, under template_content in the db concept", () => {
+    const schema = convertSchema(
+      schemaNode({
+        template: [
+          { text: "Question:", children: [{ text: "why?" }] },
+          { text: "{{roam/render}}" },
+        ],
+      }),
+    );
+    expect(schema?.template).toEqual("* Question:\n   * why?\n   \n");
+    const concept = crossAppNodeSchemaToDbConcept(schema!);
+    expect(concept.literal_content).toEqual({
+      template_content: "* Question:\n   * why?\n   \n",
+    });
+  });
+
+  it("omits the template when the node type has none", () => {
+    const schema = convertSchema(schemaNode({ template: [] }));
+    expect(schema?.template).toBeUndefined();
+    expect(
+      crossAppNodeSchemaToDbConcept(schema!).literal_content,
+    ).toBeUndefined();
+  });
+});

apps/roam/src/utils/conceptConversion.ts

@@ -1,6 +1,4 @@
-import { InputTextNode } from "roamjs-components/types";
 import getBlockProps from "./getBlockProps";
-import { DiscourseNode } from "./getDiscourseNodes";
 import getDiscourseRelations from "./getDiscourseRelations";
 import type { DiscourseRelation } from "./getDiscourseRelations";
 import type { SupabaseContext } from "~/utils/supabaseContext";
@@ -58,69 +56,6 @@ const getNodeExtraData = (
   /* eslint-enable @typescript-eslint/naming-convention */
 };
 
-const indent = (s: string): string =>
-  s
-    .split("\n")
-    .map((l) => "   " + l)
-    .join("\n") + "\n";
-
-const templateToText = (template: InputTextNode[]): string =>
-  template
-    .filter((itn) => !itn.text.startsWith("{{"))
-    .map(
-      (itn) =>
-        `* ${itn.text}\n${itn.children?.length ? indent(templateToText(itn.children)) : ""}`,
-    )
-    .join("");
-
-export const discourseNodeSchemaToLocalConcept = (
-  context: SupabaseContext,
-  node: DiscourseNode,
-): LocalConceptDataInput => {
-  const titleParts = node.text.split("/");
-  const label = titleParts[titleParts.length - 1] ?? node.text;
-  const result: LocalConceptDataInput = {
-    space_id: context.spaceId,
-    name: node.text,
-    source_local_id: node.type,
-    is_schema: true,
-    literal_content: {
-      label,
-    },
-    /* eslint-enable @typescript-eslint/naming-convention */
-    ...getNodeExtraData(node.type),
-  };
-  if (node.template !== undefined)
-    result.literal_content = {
-      label,
-      template: templateToText(node.template),
-    };
-  return result;
-};
-
-export const discourseNodeBlockToLocalConcept = (
-  context: SupabaseContext,
-  {
-    nodeUid,
-    schemaUid,
-    text,
-  }: {
-    nodeUid: string;
-    schemaUid: string;
-    text: string;
-  },
-): LocalConceptDataInput => {
-  return {
-    space_id: context.spaceId,
-    name: text,
-    source_local_id: nodeUid,
-    schema_represented_by_local_id: schemaUid,
-    is_schema: false,
-    /* eslint-enable @typescript-eslint/naming-convention */
-    ...getNodeExtraData(nodeUid),
-  };
-};
-
 const STANDARD_ROLES = ["source", "destination"];
 
 export const discourseRelationSchemaToLocalConcept = (

apps/roam/src/utils/getAllDiscourseNodesSince.ts

@@ -6,10 +6,11 @@ const DEFAULT_TIME = new Date("1970-01-01").getTime();
 
 export type RoamDiscourseNodeData = {
   author_local_id: string;
-  author_name: string;
+  author_name?: string;
   source_local_id: string;
-  created: string;
-  last_modified: string;
+  // Roam returns :create/time and :edit/time as epoch milliseconds
+  created: number;
+  last_modified: number;
   text: string;
   type: string;
   node_title?: string;