diff --git a/helm/charts/synchronizer/templates/configmap.yaml b/helm/charts/synchronizer/templates/configmap.yaml index 66a4cc0..158ddc4 100644 --- a/helm/charts/synchronizer/templates/configmap.yaml +++ b/helm/charts/synchronizer/templates/configmap.yaml @@ -32,8 +32,6 @@ data: INDEX_NAME_FORMAT: {{ .Values.env.indexNameFormat | quote }} - LAYERS_FILE_PATH: {{ .Values.env.layersFile | quote }} - ALIASES_FILE_PATH: {{ .Values.env.aliasesFile | quote }} {{- if .Values.typeMap.enabled }} @@ -47,21 +45,8 @@ data: S3_SSL_ENABLED: {{ .sslEnabled | quote }} S3_SIG_VERSION: {{ .sigVersion | quote }} S3_STORAGE_CLASS: {{ .storageClass | quote }} - S3_BUCKET: {{ .bucket | quote }} - S3_FILE_NAME: {{ .fileName | quote }} - S3_LAYERS_VARIABLE: {{ .layersVariable | quote }} {{- end }} - {{- with .Values.env.enrichment }} - ENRICHMENT_ENABLED: {{ .enabled | quote }} - {{- if .enabled }} - ENRICHMENT_API_URL: {{ .api | quote }} - ENRICHMENT_PROPERTIES_PATH: {{ .propertiesPath | quote }} - ENRICHMENT_ALIAS_FIELD: {{ .aliasField | quote }} - ENRICHMENT_REQUEST_TIMEOUT_MILLISECONDS: {{ .requestTimeoutMilliseconds | quote }} - {{- end }} - {{- end }} - {{- include "synchronizer.dbEnvBlock" (dict "prefix" "SOURCE_DB" "db" .Values.dbs.sourceDb) | nindent 2 }} {{- include "synchronizer.dbEnvBlock" (dict "prefix" "DEST_DB" "db" .Values.dbs.destinationDb) | nindent 2 }} diff --git a/helm/charts/synchronizer/templates/deployment.yaml b/helm/charts/synchronizer/templates/deployment.yaml index c0a319f..f5803b3 100644 --- a/helm/charts/synchronizer/templates/deployment.yaml +++ b/helm/charts/synchronizer/templates/deployment.yaml @@ -93,7 +93,10 @@ spec: {{- end }} volumeMounts: - name: layers-config - mountPath: {{ .Values.env.layersFile | dir | quote }} + mountPath: /usr/src/app/packages/synchronizer/dist/config/layers + - name: namespaces-config + mountPath: /usr/src/app/packages/synchronizer/dist/config/production.json + subPath: production.json - name: aliases-config mountPath: {{ .Values.env.aliasesFile | dir | quote }} {{- if .Values.typeMap.enabled }} @@ -102,11 +105,13 @@ spec: {{- end }} - name: s3-downloads mountPath: /usr/src/app/packages/synchronizer/dist/src/common/s3/downloads - {{- if .Values.dbs.sourceDb.sslAuth.enabled }} - - name: source-db-cert-conf - mountPath: /tmp/certs/source + {{- range .Values.namespaces }} + {{- if .db.ssl.enabled }} + - name: source-db-cert-{{ .name }} + mountPath: {{ printf "/tmp/certs/source-%s" .name | quote }} readOnly: true {{- end }} + {{- end }} {{- if .Values.dbs.destinationDb.sslAuth.enabled }} - name: dest-db-cert-conf mountPath: /tmp/certs/dest @@ -121,6 +126,9 @@ spec: - name: layers-config configMap: name: {{ include "synchronizer.fullname" . }}-layers + - name: namespaces-config + secret: + secretName: {{ include "synchronizer.fullname" . }}-namespaces - name: aliases-config configMap: name: {{ include "synchronizer.fullname" . }}-aliases @@ -131,16 +139,13 @@ spec: {{- end }} - name: s3-downloads emptyDir: {} - {{- if .Values.dbs.sourceDb.sslAuth.enabled }} - - name: source-db-cert-conf + {{- range .Values.namespaces }} + {{- if .db.ssl.enabled }} + - name: source-db-cert-{{ .name }} secret: - secretName: {{ .Values.dbs.sourceDb.sslAuth.secretName }} + secretName: {{ .db.sslSecretName }} + {{- end }} {{- end }} - {{- if .Values.caSecretName }} - - mountPath: {{ printf "%s/%s" .Values.caPath .Values.caKey | quote }} - name: root-ca - subPath: {{ quote .Values.caKey }} - {{- end }} {{- if .Values.dbs.destinationDb.sslAuth.enabled }} - name: dest-db-cert-conf secret: diff --git a/helm/charts/synchronizer/templates/layers-configmap.yaml b/helm/charts/synchronizer/templates/layers-configmap.yaml index 407ae25..5d7aef5 100644 --- a/helm/charts/synchronizer/templates/layers-configmap.yaml +++ b/helm/charts/synchronizer/templates/layers-configmap.yaml @@ -6,5 +6,7 @@ metadata: labels: {{- include "synchronizer.labels" . | nindent 4 }} data: - layers.json: {{ .Values.layers | toPrettyJson | quote }} + {{- range .Values.namespaces }} + {{ .name }}.json: {{ .layers | toPrettyJson | quote }} + {{- end }} {{- end }} diff --git a/helm/charts/synchronizer/templates/namespaces-secret.yaml b/helm/charts/synchronizer/templates/namespaces-secret.yaml new file mode 100644 index 0000000..53b6484 --- /dev/null +++ b/helm/charts/synchronizer/templates/namespaces-secret.yaml @@ -0,0 +1,11 @@ +{{- if .Values.enabled -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "synchronizer.fullname" . }}-namespaces + labels: + {{- include "synchronizer.labels" . | nindent 4 }} +type: Opaque +stringData: + production.json: {{ dict "namespaces" .Values.namespaces | toPrettyJson | quote }} +{{- end }} diff --git a/helm/charts/synchronizer/templates/secret.yaml b/helm/charts/synchronizer/templates/secret.yaml index 96277b1..12c441f 100644 --- a/helm/charts/synchronizer/templates/secret.yaml +++ b/helm/charts/synchronizer/templates/secret.yaml @@ -7,7 +7,6 @@ metadata: {{- include "synchronizer.labels" . | nindent 4 }} type: Opaque data: - SOURCE_DB_PASSWORD: {{ .Values.dbs.sourceDb.password | b64enc | quote }} DEST_DB_PASSWORD: {{ .Values.dbs.destinationDb.password | b64enc | quote }} S3_ACCESS_KEY: {{ .Values.dbs.s3.accessKey | b64enc | quote }} S3_SECRET_KEY: {{ .Values.dbs.s3.secretKey | b64enc | quote }} diff --git a/helm/charts/synchronizer/values.yaml b/helm/charts/synchronizer/values.yaml index a9912ee..d5604f5 100644 --- a/helm/charts/synchronizer/values.yaml +++ b/helm/charts/synchronizer/values.yaml @@ -23,7 +23,6 @@ startupProbe: enabled: true periodSeconds: 10 timeoutSeconds: 5 - # failureThreshold * periodSeconds = max startup wait (30 * 10s = 300s) failureThreshold: 30 livenessProbe: @@ -41,11 +40,41 @@ readinessProbe: failureThreshold: 6 path: /liveness -layers: - - layerName: buildings_polygon - enums: [code, attribute, sensitivity, building_type, status] - - layerName: borders_polygon - enums: [code, attribute] +namespaces: + - name: default + bucket: "" + layers: + - layerName: buildings_polygon + enums: [code, attribute, sensitivity, building_type, status] + - layerName: borders_polygon + enums: [code, attribute] + db: + host: localhost + port: 5432 + username: postgres + password: postgres + database: postgres + schema: public + type: postgres + sslSecretName: secret-name + ssl: + enabled: false + ca: "" + cert: "" + key: "" + layerSource: + type: sharedLua + fileName: mock-query.lua + layersVariable: layers + aliasFieldName: alias + idFieldName: id + nameFieldName: name + enrichment: + enabled: false + api: "" + propertiesPath: "" + aliasField: "" + requestTimeoutMilliseconds: 5000 aliases: {} @@ -70,15 +99,8 @@ env: url: http://localhost:55681/v1/metrics schedule: '0 0 * * *' indexNameFormat: '{layerName}_{column}_idx' - layersFile: /usr/src/app/packages/synchronizer/dist/config/layers/layers.json aliasesFile: /usr/src/app/packages/synchronizer/dist/config/aliases/aliases.json typeMapFile: /usr/src/app/packages/synchronizer/dist/config/typeMap/typeMap.json - enrichment: - enabled: false - api: "" - propertiesPath: "" - aliasField: "" - requestTimeoutMilliseconds: 5000 dbs: s3: @@ -88,25 +110,8 @@ dbs: sslEnabled: false sigVersion: v4 storageClass: STANDARD - bucket: "" - fileName: "" - layersVariable: layers accessKey: "" secretKey: "" - sourceDb: - host: localhost - port: 5432 - username: postgres - password: postgres - database: postgres - schema: public - type: postgres - sslAuth: - enabled: false - secretName: secret-name - cert: "" - key: "" - root: "" destinationDb: host: localhost port: 5432 diff --git a/helm/values.yaml b/helm/values.yaml index 51b1cad..0bb40ba 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -163,13 +163,44 @@ synchronizer: pullPolicy: Always tag: "v1.2.0" - layers: - - layerName: buildings_polygon - enums: [ code, attribute ] + namespaces: + - name: default + bucket: "" + layers: + - layerName: buildings_polygon + enums: [code, attribute] + db: + host: localhost + port: 5432 + username: postgres + password: postgres + database: postgres + schema: public + type: postgres + sslSecretName: secret-name + ssl: + enabled: false + ca: "" + cert: "" + key: "" + layerSource: + type: sharedLua + fileName: "" + layersVariable: layers + aliasFieldName: alias + idFieldName: id + nameFieldName: name + enrichment: + enabled: false + api: "" + propertiesPath: "" + aliasField: "" + requestTimeoutMilliseconds: 5000 aliases: "*": - num_floors: "number_of_floors" + "*": + num_floors: "number_of_floors" typeMap: enabled: false @@ -192,15 +223,8 @@ synchronizer: url: http://localhost:55681/v1/metrics schedule: '*/1 * * * *' indexNameFormat: '{layerName}_{column}_idx' - layersFile: /usr/src/app/packages/synchronizer/dist/config/layers/layers.json aliasesFile: /usr/src/app/packages/synchronizer/dist/config/aliases/aliases.json - typeMapFile: /usr/src/app/packages/synchronizer/dist/config/typeMap.json - enrichment: - enabled: false - api: "" - propertiesPath: "" - aliasField: "" - requestTimeoutMilliseconds: 5000 + typeMapFile: /usr/src/app/packages/synchronizer/dist/config/typeMap/typeMap.json dbs: s3: @@ -210,24 +234,8 @@ synchronizer: sslEnabled: false sigVersion: v4 storageClass: STANDARD - bucket: "" - fileName: "" - layersVariable: layers accessKey: "" secretKey: "" - sourceDb: - host: localhost - port: 5432 - username: postgres - password: postgres - database: postgres - schema: public - type: postgres - sslAuth: - enabled: false - cert: '' - key: '' - root: '' destinationDb: *dbConfig resources: diff --git a/openapi3.yaml b/openapi3.yaml index f38a4b6..8889fb7 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -4,27 +4,38 @@ info: description: > API for retrieving geographic layer specifications. Layers expose a set of properties; `enum`-typed properties also carry their list of allowed values. - version: 1.2.0 + version: 2.0.0 license: name: Proprietary url: https://example.com/licenses/proprietary paths: - /layers: + /namespaces: get: - summary: Get all layer names + summary: Get all namespaces + operationId: getNamespaces + security: [] + responses: + '200': + $ref: '#/components/responses/NamespaceListResponse' + /namespaces/{namespace}/layers: + get: + summary: Get all layer names within a namespace operationId: getLayers security: [] + parameters: + - $ref: '#/components/parameters/NamespaceParam' responses: '200': $ref: '#/components/responses/LayerListResponse' '404': $ref: '#/components/responses/NotFound' - /layers/{layerName}: + /namespaces/{namespace}/layers/{layerName}: get: - summary: Get layer spec by name + summary: Get layer spec by name within a namespace operationId: getLayerByName security: [] parameters: + - $ref: '#/components/parameters/NamespaceParam' - $ref: '#/components/parameters/LayerNameParam' responses: '200': @@ -33,6 +44,14 @@ paths: $ref: '#/components/responses/NotFound' components: parameters: + NamespaceParam: + name: namespace + in: path + required: true + description: The namespace the layer belongs to. Layer names are only unique within a namespace + schema: + type: string + example: data-2025 LayerNameParam: name: layerName in: path @@ -42,6 +61,16 @@ components: type: string example: buildings_polygon responses: + NamespaceListResponse: + description: A list of the namespaces this deployment serves + content: + application/json: + schema: + $ref: '#/components/schemas/NamespaceList' + example: + namespaces: + - name: data-2024 + - name: data-2025 LayerListResponse: description: A list of layers with their display aliases content: @@ -83,6 +112,26 @@ components: schema: $ref: '#/components/schemas/Error' schemas: + NamespaceSummary: + type: object + description: A namespace served by this deployment. + properties: + name: + type: string + description: Name of the namespace + example: data-2025 + required: + - name + NamespaceList: + type: object + description: Response wrapper for the list of namespaces. + properties: + namespaces: + type: array + items: + $ref: '#/components/schemas/NamespaceSummary' + required: + - namespaces LayerSummary: type: object description: Basic layer info returned in the list endpoint. diff --git a/package-lock.json b/package-lock.json index 95a8e82..f946a30 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "packages/*" ], "dependencies": { - "@map-colonies/schemas": "1.26.0" + "@map-colonies/schemas": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-11bf4605d7a17c723dd85f624c88fe32d9957863.tgz" }, "devDependencies": { "@commitlint/cli": "^20.4.1", @@ -345,6 +345,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=6.9.0" } @@ -383,6 +384,7 @@ "version": "6.3.1", "dev": true, "license": "ISC", + "peer": true, "bin": { "semver": "bin/semver.js" } @@ -391,6 +393,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/parser": "^7.29.7", "@babel/types": "^7.29.7", @@ -406,6 +409,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/compat-data": "^7.29.7", "@babel/helper-validator-option": "^7.29.7", @@ -421,6 +425,7 @@ "version": "6.3.1", "dev": true, "license": "ISC", + "peer": true, "bin": { "semver": "bin/semver.js" } @@ -429,6 +434,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=6.9.0" } @@ -437,6 +443,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/traverse": "^7.29.7", "@babel/types": "^7.29.7" @@ -449,6 +456,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-module-imports": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7", @@ -465,6 +473,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=6.9.0" } @@ -488,6 +497,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=6.9.0" } @@ -496,6 +506,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/template": "^7.29.7", "@babel/types": "^7.29.7" @@ -522,6 +533,7 @@ "version": "7.8.4", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, @@ -533,6 +545,7 @@ "version": "7.8.3", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, @@ -544,6 +557,7 @@ "version": "7.12.13", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.12.13" }, @@ -555,6 +569,7 @@ "version": "7.14.5", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.14.5" }, @@ -569,6 +584,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.29.7" }, @@ -583,6 +599,7 @@ "version": "7.10.4", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.10.4" }, @@ -594,6 +611,7 @@ "version": "7.8.3", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, @@ -605,6 +623,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.29.7" }, @@ -619,6 +638,7 @@ "version": "7.10.4", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.10.4" }, @@ -630,6 +650,7 @@ "version": "7.8.3", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, @@ -641,6 +662,7 @@ "version": "7.10.4", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.10.4" }, @@ -652,6 +674,7 @@ "version": "7.8.3", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, @@ -663,6 +686,7 @@ "version": "7.8.3", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, @@ -674,6 +698,7 @@ "version": "7.8.3", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.8.0" }, @@ -685,6 +710,7 @@ "version": "7.14.5", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.14.5" }, @@ -699,6 +725,7 @@ "version": "7.14.5", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.14.5" }, @@ -713,6 +740,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/helper-plugin-utils": "^7.29.7" }, @@ -734,6 +762,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/parser": "^7.29.7", @@ -747,6 +776,7 @@ "version": "7.29.7", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/generator": "^7.29.7", @@ -1129,6 +1159,43 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@emnapi/core": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.3.tgz", + "integrity": "sha512-zLpS5asjEb7lq8jYLq37N6XKaE41DIexlY1rF/z4/tIl3wo13Sqm28fRyfIsKZD+NZ8mM5RoKkpW/rBcuoSZSg==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.3", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", + "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.3.tgz", + "integrity": "sha512-ELEBe8PsLvvJ6QMr0zLt8ffvOHW/dc1m3CEzNMg7aJUv3bMaoDtw2TXyDAwkYBuroxxuHEwhRTLJSe5sya547g==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@epic-web/invariant": { "version": "1.0.0", "dev": true, @@ -1542,6 +1609,7 @@ "version": "1.1.0", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "camelcase": "^5.3.1", "find-up": "^4.1.0", @@ -1557,6 +1625,7 @@ "version": "1.0.10", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "sprintf-js": "~1.0.2" } @@ -1565,6 +1634,7 @@ "version": "4.1.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" @@ -1577,6 +1647,7 @@ "version": "3.15.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" @@ -1589,6 +1660,7 @@ "version": "5.0.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "p-locate": "^4.1.0" }, @@ -1600,6 +1672,7 @@ "version": "2.3.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "p-try": "^2.0.0" }, @@ -1614,6 +1687,7 @@ "version": "4.1.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "p-limit": "^2.2.0" }, @@ -1625,6 +1699,7 @@ "version": "0.1.6", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=8" } @@ -1633,6 +1708,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/types": "30.4.1", "@types/node": "*", @@ -1649,6 +1725,7 @@ "version": "30.4.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/console": "30.4.1", "@jest/pattern": "30.4.0", @@ -1695,6 +1772,7 @@ "version": "30.4.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } @@ -1703,6 +1781,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/fake-timers": "30.4.1", "@jest/types": "30.4.1", @@ -1717,6 +1796,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "expect": "30.4.1", "jest-snapshot": "30.4.1" @@ -1729,6 +1809,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/get-type": "30.1.0" }, @@ -1740,6 +1821,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/types": "30.4.1", "@sinonjs/fake-timers": "^15.4.0", @@ -1756,6 +1838,7 @@ "version": "30.1.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } @@ -1764,6 +1847,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/environment": "30.4.1", "@jest/expect": "30.4.1", @@ -1778,6 +1862,7 @@ "version": "30.4.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@types/node": "*", "jest-regex-util": "30.4.0" @@ -1790,6 +1875,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@bcoe/v8-coverage": "^0.2.3", "@jest/console": "30.4.1", @@ -1830,17 +1916,20 @@ "node_modules/@jest/reporters/node_modules/@bcoe/v8-coverage": { "version": "0.2.3", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@jest/reporters/node_modules/balanced-match": { "version": "1.0.2", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@jest/reporters/node_modules/brace-expansion": { "version": "2.1.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "balanced-match": "^1.0.0" } @@ -1849,6 +1938,7 @@ "version": "10.5.0", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", @@ -1868,6 +1958,7 @@ "version": "9.0.9", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "brace-expansion": "^2.0.2" }, @@ -1882,6 +1973,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@sinclair/typebox": "^0.34.0" }, @@ -1893,6 +1985,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/types": "30.4.1", "chalk": "^4.1.2", @@ -1907,6 +2000,7 @@ "version": "30.0.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jridgewell/trace-mapping": "^0.3.25", "callsites": "^3.1.0", @@ -1920,6 +2014,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/console": "30.4.1", "@jest/types": "30.4.1", @@ -1934,6 +2029,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/test-result": "30.4.1", "graceful-fs": "^4.2.11", @@ -1948,6 +2044,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/core": "^7.27.4", "@jest/types": "30.4.1", @@ -1972,6 +2069,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/pattern": "30.4.0", "@jest/schemas": "30.4.1", @@ -1989,6 +2087,7 @@ "version": "0.3.13", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" @@ -1998,6 +2097,7 @@ "version": "2.3.5", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" @@ -3232,10 +3332,9 @@ }, "node_modules/@map-colonies/schemas": { "version": "1.26.0", - "resolved": "https://registry.npmjs.org/@map-colonies/schemas/-/schemas-1.26.0.tgz", - "integrity": "sha512-XUQ2/gvC6d6/nGMKIGVyZo/kemGH5uGOy4xhkYCVLNV/CJsHvJXmhiov7QpqVpb58x+/2WVVUThAStj9ogipug==", - "license": "MIT", - "peer": true + "resolved": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-11bf4605d7a17c723dd85f624c88fe32d9957863.tgz", + "integrity": "sha512-jGavRY1lGItcyyNlaO/Sa4hpV1TZ15NXH2vFbcZnNlNmHtFtU3ZnGqLpRZtSOxAbBnQY7xga4LRGZpZea+c0VQ==", + "license": "MIT" }, "node_modules/@map-colonies/tracing": { "version": "1.0.0", @@ -4115,7 +4214,6 @@ "version": "4.2.4", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@octokit/auth-token": "^3.0.0", "@octokit/graphql": "^5.0.0", @@ -4301,7 +4399,6 @@ "node_modules/@opentelemetry/api": { "version": "1.9.1", "license": "Apache-2.0", - "peer": true, "engines": { "node": ">=8.0.0" } @@ -4391,7 +4488,6 @@ "node_modules/@opentelemetry/core": { "version": "2.9.0", "license": "Apache-2.0", - "peer": true, "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, @@ -6285,6 +6381,7 @@ "node_modules/@redocly/ajv": { "version": "8.11.2", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "json-schema-traverse": "^1.0.0", @@ -6311,11 +6408,13 @@ }, "node_modules/@redocly/config": { "version": "0.22.0", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@redocly/openapi-core": { "version": "1.34.17", "license": "MIT", + "peer": true, "dependencies": { "@redocly/ajv": "8.11.2", "@redocly/config": "0.22.0", @@ -6335,17 +6434,20 @@ "node_modules/@redocly/openapi-core/node_modules/agent-base": { "version": "7.1.4", "license": "MIT", + "peer": true, "engines": { "node": ">= 14" } }, "node_modules/@redocly/openapi-core/node_modules/balanced-match": { "version": "1.0.2", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@redocly/openapi-core/node_modules/brace-expansion": { "version": "2.1.2", "license": "MIT", + "peer": true, "dependencies": { "balanced-match": "^1.0.0" } @@ -6353,6 +6455,7 @@ "node_modules/@redocly/openapi-core/node_modules/https-proxy-agent": { "version": "7.0.6", "license": "MIT", + "peer": true, "dependencies": { "agent-base": "^7.1.2", "debug": "4" @@ -6374,6 +6477,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "argparse": "^2.0.1" }, @@ -6384,6 +6488,7 @@ "node_modules/@redocly/openapi-core/node_modules/minimatch": { "version": "5.1.9", "license": "ISC", + "peer": true, "dependencies": { "brace-expansion": "^2.0.1" }, @@ -6587,12 +6692,14 @@ "node_modules/@sinclair/typebox": { "version": "0.34.52", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@sinonjs/commons": { "version": "3.0.1", "dev": true, "license": "BSD-3-Clause", + "peer": true, "dependencies": { "type-detect": "4.0.8" } @@ -6601,6 +6708,7 @@ "version": "15.4.0", "dev": true, "license": "BSD-3-Clause", + "peer": true, "dependencies": { "@sinonjs/commons": "^3.0.1" } @@ -6775,6 +6883,7 @@ "version": "7.20.5", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/parser": "^7.20.7", "@babel/types": "^7.20.7", @@ -6787,6 +6896,7 @@ "version": "7.27.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/types": "^7.0.0" } @@ -6795,6 +6905,7 @@ "version": "7.4.4", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/parser": "^7.1.0", "@babel/types": "^7.0.0" @@ -6804,6 +6915,7 @@ "version": "7.28.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/types": "^7.28.2" } @@ -6871,7 +6983,6 @@ "node_modules/@types/express": { "version": "4.17.25", "license": "MIT", - "peer": true, "dependencies": { "@types/body-parser": "*", "@types/express-serve-static-core": "^4.17.33", @@ -6916,8 +7027,7 @@ }, "node_modules/@types/json-schema": { "version": "7.0.15", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@types/lodash": { "version": "4.17.24", @@ -6973,10 +7083,12 @@ } }, "node_modules/@types/node": { - "version": "26.1.1", + "version": "24.13.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", + "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", "license": "MIT", "dependencies": { - "undici-types": "~8.3.0" + "undici-types": "~7.18.0" } }, "node_modules/@types/node-cron": { @@ -7098,7 +7210,8 @@ "node_modules/@types/stack-utils": { "version": "2.0.3", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/@types/superagent": { "version": "8.1.10", @@ -7145,6 +7258,7 @@ "version": "17.0.35", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@types/yargs-parser": "*" } @@ -7339,7 +7453,6 @@ "version": "8.63.0", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", "@typescript-eslint/scope-manager": "8.63.0", @@ -7388,7 +7501,8 @@ "node_modules/@ungap/structured-clone": { "version": "1.3.3", "dev": true, - "license": "ISC" + "license": "ISC", + "peer": true }, "node_modules/@unrs/resolver-binding-android-arm-eabi": { "version": "1.12.2", @@ -7657,6 +7771,40 @@ "node": ">=14.0.0" } }, + "node_modules/@unrs/resolver-binding-wasm32-wasi/node_modules/@emnapi/core": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", + "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.1", + "tslib": "^2.4.0" + } + }, + "node_modules/@unrs/resolver-binding-wasm32-wasi/node_modules/@emnapi/runtime": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", + "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@unrs/resolver-binding-wasm32-wasi/node_modules/@emnapi/wasi-threads": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", + "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@unrs/resolver-binding-win32-arm64-msvc": { "version": "1.12.2", "resolved": "https://registry.npmjs.org/@unrs/resolver-binding-win32-arm64-msvc/-/resolver-binding-win32-arm64-msvc-1.12.2.tgz", @@ -7732,7 +7880,6 @@ "version": "1.6.23", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "^8.58.0", "@typescript-eslint/utils": "^8.58.0" @@ -7954,7 +8101,6 @@ "node_modules/acorn": { "version": "8.17.0", "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -8036,8 +8182,9 @@ }, "node_modules/ajv": { "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -8312,6 +8459,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/transform": "30.4.1", "@types/babel__core": "^7.20.5", @@ -8332,6 +8480,7 @@ "version": "7.0.1", "dev": true, "license": "BSD-3-Clause", + "peer": true, "workspaces": [ "test/babel-8" ], @@ -8350,6 +8499,7 @@ "version": "30.4.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@types/babel__core": "^7.20.5" }, @@ -8361,6 +8511,7 @@ "version": "1.2.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/plugin-syntax-async-generators": "^7.8.4", "@babel/plugin-syntax-bigint": "^7.8.3", @@ -8386,6 +8537,7 @@ "version": "30.4.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "babel-plugin-jest-hoist": "30.4.0", "babel-preset-current-node-syntax": "^1.2.0" @@ -8578,7 +8730,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.10.42", "caniuse-lite": "^1.0.30001803", @@ -8608,6 +8759,7 @@ "version": "2.1.1", "dev": true, "license": "Apache-2.0", + "peer": true, "dependencies": { "node-int64": "^0.4.0" } @@ -8896,6 +9048,7 @@ "version": "1.0.2", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=10" } @@ -9084,6 +9237,7 @@ "version": "4.6.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "iojs": ">= 1.0.0", "node": ">= 0.12.0" @@ -9092,7 +9246,8 @@ "node_modules/collect-v8-coverage": { "version": "1.0.3", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/color-convert": { "version": "2.0.1", @@ -9118,7 +9273,8 @@ }, "node_modules/colorette": { "version": "1.4.0", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/columnify": { "version": "1.6.0", @@ -9150,7 +9306,6 @@ "node_modules/commander": { "version": "14.0.3", "license": "MIT", - "peer": true, "engines": { "node": ">=20" } @@ -10270,7 +10425,6 @@ "version": "9.0.2", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "env-paths": "^2.2.1", "import-fresh": "^3.3.0", @@ -10608,6 +10762,7 @@ "version": "3.1.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=8" } @@ -10717,6 +10872,7 @@ "version": "0.13.1", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -10735,6 +10891,29 @@ "node": ">= 0.8" } }, + "node_modules/encoding": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", + "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", + "license": "MIT", + "optional": true, + "dependencies": { + "iconv-lite": "^0.6.2" + } + }, + "node_modules/encoding/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "optional": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/end-of-stream": { "version": "1.4.5", "license": "MIT", @@ -10866,7 +11045,6 @@ "version": "9.39.5", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -10995,7 +11173,6 @@ "version": "4.17.1", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/types": "^8.56.0", "comment-parser": "^1.4.1", @@ -11042,7 +11219,6 @@ "version": "28.14.0", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/utils": "^6.0.0 || ^7.0.0 || ^8.0.0" }, @@ -11333,6 +11509,7 @@ "version": "0.2.2", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">= 0.8.0" } @@ -11341,6 +11518,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/expect-utils": "30.4.1", "@jest/get-type": "30.1.0", @@ -11369,7 +11547,6 @@ "node_modules/express": { "version": "5.2.1", "license": "MIT", - "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -11830,6 +12007,7 @@ "version": "2.0.2", "dev": true, "license": "Apache-2.0", + "peer": true, "dependencies": { "bser": "2.1.1" } @@ -12251,6 +12429,7 @@ "version": "1.0.0-beta.2", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=6.9.0" } @@ -12298,6 +12477,7 @@ "version": "0.1.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=8.0.0" } @@ -12771,7 +12951,6 @@ "version": "14.0.0", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=18" }, @@ -13185,6 +13364,7 @@ "node_modules/index-to-position": { "version": "1.2.0", "license": "MIT", + "peer": true, "engines": { "node": ">=18" }, @@ -13494,6 +13674,7 @@ "version": "2.1.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=6" } @@ -13680,6 +13861,7 @@ "version": "6.0.3", "dev": true, "license": "BSD-3-Clause", + "peer": true, "dependencies": { "@babel/core": "^7.23.9", "@babel/parser": "^7.23.9", @@ -13708,6 +13890,7 @@ "version": "5.0.6", "dev": true, "license": "BSD-3-Clause", + "peer": true, "dependencies": { "@jridgewell/trace-mapping": "^0.3.23", "debug": "^4.1.1", @@ -13788,6 +13971,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "execa": "^5.1.1", "jest-util": "30.4.1", @@ -13801,6 +13985,7 @@ "version": "30.4.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/environment": "30.4.1", "@jest/expect": "30.4.1", @@ -13831,6 +14016,7 @@ "version": "1.7.2", "dev": true, "license": "MIT", + "peer": true, "peerDependencies": { "babel-plugin-macros": "^3.1.0" }, @@ -13844,6 +14030,7 @@ "version": "30.4.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/core": "30.4.2", "@jest/test-result": "30.4.1", @@ -13875,6 +14062,7 @@ "version": "30.4.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/core": "^7.27.4", "@jest/get-type": "30.1.0", @@ -13923,12 +14111,14 @@ "node_modules/jest-config/node_modules/balanced-match": { "version": "1.0.2", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/jest-config/node_modules/brace-expansion": { "version": "2.1.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "balanced-match": "^1.0.0" } @@ -13937,6 +14127,7 @@ "version": "10.5.0", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", @@ -13956,6 +14147,7 @@ "version": "9.0.9", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "brace-expansion": "^2.0.2" }, @@ -13970,6 +14162,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/diff-sequences": "30.4.0", "@jest/get-type": "30.1.0", @@ -13984,6 +14177,7 @@ "version": "30.4.0", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "detect-newline": "^3.1.0" }, @@ -13995,6 +14189,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/get-type": "30.1.0", "@jest/types": "30.4.1", @@ -14010,6 +14205,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/environment": "30.4.1", "@jest/fake-timers": "30.4.1", @@ -14035,6 +14231,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/types": "30.4.1", "@types/node": "*", @@ -14058,6 +14255,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/get-type": "30.1.0", "pretty-format": "30.4.1" @@ -14070,6 +14268,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/get-type": "30.1.0", "chalk": "^4.1.2", @@ -14084,6 +14283,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.27.1", "@jest/types": "30.4.1", @@ -14104,6 +14304,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/types": "30.4.1", "@types/node": "*", @@ -14196,6 +14397,7 @@ "version": "1.2.3", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=6" }, @@ -14212,6 +14414,7 @@ "version": "30.4.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } @@ -14220,6 +14423,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "chalk": "^4.1.2", "graceful-fs": "^4.2.11", @@ -14238,6 +14442,7 @@ "version": "30.4.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "jest-regex-util": "30.4.0", "jest-snapshot": "30.4.1" @@ -14250,6 +14455,7 @@ "version": "30.4.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/console": "30.4.1", "@jest/environment": "30.4.1", @@ -14282,6 +14488,7 @@ "version": "30.4.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/environment": "30.4.1", "@jest/fake-timers": "30.4.1", @@ -14313,12 +14520,14 @@ "node_modules/jest-runtime/node_modules/balanced-match": { "version": "1.0.2", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/jest-runtime/node_modules/brace-expansion": { "version": "2.1.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "balanced-match": "^1.0.0" } @@ -14327,6 +14536,7 @@ "version": "10.5.0", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", @@ -14346,6 +14556,7 @@ "version": "9.0.9", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "brace-expansion": "^2.0.2" }, @@ -14360,6 +14571,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/core": "^7.27.4", "@babel/generator": "^7.27.5", @@ -14391,6 +14603,7 @@ "version": "7.8.5", "dev": true, "license": "ISC", + "peer": true, "bin": { "semver": "bin/semver.js" }, @@ -14402,6 +14615,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/types": "30.4.1", "@types/node": "*", @@ -14418,6 +14632,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/get-type": "30.1.0", "@jest/types": "30.4.1", @@ -14434,6 +14649,7 @@ "version": "6.3.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=10" }, @@ -14445,6 +14661,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/test-result": "30.4.1", "@jest/types": "30.4.1", @@ -14463,6 +14680,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@types/node": "*", "@ungap/structured-clone": "^1.3.0", @@ -14478,6 +14696,7 @@ "version": "8.1.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "has-flag": "^4.0.0" }, @@ -14492,7 +14711,6 @@ "version": "2.6.1", "dev": true, "license": "MIT", - "peer": true, "bin": { "jiti": "lib/jiti-cli.mjs" } @@ -14507,6 +14725,7 @@ "node_modules/js-levenshtein": { "version": "1.1.6", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -15374,6 +15593,7 @@ "version": "5.1.1", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "yallist": "^3.0.2" } @@ -15600,6 +15820,7 @@ "version": "1.0.12", "dev": true, "license": "BSD-3-Clause", + "peer": true, "dependencies": { "tmpl": "1.0.5" } @@ -16399,7 +16620,8 @@ "node_modules/node-int64": { "version": "0.4.0", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/node-readfiles": { "version": "0.2.0", @@ -16784,7 +17006,6 @@ "dev": true, "hasInstallScript": true, "license": "MIT", - "peer": true, "dependencies": { "@nrwl/cli": "15.9.7", "@nrwl/tao": "15.9.7", @@ -17170,12 +17391,12 @@ }, "node_modules/openapi-types": { "version": "12.1.3", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/openapi-typescript": { "version": "7.13.0", "license": "MIT", + "peer": true, "dependencies": { "@redocly/openapi-core": "^1.34.6", "ansi-colors": "^4.1.3", @@ -17194,6 +17415,7 @@ "node_modules/openapi-typescript/node_modules/parse-json": { "version": "8.3.0", "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.26.2", "index-to-position": "^1.1.0", @@ -17209,6 +17431,7 @@ "node_modules/openapi-typescript/node_modules/supports-color": { "version": "10.2.2", "license": "MIT", + "peer": true, "engines": { "node": ">=18" }, @@ -17219,6 +17442,7 @@ "node_modules/openapi-typescript/node_modules/yargs-parser": { "version": "21.1.1", "license": "ISC", + "peer": true, "engines": { "node": ">=12" } @@ -18095,7 +18319,6 @@ "node_modules/pg": { "version": "8.22.0", "license": "MIT", - "peer": true, "dependencies": { "pg-connection-string": "^2.14.0", "pg-pool": "^3.14.0", @@ -18195,7 +18418,6 @@ "node_modules/pino": { "version": "10.3.1", "license": "MIT", - "peer": true, "dependencies": { "@pinojs/redact": "^0.4.0", "atomic-sleep": "^1.0.0", @@ -18302,6 +18524,7 @@ "version": "4.0.7", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">= 6" } @@ -18471,7 +18694,6 @@ "node_modules/prettier": { "version": "3.9.5", "license": "MIT", - "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -18486,6 +18708,7 @@ "version": "30.4.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/schemas": "30.4.1", "ansi-styles": "^5.2.0", @@ -18500,6 +18723,7 @@ "version": "5.2.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=10" }, @@ -18576,7 +18800,6 @@ "node_modules/prom-client": { "version": "15.1.3", "license": "Apache-2.0", - "peer": true, "dependencies": { "@opentelemetry/api": "^1.4.0", "tdigest": "^0.1.1" @@ -18712,7 +18935,8 @@ "url": "https://opencollective.com/fast-check" } ], - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/q": { "version": "1.5.1", @@ -18809,13 +19033,15 @@ "name": "react-is", "version": "18.3.1", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/react-is-19": { "name": "react-is", "version": "19.2.7", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/read": { "version": "1.0.7", @@ -20031,6 +20257,7 @@ "version": "2.0.6", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "escape-string-regexp": "^2.0.0" }, @@ -20042,6 +20269,7 @@ "version": "2.0.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=8" } @@ -20103,6 +20331,7 @@ "version": "4.0.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "char-regex": "^1.0.2", "strip-ansi": "^6.0.0" @@ -20232,7 +20461,6 @@ "node_modules/supertest": { "version": "7.2.2", "license": "MIT", - "peer": true, "dependencies": { "cookie-signature": "^1.2.2", "methods": "^1.1.2", @@ -20320,6 +20548,7 @@ "version": "0.11.13", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@pkgr/core": "^0.3.6" }, @@ -20334,6 +20563,7 @@ "version": "0.3.6", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": "^14.18.0 || >=16.0.0" }, @@ -20465,6 +20695,7 @@ "version": "6.0.0", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "@istanbuljs/schema": "^0.1.2", "glob": "^7.1.4", @@ -20477,12 +20708,14 @@ "node_modules/test-exclude/node_modules/balanced-match": { "version": "1.0.2", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/test-exclude/node_modules/brace-expansion": { "version": "1.1.16", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -20492,6 +20725,7 @@ "version": "7.2.3", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -20511,6 +20745,7 @@ "version": "3.1.5", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "brace-expansion": "^1.1.7" }, @@ -20637,7 +20872,8 @@ "node_modules/tmpl": { "version": "1.0.5", "dev": true, - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "peer": true }, "node_modules/to-buffer": { "version": "1.2.2", @@ -21011,6 +21247,7 @@ "version": "4.0.8", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=4" } @@ -21331,7 +21568,9 @@ } }, "node_modules/undici-types": { - "version": "8.3.0", + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", "license": "MIT" }, "node_modules/unique-filename": { @@ -21479,7 +21718,8 @@ }, "node_modules/uri-js-replace": { "version": "1.0.1", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/url-value-parser": { "version": "2.2.0", @@ -21524,6 +21764,7 @@ "version": "9.3.0", "dev": true, "license": "ISC", + "peer": true, "dependencies": { "@jridgewell/trace-mapping": "^0.3.12", "@types/istanbul-lib-coverage": "^2.0.1", @@ -21564,7 +21805,6 @@ "version": "8.1.4", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.5", @@ -21641,7 +21881,6 @@ "version": "4.1.10", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@vitest/expect": "4.1.10", "@vitest/mocker": "4.1.10", @@ -21735,6 +21974,7 @@ "version": "1.0.8", "dev": true, "license": "Apache-2.0", + "peer": true, "dependencies": { "makeerror": "1.0.12" } @@ -22062,7 +22302,8 @@ "node_modules/yallist": { "version": "3.1.1", "dev": true, - "license": "ISC" + "license": "ISC", + "peer": true }, "node_modules/yaml": { "version": "2.9.0", @@ -22079,7 +22320,8 @@ }, "node_modules/yaml-ast-parser": { "version": "0.0.43", - "license": "Apache-2.0" + "license": "Apache-2.0", + "peer": true }, "node_modules/yargs": { "version": "17.7.3", @@ -22533,6 +22775,7 @@ "@map-colonies/tracing-utils": "^1.0.0", "@opentelemetry/api": "^1.9.1", "@smithy/node-http-handler": "^4.7.6", + "ajv": "^8.20.0", "axios": "^1.7.0", "express": "^5.2.1", "node-cron": "^4.2.1", @@ -22545,7 +22788,7 @@ }, "devDependencies": { "@types/nock": "^10.0.3", - "@types/node": "^20.11.0", + "@types/node": "^24.0.0", "@types/node-cron": "^3.0.11", "@vitest/coverage-v8": "^4.0.18", "@vitest/ui": "^4.0.18", @@ -22561,19 +22804,6 @@ "peerDependencies": { "typeorm": "^0.3.20" } - }, - "packages/synchronizer/node_modules/@types/node": { - "version": "20.19.43", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~6.21.0" - } - }, - "packages/synchronizer/node_modules/undici-types": { - "version": "6.21.0", - "dev": true, - "license": "MIT" } } } diff --git a/package.json b/package.json index 67e5b2b..2b3b9aa 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,9 @@ "lerna": "^6.6.2" }, "dependencies": { - "@map-colonies/schemas": "1.26.0" + "@map-colonies/schemas": "https://ghatmpstorage.blob.core.windows.net/npm-packages/schemas-11bf4605d7a17c723dd85f624c88fe32d9957863.tgz" + }, + "overrides": { + "@types/node": "^24.0.0" } } diff --git a/packages/api/src/common/db/connection.ts b/packages/api/src/common/db/connection.ts deleted file mode 100644 index c5b451f..0000000 --- a/packages/api/src/common/db/connection.ts +++ /dev/null @@ -1,15 +0,0 @@ -import type { FactoryFunction, DependencyContainer } from 'tsyringe'; -import { DataSource } from 'typeorm'; -import { createDataSourceOptions, createDataSourceHealthCheck, DATA_SOURCE_PROVIDER } from '@db'; -import { type HealthCheck } from '@godaddy/terminus'; -import { SERVICE_NAME, SERVICES } from '../constants'; -import type { ConfigType } from '../config'; - -export const dataSourceFactory: FactoryFunction = (container: DependencyContainer): DataSource => { - const config = container.resolve(SERVICES.CONFIG); - const dbConfig = config.get('db'); - return new DataSource(createDataSourceOptions(dbConfig, SERVICE_NAME)); -}; - -export const healthCheckFactory: FactoryFunction = (container: DependencyContainer): HealthCheck => - createDataSourceHealthCheck(container, [DATA_SOURCE_PROVIDER]); diff --git a/packages/api/src/containerConfig.ts b/packages/api/src/containerConfig.ts index 8c1f4e3..5e579db 100644 --- a/packages/api/src/containerConfig.ts +++ b/packages/api/src/containerConfig.ts @@ -4,14 +4,15 @@ import { Registry } from 'prom-client'; import type { DependencyContainer } from 'tsyringe/dist/typings/types'; import { jsLogger, type Logger } from '@map-colonies/js-logger'; import { CleanupRegistry } from '@map-colonies/cleanup-registry'; -import { DATA_SOURCE_PROVIDER, Layer, LAYER_REPOSITORY_SYMBOL } from '@db'; +import { DATA_SOURCE_PROVIDER, Layer, LAYER_REPOSITORY_SYMBOL, createDataSource, createDataSourceHealthCheck } from '@db'; +import type { HealthCheck } from '@godaddy/terminus'; import type { Repository, DataSource } from 'typeorm'; import { instancePerContainerCachingFactory } from 'tsyringe'; import { type InjectionObject, registerDependencies } from '@common/dependencyRegistration'; import { HEALTHCHECK, ON_SIGNAL, SERVICES, SERVICE_NAME } from '@common/constants'; import { getTracing } from '@common/tracing'; import { LAYER_ROUTER_SYMBOL, layerRouterFactory } from './layer/routes/layer'; -import { dataSourceFactory, healthCheckFactory } from './common/db/connection'; +import { NAMESPACE_ROUTER_SYMBOL, namespaceRouterFactory } from './namespace/routes/namespace'; import { type ConfigType, getConfig } from './common/config'; export interface RegisterOptions { @@ -66,6 +67,7 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise }, }, { token: LAYER_ROUTER_SYMBOL, provider: { useFactory: layerRouterFactory } }, + { token: NAMESPACE_ROUTER_SYMBOL, provider: { useFactory: namespaceRouterFactory } }, { token: LAYER_REPOSITORY_SYMBOL, provider: { @@ -78,7 +80,10 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: DATA_SOURCE_PROVIDER, provider: { - useFactory: instancePerContainerCachingFactory(dataSourceFactory), + useFactory: instancePerContainerCachingFactory((container) => { + const config = container.resolve(SERVICES.CONFIG); + return createDataSource(config.get('db'), SERVICE_NAME); + }), }, postInjectionHook: async (container: DependencyContainer): Promise => { const dataSource = container.resolve(DATA_SOURCE_PROVIDER); @@ -96,7 +101,7 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: HEALTHCHECK, provider: { - useFactory: healthCheckFactory, + useFactory: (container: DependencyContainer): HealthCheck => createDataSourceHealthCheck(container, [DATA_SOURCE_PROVIDER]), }, }, ]; diff --git a/packages/api/src/layer/controllers/layerController.ts b/packages/api/src/layer/controllers/layerController.ts index c40c803..83b03eb 100644 --- a/packages/api/src/layer/controllers/layerController.ts +++ b/packages/api/src/layer/controllers/layerController.ts @@ -4,9 +4,9 @@ import { injectable, inject } from 'tsyringe'; import { RequestHandler } from 'express'; import { SERVICES } from '@common/constants'; import { LayerManager } from '../models/layerManager'; -import { GetLayerByNameParam, GetLayersResponse } from '../types/layerTypes'; +import { GetLayerByNameParam, GetLayersParam, GetLayersResponse } from '../types/layerTypes'; -type GetLayersHandler = RequestHandler; +type GetLayersHandler = RequestHandler; type GetLayerByNameHandler = RequestHandler; @injectable() @@ -17,24 +17,25 @@ export class LayerController { ) {} public getLayers: GetLayersHandler = async (req, res, next) => { + const { namespace } = req.params; try { - const layers = await this.manager.getLayers(); - this.logger.debug({ msg: `got ${layers.length} layers` }); + const layers = await this.manager.getLayers(namespace); + this.logger.debug({ msg: `got ${layers.length} layers for namespace ${namespace}` }); return res.json({ layers }); } catch (error) { - this.logger.error({ msg: 'failed to get layers', err: error }); + this.logger.error({ msg: `failed to get layers for namespace ${namespace}`, err: error }); next(error); } }; public getLayerByName: GetLayerByNameHandler = async (req, res, next) => { - const { layerName } = req.params; + const { namespace, layerName } = req.params; try { - const layer = await this.manager.getLayerSpecByName(layerName); - this.logger.debug({ msg: `got layer: ${layerName}` }); + const layer = await this.manager.getLayerSpecByName(namespace, layerName); + this.logger.debug({ msg: `got layer: ${layerName} for namespace ${namespace}` }); return res.json(layer); } catch (error) { - this.logger.error({ msg: 'failed to get layer by name', layerName, err: error }); + this.logger.error({ msg: `failed to get layer ${layerName} for namespace ${namespace}`, err: error }); next(error); } }; diff --git a/packages/api/src/layer/models/layerManager.ts b/packages/api/src/layer/models/layerManager.ts index 47ad673..e6bce80 100644 --- a/packages/api/src/layer/models/layerManager.ts +++ b/packages/api/src/layer/models/layerManager.ts @@ -14,16 +14,19 @@ export class LayerManager { @inject(SERVICES.LOGGER) private readonly logger: Logger ) {} - public async getLayers(): Promise { - this.logger.debug({ msg: 'getting layers' }); - const layers = await this.repository.find({ select: { layerName: true, alias: true } }); + public async getLayers(namespace: string): Promise { + this.logger.debug({ msg: `getting layers for namespace ${namespace}` }); + const layers = await this.repository.find({ where: { namespace }, select: { layerName: true, alias: true } }); + if (layers.length === 0) { + throw new NotFoundError(`Namespace doesn't exist`); + } return layers.map(({ layerName, alias }) => ({ layerName, alias })); } - public async getLayerSpecByName(name: string): Promise { - this.logger.debug({ msg: 'getting layer spec by name', name }); + public async getLayerSpecByName(namespace: string, name: string): Promise { + this.logger.debug({ msg: `getting layer spec ${name} for namespace ${namespace}` }); const layer = await this.repository.findOne({ - where: { layerName: name }, + where: { namespace, layerName: name }, relations: { properties: { possibleValues: true } }, }); if (!layer) { diff --git a/packages/api/src/layer/routes/layer.ts b/packages/api/src/layer/routes/layer.ts index 09b45d0..d5f3952 100644 --- a/packages/api/src/layer/routes/layer.ts +++ b/packages/api/src/layer/routes/layer.ts @@ -6,8 +6,8 @@ const layerRouterFactory: FactoryFunction = (dependencyContainer) => { const router = Router(); const controller = dependencyContainer.resolve(LayerController); - router.get('/', controller.getLayers); - router.get('/:layerName', controller.getLayerByName); + router.get('/:namespace/layers', controller.getLayers); + router.get('/:namespace/layers/:layerName', controller.getLayerByName); return router; }; diff --git a/packages/api/src/layer/types/layerTypes.ts b/packages/api/src/layer/types/layerTypes.ts index 27d53d1..9d292f0 100644 --- a/packages/api/src/layer/types/layerTypes.ts +++ b/packages/api/src/layer/types/layerTypes.ts @@ -1,6 +1,11 @@ import type { LayerSummary } from '@db'; +export interface GetLayersParam { + namespace: string; +} + export interface GetLayerByNameParam { + namespace: string; layerName: string; } diff --git a/packages/api/src/namespace/controllers/namespaceController.ts b/packages/api/src/namespace/controllers/namespaceController.ts new file mode 100644 index 0000000..1c72f3d --- /dev/null +++ b/packages/api/src/namespace/controllers/namespaceController.ts @@ -0,0 +1,27 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { injectable, inject } from 'tsyringe'; +import { RequestHandler } from 'express'; +import { SERVICES } from '@common/constants'; +import { NamespaceManager } from '../models/namespaceManager'; +import { GetNamespacesResponse } from '../types/namespaceTypes'; + +type GetNamespacesHandler = RequestHandler; + +@injectable() +export class NamespaceController { + public constructor( + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(NamespaceManager) private readonly manager: NamespaceManager + ) {} + + public getNamespaces: GetNamespacesHandler = async (req, res, next) => { + try { + const namespaces = await this.manager.getNamespaces(); + this.logger.debug({ msg: `got ${namespaces.length} namespaces` }); + return res.json({ namespaces }); + } catch (error) { + this.logger.error({ msg: 'failed to get namespaces', err: error }); + next(error); + } + }; +} diff --git a/packages/api/src/namespace/models/namespaceManager.ts b/packages/api/src/namespace/models/namespaceManager.ts new file mode 100644 index 0000000..977ebf9 --- /dev/null +++ b/packages/api/src/namespace/models/namespaceManager.ts @@ -0,0 +1,24 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { injectable, inject } from 'tsyringe'; +import { Repository } from 'typeorm'; +import { Layer, LAYER_REPOSITORY_SYMBOL } from '@db'; +import type { NamespaceSummary } from '@db'; +import { SERVICES } from '@common/constants'; + +@injectable() +export class NamespaceManager { + public constructor( + @inject(LAYER_REPOSITORY_SYMBOL) private readonly repository: Repository, + @inject(SERVICES.LOGGER) private readonly logger: Logger + ) {} + + public async getNamespaces(): Promise { + this.logger.debug({ msg: 'getting namespaces' }); + return this.repository + .createQueryBuilder('layer') + .select('layer.namespace', 'name') + .distinct(true) + .orderBy('name', 'ASC') + .getRawMany(); + } +} diff --git a/packages/api/src/namespace/routes/namespace.ts b/packages/api/src/namespace/routes/namespace.ts new file mode 100644 index 0000000..e308e0f --- /dev/null +++ b/packages/api/src/namespace/routes/namespace.ts @@ -0,0 +1,16 @@ +import { Router } from 'express'; +import type { FactoryFunction } from 'tsyringe'; +import { NamespaceController } from '../controllers/namespaceController'; + +const namespaceRouterFactory: FactoryFunction = (dependencyContainer) => { + const router = Router(); + const controller = dependencyContainer.resolve(NamespaceController); + + router.get('/', controller.getNamespaces); + + return router; +}; + +export const NAMESPACE_ROUTER_SYMBOL = Symbol('namespaceRouterFactory'); + +export { namespaceRouterFactory }; diff --git a/packages/api/src/namespace/types/namespaceTypes.ts b/packages/api/src/namespace/types/namespaceTypes.ts new file mode 100644 index 0000000..3f2e445 --- /dev/null +++ b/packages/api/src/namespace/types/namespaceTypes.ts @@ -0,0 +1,5 @@ +import type { NamespaceSummary } from '@db'; + +export interface GetNamespacesResponse { + namespaces: NamespaceSummary[]; +} diff --git a/packages/api/src/serverBuilder.ts b/packages/api/src/serverBuilder.ts index f2e28e6..b56043d 100644 --- a/packages/api/src/serverBuilder.ts +++ b/packages/api/src/serverBuilder.ts @@ -11,6 +11,7 @@ import { collectMetricsExpressMiddleware } from '@map-colonies/prometheus'; import { Registry } from 'prom-client'; import { SERVICES } from '@common/constants'; import { LAYER_ROUTER_SYMBOL } from './layer/routes/layer'; +import { NAMESPACE_ROUTER_SYMBOL } from './namespace/routes/namespace'; import { ConfigType } from './common/config'; @injectable() @@ -21,7 +22,8 @@ export class ServerBuilder { @inject(SERVICES.CONFIG) private readonly config: ConfigType, @inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(SERVICES.METRICS) private readonly metricsRegistry: Registry, - @inject(LAYER_ROUTER_SYMBOL) private readonly layerRouter: Router + @inject(LAYER_ROUTER_SYMBOL) private readonly layerRouter: Router, + @inject(NAMESPACE_ROUTER_SYMBOL) private readonly namespaceRouter: Router ) { this.serverInstance = express(); } @@ -44,7 +46,8 @@ export class ServerBuilder { } private buildRoutes(): void { - this.serverInstance.use('/layers', this.layerRouter); + this.serverInstance.use('/namespaces', this.namespaceRouter); + this.serverInstance.use('/namespaces', this.layerRouter); this.buildDocsRoutes(); } diff --git a/packages/api/tests/integration/layer/helpers/layerRequestSender.ts b/packages/api/tests/integration/layer/helpers/layerRequestSender.ts index 9a73f66..99f4ad1 100644 --- a/packages/api/tests/integration/layer/helpers/layerRequestSender.ts +++ b/packages/api/tests/integration/layer/helpers/layerRequestSender.ts @@ -3,6 +3,7 @@ import type supertest from 'supertest'; import { agent } from 'supertest'; import type { Layer } from '@db'; import type { GetLayersResponse } from '@src/layer/types/layerTypes'; +import type { GetNamespacesResponse } from '@src/namespace/types/namespaceTypes'; interface TypedResponse extends Omit { body: T; @@ -11,11 +12,15 @@ interface TypedResponse extends Omit { export class LayerRequestSender { public constructor(private readonly app: Application) {} - public async getLayers(): Promise> { - return agent(this.app).get('/layers'); + public async getNamespaces(): Promise> { + return agent(this.app).get('/namespaces'); } - public async getLayerByName(layerName: string): Promise> { - return agent(this.app).get(`/layers/${layerName}`); + public async getLayers(namespace: string): Promise> { + return agent(this.app).get(`/namespaces/${namespace}/layers`); + } + + public async getLayerByName(namespace: string, layerName: string): Promise> { + return agent(this.app).get(`/namespaces/${namespace}/layers/${layerName}`); } } diff --git a/packages/api/tests/integration/layer/layer.spec.ts b/packages/api/tests/integration/layer/layer.spec.ts index 479cada..fbbf18b 100644 --- a/packages/api/tests/integration/layer/layer.spec.ts +++ b/packages/api/tests/integration/layer/layer.spec.ts @@ -3,13 +3,16 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from import { trace } from '@opentelemetry/api'; import httpStatusCodes from 'http-status-codes'; import type { DataSource, Repository } from 'typeorm'; -import { DATA_SOURCE_PROVIDER, EnumValue, LAYER_REPOSITORY_SYMBOL, Property, columnType } from '@map-colonies/vector-standard-db'; +import { DATA_SOURCE_PROVIDER, EnumValue, LAYER_REPOSITORY_SYMBOL, Property, columnType, layerSource } from '@map-colonies/vector-standard-db'; import type { Layer } from '@map-colonies/vector-standard-db'; import { getApp } from '@src/app'; import { SERVICES } from '@src/common/constants'; import { initConfig } from '@src/common/config'; import { LayerRequestSender } from './helpers/layerRequestSender'; +const NAMESPACE = 'data-2025'; +const OTHER_NAMESPACE = 'data-2024'; + describe('layer', function () { let requestSender: LayerRequestSender; let dataSource: DataSource; @@ -44,37 +47,76 @@ describe('layer', function () { }); describe('Happy Path', function () { - describe('GET /layers', function () { + describe('GET /namespaces', function () { + it('should return 200 with every namespace holding layers, without duplicates', async function () { + await layerRepository.save([ + { namespace: NAMESPACE, layerName: 'buildings_polygon', layerId: 1, source: layerSource.sharedLua, alias: 'Buildings Polygon' }, + { namespace: NAMESPACE, layerName: 'fences_line', layerId: 2, source: layerSource.sharedLua, alias: 'Fences Line' }, + { namespace: OTHER_NAMESPACE, layerName: 'buildings_polygon', layerId: null, source: layerSource.perLayerJson, alias: 'Old Buildings' }, + ]); + + const response = await requestSender.getNamespaces(); + const { namespaces } = response.body; + + expect(response.statusCode).toBe(httpStatusCodes.OK); + expect(namespaces).toEqual([{ name: OTHER_NAMESPACE }, { name: NAMESPACE }]); + }); + }); + + describe('GET /namespaces/:namespace/layers', function () { beforeEach(async function () { await layerRepository.save([ - { layerName: 'buildings_polygon', layerId: 1, alias: 'Buildings Polygon' }, - { layerName: 'fences_line', layerId: 2, alias: 'Fences Line' }, + { namespace: NAMESPACE, layerName: 'buildings_polygon', layerId: 1, source: layerSource.sharedLua, alias: 'Buildings Polygon' }, + { namespace: NAMESPACE, layerName: 'fences_line', layerId: 2, source: layerSource.sharedLua, alias: 'Fences Line' }, ]); await propertyRepository.save([ - { layerName: 'buildings_polygon', property: 'code', type: columnType.text }, - { layerName: 'fences_line', property: 'height', type: columnType.real }, + { namespace: NAMESPACE, layerName: 'buildings_polygon', property: 'code', type: columnType.text }, + { namespace: NAMESPACE, layerName: 'fences_line', property: 'height', type: columnType.real }, ]); }); it('should return 200 with all layers including their aliases', async function () { - const response = await requestSender.getLayers(); + const response = await requestSender.getLayers(NAMESPACE); const { layers } = response.body; expect(response.statusCode).toBe(httpStatusCodes.OK); expect(layers).toContainEqual({ layerName: 'buildings_polygon', alias: 'Buildings Polygon' }); expect(layers).toContainEqual({ layerName: 'fences_line', alias: 'Fences Line' }); }); + + it('should not return layers belonging to another namespace', async function () { + await layerRepository.save({ + namespace: OTHER_NAMESPACE, + layerName: 'rivers_line', + layerId: null, + source: layerSource.perLayerJson, + alias: 'Rivers', + }); + + const response = await requestSender.getLayers(NAMESPACE); + const { layers } = response.body; + + expect(response.statusCode).toBe(httpStatusCodes.OK); + expect(layers).toHaveLength(2); + expect(layers).not.toContainEqual({ layerName: 'rivers_line', alias: 'Rivers' }); + }); }); - describe('GET /layers/:layerName', function () { + describe('GET /namespaces/:namespace/layers/:layerName', function () { it('should return 200 with the layer spec for non-enum properties', async function () { - await layerRepository.save({ layerName: 'buildings_polygon', layerId: 1, alias: 'Buildings Polygon' }); + await layerRepository.save({ + namespace: NAMESPACE, + layerName: 'buildings_polygon', + layerId: 1, + source: layerSource.sharedLua, + alias: 'Buildings Polygon', + }); await propertyRepository.save([ - { layerName: 'buildings_polygon', property: 'code', type: columnType.text }, - { layerName: 'buildings_polygon', property: 'height', type: columnType.real }, + { namespace: NAMESPACE, layerName: 'buildings_polygon', property: 'code', type: columnType.text }, + { namespace: NAMESPACE, layerName: 'buildings_polygon', property: 'height', type: columnType.real }, ]); - const response = await requestSender.getLayerByName('buildings_polygon'); + const response = await requestSender.getLayerByName(NAMESPACE, 'buildings_polygon'); const { layerName, properties } = response.body; expect(response.statusCode).toBe(httpStatusCodes.OK); @@ -84,14 +126,20 @@ describe('layer', function () { }); it('should return 200 with possibleValues for enum properties', async function () { - await layerRepository.save({ layerName: 'buildings_polygon', layerId: 1, alias: 'Buildings Polygon' }); - await propertyRepository.save({ layerName: 'buildings_polygon', property: 'classification', type: columnType.text }); + await layerRepository.save({ + namespace: NAMESPACE, + layerName: 'buildings_polygon', + layerId: 1, + source: layerSource.sharedLua, + alias: 'Buildings Polygon', + }); + await propertyRepository.save({ namespace: NAMESPACE, layerName: 'buildings_polygon', property: 'classification', type: columnType.text }); await enumValueRepository.save([ - { value: 'A', layerName: 'buildings_polygon', property: 'classification' }, - { value: 'B', layerName: 'buildings_polygon', property: 'classification' }, + { namespace: NAMESPACE, value: 'A', layerName: 'buildings_polygon', property: 'classification' }, + { namespace: NAMESPACE, value: 'B', layerName: 'buildings_polygon', property: 'classification' }, ]); - const response = await requestSender.getLayerByName('buildings_polygon'); + const response = await requestSender.getLayerByName(NAMESPACE, 'buildings_polygon'); const { properties } = response.body; const classificationProp = properties.find((p) => p.property === 'classification'); @@ -100,23 +148,64 @@ describe('layer', function () { expect(classificationProp?.possibleValues).toContain('A'); expect(classificationProp?.possibleValues).toContain('B'); }); + + it('should return the spec of the requested namespace when the layer name exists in both', async function () { + await layerRepository.save([ + { namespace: NAMESPACE, layerName: 'buildings_polygon', layerId: 1, source: layerSource.sharedLua, alias: 'New Buildings' }, + { namespace: OTHER_NAMESPACE, layerName: 'buildings_polygon', layerId: null, source: layerSource.perLayerJson, alias: 'Old Buildings' }, + ]); + await propertyRepository.save([ + { namespace: NAMESPACE, layerName: 'buildings_polygon', property: 'code', type: columnType.text }, + { namespace: OTHER_NAMESPACE, layerName: 'buildings_polygon', property: 'legacy_code', type: columnType.text }, + ]); + + const response = await requestSender.getLayerByName(OTHER_NAMESPACE, 'buildings_polygon'); + const { alias, properties } = response.body; + + expect(response.statusCode).toBe(httpStatusCodes.OK); + expect(alias).toBe('Old Buildings'); + expect(properties).toContainEqual({ property: 'legacy_code', type: columnType.text }); + expect(properties).not.toContainEqual({ property: 'code', type: columnType.text }); + }); }); }); describe('Sad Path', function () { - describe('GET /layers', function () { + describe('GET /namespaces', function () { it('should return 200 with an empty list when there are no layers', async function () { - const response = await requestSender.getLayers(); - const { layers } = response.body; + const response = await requestSender.getNamespaces(); + const { namespaces } = response.body; expect(response.statusCode).toBe(httpStatusCodes.OK); - expect(layers).toHaveLength(0); + expect(namespaces).toHaveLength(0); }); }); - describe('GET /layers/:layerName', function () { + describe('GET /namespaces/:namespace/layers', function () { + it('should return 404 not found for a namespace holding no layers', async function () { + const response = await requestSender.getLayers('nonexistent'); + + expect(response.statusCode).toBe(httpStatusCodes.NOT_FOUND); + }); + }); + + describe('GET /namespaces/:namespace/layers/:layerName', function () { it('should return 404 not found for a non-existent layer', async function () { - const response = await requestSender.getLayerByName('nonexistent'); + const response = await requestSender.getLayerByName(NAMESPACE, 'nonexistent'); + + expect(response.statusCode).toBe(httpStatusCodes.NOT_FOUND); + }); + + it('should return 404 not found when the layer exists only in another namespace', async function () { + await layerRepository.save({ + namespace: OTHER_NAMESPACE, + layerName: 'rivers_line', + layerId: null, + source: layerSource.perLayerJson, + alias: 'Rivers', + }); + + const response = await requestSender.getLayerByName(NAMESPACE, 'rivers_line'); expect(response.statusCode).toBe(httpStatusCodes.NOT_FOUND); }); diff --git a/packages/api/tests/unit/layer/models/layerManager.spec.ts b/packages/api/tests/unit/layer/models/layerManager.spec.ts index 7894896..3c37a40 100644 --- a/packages/api/tests/unit/layer/models/layerManager.spec.ts +++ b/packages/api/tests/unit/layer/models/layerManager.spec.ts @@ -7,6 +7,8 @@ import { SERVICES } from '@src/common/constants'; import { registerDependencies } from '@src/common/dependencyRegistration'; describe('LayerManager', function () { + const TEST_NAMESPACE = 'data-2026'; + let manager: LayerManager; const find = vi.fn(); @@ -38,7 +40,7 @@ describe('LayerManager', function () { ]; find.mockResolvedValue(layers); - const result = await manager.getLayers(); + const result = await manager.getLayers(TEST_NAMESPACE); expect(result).toEqual(layers); }); @@ -48,9 +50,12 @@ describe('LayerManager', function () { it('should query using the provided layer name', async function () { findOne.mockResolvedValue({ layerName: 'buildings_polygon', alias: 'Buildings', properties: [] }); - await manager.getLayerSpecByName('buildings_polygon'); + await manager.getLayerSpecByName(TEST_NAMESPACE, 'buildings_polygon'); - expect(findOne).toHaveBeenCalledWith({ where: { layerName: 'buildings_polygon' }, relations: { properties: { possibleValues: true } } }); + expect(findOne).toHaveBeenCalledWith({ + where: { layerName: 'buildings_polygon', namespace: TEST_NAMESPACE }, + relations: { properties: { possibleValues: true } }, + }); }); it.each([ @@ -84,7 +89,7 @@ describe('LayerManager', function () { ])('should $name', async function ({ inputProperties, expectedPossibleValues }) { findOne.mockResolvedValue({ layerName: 'buildings_polygon', alias: 'Buildings', properties: inputProperties }); - const { properties } = await manager.getLayerSpecByName('buildings_polygon'); + const { properties } = await manager.getLayerSpecByName(TEST_NAMESPACE, 'buildings_polygon'); for (const { property, possibleValues } of expectedPossibleValues) { expect(properties.find((p: { property: string }) => p.property === property)?.possibleValues).toEqual(possibleValues); @@ -95,12 +100,10 @@ describe('LayerManager', function () { describe('Sad Path', function () { describe('getLayers', function () { - it('should return an empty array when there are no layers', async function () { + it("should throw an error when the namespace doesn't exist", async function () { find.mockResolvedValue([]); - const result = await manager.getLayers(); - - expect(result).toEqual([]); + await expect(manager.getLayers(TEST_NAMESPACE)).rejects.toThrow(); }); }); @@ -108,7 +111,7 @@ describe('LayerManager', function () { it('should throw a NotFoundError when the layer does not exist', async function () { findOne.mockResolvedValue(null); - await expect(manager.getLayerSpecByName('nonexistent')).rejects.toThrow(NotFoundError); + await expect(manager.getLayerSpecByName(TEST_NAMESPACE, 'nonexistent')).rejects.toThrow(NotFoundError); }); }); }); diff --git a/packages/db/src/db/connection.ts b/packages/db/src/db/connection.ts index 290d847..c093184 100644 --- a/packages/db/src/db/connection.ts +++ b/packages/db/src/db/connection.ts @@ -1,5 +1,6 @@ import { readFileSync } from 'node:fs'; -import type { DataSource, DataSourceOptions } from 'typeorm'; +import { DataSource } from 'typeorm'; +import type { DataSourceOptions } from 'typeorm'; import type { DependencyContainer, InjectionToken } from 'tsyringe'; import { Property } from '../entities/property'; import { EnumValue } from '../entities/enumValue'; @@ -51,3 +52,6 @@ export const createDataSourceOptions = (dbConfig: DbConfig, applicationName: str ssl: createSslOptions(ssl), }; }; + +export const createDataSource = (dbConfig: DbConfig, applicationName: string): DataSource => + new DataSource(createDataSourceOptions(dbConfig, applicationName)); diff --git a/packages/db/src/entities/enumValue.ts b/packages/db/src/entities/enumValue.ts index d73b465..cdb60c2 100644 --- a/packages/db/src/entities/enumValue.ts +++ b/packages/db/src/entities/enumValue.ts @@ -5,6 +5,9 @@ export const ENUMS_REPOSITORY_SYMBOL = Symbol('EnumsRepository'); @Entity('enum_value') export class EnumValue { + @PrimaryColumn() + public namespace!: string; + @PrimaryColumn() public value!: string; @@ -16,6 +19,7 @@ export class EnumValue { @ManyToOne(() => Property, (p) => p.possibleValues, { onDelete: 'CASCADE' }) @JoinColumn([ + { name: 'namespace', referencedColumnName: 'namespace' }, { name: 'layer_name', referencedColumnName: 'layerName' }, { name: 'property', referencedColumnName: 'property' }, ]) diff --git a/packages/db/src/entities/layer.ts b/packages/db/src/entities/layer.ts index d28902a..fc8beb8 100644 --- a/packages/db/src/entities/layer.ts +++ b/packages/db/src/entities/layer.ts @@ -1,16 +1,29 @@ -import { Column, Entity, OneToMany, PrimaryColumn } from 'typeorm'; +import { Check, Column, Entity, OneToMany, PrimaryColumn } from 'typeorm'; import { Layer as ILayer } from '../types/layer'; +import { LayerSource, layerSource } from '../types/enums'; import { Property } from './property'; export const LAYER_REPOSITORY_SYMBOL = Symbol('LayerRepository'); +export const LAYER_SOURCE_CHECK = 'CHK_layer_source_layer_id'; + @Entity('layer') +@Check( + LAYER_SOURCE_CHECK, + `("source" = '${layerSource.sharedLua}' AND "layer_id" IS NOT NULL) OR ("source" = '${layerSource.perLayerJson}' AND "layer_id" IS NULL)` +) export class Layer implements ILayer { + @PrimaryColumn() + public namespace!: string; + @PrimaryColumn({ name: 'layer_name' }) public layerName!: string; - @Column({ name: 'layer_id' }) - public layerId!: number; + @Column({ name: 'layer_id', type: 'integer', nullable: true }) + public layerId!: number | null; + + @Column({ type: 'enum', enum: Object.values(layerSource), enumName: 'layer_source' }) + public source!: LayerSource; @Column() public alias!: string; diff --git a/packages/db/src/entities/property.ts b/packages/db/src/entities/property.ts index 9f72d9d..d6b50b3 100644 --- a/packages/db/src/entities/property.ts +++ b/packages/db/src/entities/property.ts @@ -7,6 +7,9 @@ export const PROPERTY_REPOSITORY_SYMBOL = Symbol('PropertyRepository'); @Entity('property') export class Property { + @PrimaryColumn() + public namespace!: string; + @PrimaryColumn({ name: 'layer_name' }) public layerName!: string; @@ -27,6 +30,9 @@ export class Property { public possibleValues?: EnumValue[]; @ManyToOne(() => Layer, (layer) => layer.properties) - @JoinColumn({ name: 'layer_name', referencedColumnName: 'layerName' }) + @JoinColumn([ + { name: 'namespace', referencedColumnName: 'namespace' }, + { name: 'layer_name', referencedColumnName: 'layerName' }, + ]) public layerRelation!: Layer; } diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 5cc38c5..e0807f8 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -1,8 +1,9 @@ -export { Layer, LAYER_REPOSITORY_SYMBOL } from './entities/layer'; +export { Layer, LAYER_REPOSITORY_SYMBOL, LAYER_SOURCE_CHECK } from './entities/layer'; export type { Layer as ILayer, LayerSpec, LayerSummary } from './types/layer'; +export type { NamespaceSummary } from './types/namespace'; export type { Property as PropertySpec } from './types/property'; export { Property, PROPERTY_REPOSITORY_SYMBOL } from './entities/property'; export { EnumValue, ENUMS_REPOSITORY_SYMBOL } from './entities/enumValue'; -export { ColumnType, columnType } from './types/enums'; -export { DATA_SOURCE_PROVIDER, createDataSourceOptions, createDataSourceHealthCheck, createSslOptions } from './db/connection'; +export { ColumnType, columnType, LayerSource, layerSource } from './types/enums'; +export { DATA_SOURCE_PROVIDER, createDataSource, createDataSourceOptions, createDataSourceHealthCheck, createSslOptions } from './db/connection'; export { DbConfig } from './interfaces'; diff --git a/packages/db/src/interfaces.ts b/packages/db/src/interfaces.ts index d52dbec..9eb298c 100644 --- a/packages/db/src/interfaces.ts +++ b/packages/db/src/interfaces.ts @@ -1,8 +1,7 @@ -import type { DataSourceOptions } from 'typeorm'; import type { ConfigType } from './db/config'; export type DbConfig = ReturnType['db']; export type SslConfig = DbConfig['ssl']; -export type SslOptions = Extract['ssl']; +export type SslOptions = false | { key?: Buffer; cert?: Buffer; ca?: Buffer }; diff --git a/packages/db/src/migrations/1786885317922-Migration.ts b/packages/db/src/migrations/1786885317922-Migration.ts new file mode 100644 index 0000000..c5f4330 --- /dev/null +++ b/packages/db/src/migrations/1786885317922-Migration.ts @@ -0,0 +1,61 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class Migration1786885317922 implements MigrationInterface { + name = 'Migration1786885317922'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "enum_value" DROP CONSTRAINT "FK_8548a0384cc733008852a8f4e70"`); + await queryRunner.query(`ALTER TABLE "property" DROP CONSTRAINT "FK_34ce6cedb279510bb20da63cf21"`); + await queryRunner.query(`ALTER TABLE "enum_value" ADD "namespace" character varying NOT NULL`); + await queryRunner.query(`ALTER TABLE "enum_value" DROP CONSTRAINT "PK_08cf35141adc487a902990ccff2"`); + await queryRunner.query( + `ALTER TABLE "enum_value" ADD CONSTRAINT "PK_0607639cc6d4c22cad2fa8552e9" PRIMARY KEY ("value", "layer_name", "property", "namespace")` + ); + await queryRunner.query(`ALTER TABLE "layer" ADD "namespace" character varying NOT NULL`); + await queryRunner.query(`ALTER TABLE "layer" DROP CONSTRAINT "PK_ee22a436b5250ea132444cd9baa"`); + await queryRunner.query(`ALTER TABLE "layer" ADD CONSTRAINT "PK_0080403b718d78e8dc9590e56ad" PRIMARY KEY ("layer_name", "namespace")`); + await queryRunner.query(`CREATE TYPE "public"."layer_source" AS ENUM('sharedLua', 'perLayerJson')`); + await queryRunner.query(`ALTER TABLE "layer" ADD "source" "public"."layer_source" NOT NULL`); + await queryRunner.query(`ALTER TABLE "property" ADD "namespace" character varying NOT NULL`); + await queryRunner.query(`ALTER TABLE "property" DROP CONSTRAINT "PK_c92abce6639221fa0e46b3d1380"`); + await queryRunner.query( + `ALTER TABLE "property" ADD CONSTRAINT "PK_f92b1edaa8f549c489fc2de4a05" PRIMARY KEY ("layer_name", "property", "namespace")` + ); + await queryRunner.query(`ALTER TABLE "layer" ALTER COLUMN "layer_id" DROP NOT NULL`); + await queryRunner.query( + `ALTER TABLE "layer" ADD CONSTRAINT "CHK_layer_source_layer_id" CHECK (("source" = 'sharedLua' AND "layer_id" IS NOT NULL) OR ("source" = 'perLayerJson' AND "layer_id" IS NULL))` + ); + await queryRunner.query( + `ALTER TABLE "enum_value" ADD CONSTRAINT "FK_10b4583c2705fa3454d63f05534" FOREIGN KEY ("namespace", "layer_name", "property") REFERENCES "property"("namespace","layer_name","property") ON DELETE CASCADE ON UPDATE NO ACTION` + ); + await queryRunner.query( + `ALTER TABLE "property" ADD CONSTRAINT "FK_d4a43c70a881756a56e51b86b84" FOREIGN KEY ("namespace", "layer_name") REFERENCES "layer"("namespace","layer_name") ON DELETE NO ACTION ON UPDATE NO ACTION` + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "property" DROP CONSTRAINT "FK_d4a43c70a881756a56e51b86b84"`); + await queryRunner.query(`ALTER TABLE "enum_value" DROP CONSTRAINT "FK_10b4583c2705fa3454d63f05534"`); + await queryRunner.query(`ALTER TABLE "layer" DROP CONSTRAINT "CHK_layer_source_layer_id"`); + await queryRunner.query(`ALTER TABLE "layer" ALTER COLUMN "layer_id" SET NOT NULL`); + await queryRunner.query(`ALTER TABLE "property" DROP CONSTRAINT "PK_f92b1edaa8f549c489fc2de4a05"`); + await queryRunner.query(`ALTER TABLE "property" ADD CONSTRAINT "PK_c92abce6639221fa0e46b3d1380" PRIMARY KEY ("layer_name", "property")`); + await queryRunner.query(`ALTER TABLE "property" DROP COLUMN "namespace"`); + await queryRunner.query(`ALTER TABLE "layer" DROP COLUMN "source"`); + await queryRunner.query(`DROP TYPE "public"."layer_source"`); + await queryRunner.query(`ALTER TABLE "layer" DROP CONSTRAINT "PK_0080403b718d78e8dc9590e56ad"`); + await queryRunner.query(`ALTER TABLE "layer" ADD CONSTRAINT "PK_ee22a436b5250ea132444cd9baa" PRIMARY KEY ("layer_name")`); + await queryRunner.query(`ALTER TABLE "layer" DROP COLUMN "namespace"`); + await queryRunner.query(`ALTER TABLE "enum_value" DROP CONSTRAINT "PK_0607639cc6d4c22cad2fa8552e9"`); + await queryRunner.query( + `ALTER TABLE "enum_value" ADD CONSTRAINT "PK_08cf35141adc487a902990ccff2" PRIMARY KEY ("value", "layer_name", "property")` + ); + await queryRunner.query(`ALTER TABLE "enum_value" DROP COLUMN "namespace"`); + await queryRunner.query( + `ALTER TABLE "property" ADD CONSTRAINT "FK_34ce6cedb279510bb20da63cf21" FOREIGN KEY ("layer_name") REFERENCES "layer"("layer_name") ON DELETE NO ACTION ON UPDATE NO ACTION` + ); + await queryRunner.query( + `ALTER TABLE "enum_value" ADD CONSTRAINT "FK_8548a0384cc733008852a8f4e70" FOREIGN KEY ("layer_name", "property") REFERENCES "property"("layer_name","property") ON DELETE CASCADE ON UPDATE NO ACTION` + ); + } +} diff --git a/packages/db/src/openapi.d.ts b/packages/db/src/openapi.d.ts index c6916e9..71613fd 100644 --- a/packages/db/src/openapi.d.ts +++ b/packages/db/src/openapi.d.ts @@ -4,14 +4,31 @@ import type { TypedRequestHandlers as ImportedTypedRequestHandlers } from '@map-colonies/openapi-helpers/typedRequestHandler'; export type paths = { - '/layers': { + '/namespaces': { parameters: { query?: never; header?: never; path?: never; cookie?: never; }; - /** Get all layer names */ + /** Get all namespaces */ + get: operations['getNamespaces']; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + '/namespaces/{namespace}/layers': { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get all layer names within a namespace */ get: operations['getLayers']; put?: never; post?: never; @@ -21,14 +38,14 @@ export type paths = { patch?: never; trace?: never; }; - '/layers/{layerName}': { + '/namespaces/{namespace}/layers/{layerName}': { parameters: { query?: never; header?: never; path?: never; cookie?: never; }; - /** Get layer spec by name */ + /** Get layer spec by name within a namespace */ get: operations['getLayerByName']; put?: never; post?: never; @@ -42,6 +59,18 @@ export type paths = { export type webhooks = Record; export type components = { schemas: { + /** @description A namespace served by this deployment. */ + NamespaceSummary: { + /** + * @description Name of the namespace + * @example data-2025 + */ + name: string; + }; + /** @description Response wrapper for the list of namespaces. */ + NamespaceList: { + namespaces: components['schemas']['NamespaceSummary'][]; + }; /** @description Basic layer info returned in the list endpoint. */ LayerSummary: { /** @@ -103,6 +132,27 @@ export type components = { }; }; responses: { + /** @description A list of the namespaces this deployment serves */ + NamespaceListResponse: { + headers: { + [name: string]: unknown; + }; + content: { + /** + * @example { + * "namespaces": [ + * { + * "name": "data-2024" + * }, + * { + * "name": "data-2025" + * } + * ] + * } + */ + 'application/json': components['schemas']['NamespaceList']; + }; + }; /** @description A list of layers with their display aliases */ LayerListResponse: { headers: { @@ -175,6 +225,8 @@ export type components = { }; }; parameters: { + /** @description The namespace the layer belongs to. Layer names are only unique within a namespace */ + NamespaceParam: string; /** @description The name of the layer (matches `Property.layerName`) */ LayerNameParam: string; }; @@ -184,7 +236,7 @@ export type components = { }; export type $defs = Record; export interface operations { - getLayers: { + getNamespaces: { parameters: { query?: never; header?: never; @@ -192,6 +244,21 @@ export interface operations { cookie?: never; }; requestBody?: never; + responses: { + 200: components['responses']['NamespaceListResponse']; + }; + }; + getLayers: { + parameters: { + query?: never; + header?: never; + path: { + /** @description The namespace the layer belongs to. Layer names are only unique within a namespace */ + namespace: components['parameters']['NamespaceParam']; + }; + cookie?: never; + }; + requestBody?: never; responses: { 200: components['responses']['LayerListResponse']; 404: components['responses']['NotFound']; @@ -202,6 +269,8 @@ export interface operations { query?: never; header?: never; path: { + /** @description The namespace the layer belongs to. Layer names are only unique within a namespace */ + namespace: components['parameters']['NamespaceParam']; /** @description The name of the layer (matches `Property.layerName`) */ layerName: components['parameters']['LayerNameParam']; }; diff --git a/packages/db/src/types/enums.ts b/packages/db/src/types/enums.ts index ef26b74..2d0ab34 100644 --- a/packages/db/src/types/enums.ts +++ b/packages/db/src/types/enums.ts @@ -14,3 +14,10 @@ export const columnType = { } as const; export type ColumnType = (typeof columnType)[keyof typeof columnType]; + +export const layerSource = { + sharedLua: 'sharedLua', + perLayerJson: 'perLayerJson', +} as const; + +export type LayerSource = (typeof layerSource)[keyof typeof layerSource]; diff --git a/packages/db/src/types/namespace.ts b/packages/db/src/types/namespace.ts new file mode 100644 index 0000000..57b6be8 --- /dev/null +++ b/packages/db/src/types/namespace.ts @@ -0,0 +1,3 @@ +import type { components } from '../openapi'; + +export type NamespaceSummary = components['schemas']['NamespaceSummary']; diff --git a/packages/synchronizer/config/default.json b/packages/synchronizer/config/default.json index 9067f03..34d28d4 100644 --- a/packages/synchronizer/config/default.json +++ b/packages/synchronizer/config/default.json @@ -3,27 +3,11 @@ "offlineMode": true, "name": "standard-synchronizer" }, - "layersFile": "./config/layers.json", "aliasesFile": "./config/aliases.json", "typeMapFile": "./config/typeMap.json", "schedule": "*/1 * * * *", "indexNameFormat": "{layerName}_{column}_idx", - "enrichment": { - "enabled": false - }, "dbs": { - "source": { - "type": "postgres", - "host": "localhost", - "port": 5432, - "username": "postgres", - "password": "postgres", - "database": "postgres", - "ssl": { - "enabled": false - }, - "schema": "query" - }, "destination": { "type": "postgres", "host": "localhost", @@ -41,15 +25,39 @@ "accessKeyId": "key", "secretAccessKey": "keySecret", "forcePathStyle": true, - "region": "local", - "bucket": "vector-query-example", - "fileName": "mock-query.lua", - "layersVariable": "layers", - "aliasFieldName": "alias", - "idFieldName": "id", - "nameFieldName": "name" + "region": "local" } }, + "namespaces": [ + { + "name": "default", + "bucket": "vector-query-example", + "layersFile": "./config/layers/default.json", + "db": { + "type": "postgres", + "host": "localhost", + "port": 5432, + "username": "postgres", + "password": "postgres", + "database": "postgres", + "ssl": { + "enabled": false + }, + "schema": "query" + }, + "layerSource": { + "type": "sharedLua", + "fileName": "mock-query.lua", + "layersVariable": "layers", + "aliasFieldName": "alias", + "idFieldName": "id", + "nameFieldName": "name", + "enrichment": { + "enabled": false + } + } + } + ], "openapiConfig": { "filePath": "./openapi3.yaml", "basePath": "/docs", diff --git a/packages/synchronizer/config/layers.json b/packages/synchronizer/config/layers/default.json similarity index 100% rename from packages/synchronizer/config/layers.json rename to packages/synchronizer/config/layers/default.json diff --git a/packages/synchronizer/config/layers/test.json b/packages/synchronizer/config/layers/test.json new file mode 100644 index 0000000..23709ec --- /dev/null +++ b/packages/synchronizer/config/layers/test.json @@ -0,0 +1,7 @@ +[ + { + "layerName": "buildings_polygon", + "excludeProperties": ["id"], + "enums": ["code", "attribute"] + } +] diff --git a/packages/synchronizer/config/test.json b/packages/synchronizer/config/test.json index 1884e75..8d2d4ee 100644 --- a/packages/synchronizer/config/test.json +++ b/packages/synchronizer/config/test.json @@ -1,17 +1,41 @@ { "dbs": { - "source": { - "schema": "test_source" - }, "destination": { "schema": "test_dest" } }, - "enrichmentApi": { - "enabled": true, - "api": "https://example.com/{layerName}", - "propertiesPath": "fields_list", - "aliasField": "display_name", - "requestTimeoutMilliseconds": 10000 - } + "namespaces": [ + { + "name": "test", + "bucket": "vector-query-example", + "layersFile": "./config/layers/test.json", + "db": { + "type": "postgres", + "host": "localhost", + "port": 5432, + "username": "postgres", + "password": "postgres", + "database": "postgres", + "ssl": { + "enabled": false + }, + "schema": "test_source" + }, + "layerSource": { + "type": "sharedLua", + "fileName": "mock-query.lua", + "layersVariable": "layers", + "aliasFieldName": "alias", + "idFieldName": "id", + "nameFieldName": "name", + "enrichment": { + "enabled": true, + "api": "https://example.com/{layerName}", + "propertiesPath": "fields_list", + "aliasField": "display_name", + "requestTimeoutMilliseconds": 10000 + } + } + } + ] } diff --git a/packages/synchronizer/package.json b/packages/synchronizer/package.json index 776d0d0..9e8ff0c 100644 --- a/packages/synchronizer/package.json +++ b/packages/synchronizer/package.json @@ -9,7 +9,7 @@ "start": "npm run build && node dist/src/index.js", "start:dev": "npm run build && cd dist && cross-env CONFIG_OFFLINE_MODE=true node --enable-source-maps --import ./instrumentation.mjs ./src/index.js", "dev": "ts-node src/index.ts", - "assets:copy": "copyfiles -f ./config/* ./dist/config && copyfiles -f copyfiles ./package.json dist", + "assets:copy": "copyfiles -u 1 \"./config/**/*\" ./dist/config && copyfiles -f copyfiles ./package.json dist", "watch": "ts-node-dev --respawn src/index.ts", "test:integration": "vitest run --coverage.enabled=false --project integration", "test": "vitest run", @@ -32,6 +32,7 @@ "@map-colonies/tracing-utils": "^1.0.0", "@opentelemetry/api": "^1.9.1", "@smithy/node-http-handler": "^4.7.6", + "ajv": "^8.20.0", "axios": "^1.7.0", "express": "^5.2.1", "node-cron": "^4.2.1", @@ -44,7 +45,7 @@ }, "devDependencies": { "@types/nock": "^10.0.3", - "@types/node": "^20.11.0", + "@types/node": "^24.0.0", "@types/node-cron": "^3.0.11", "@vitest/coverage-v8": "^4.0.18", "@vitest/ui": "^4.0.18", diff --git a/packages/synchronizer/src/common/config.ts b/packages/synchronizer/src/common/config.ts index b472e6d..17bd753 100644 --- a/packages/synchronizer/src/common/config.ts +++ b/packages/synchronizer/src/common/config.ts @@ -1,8 +1,8 @@ import { type ConfigInstance, config } from '@map-colonies/config'; -import { vectorVectorStandardSynchronizerV3, type vectorVectorStandardSynchronizerV3Type } from '@map-colonies/schemas'; +import { vectorVectorStandardSynchronizerV4, type vectorVectorStandardSynchronizerV4Type } from '@map-colonies/schemas'; // Choose here the type of the config instance and import this type from the entire application -type ConfigType = ConfigInstance; +type ConfigType = ConfigInstance; let configInstance: ConfigType | undefined; @@ -13,7 +13,7 @@ let configInstance: ConfigType | undefined; */ async function initConfig(offlineMode?: boolean): Promise { configInstance = await config({ - schema: vectorVectorStandardSynchronizerV3, + schema: vectorVectorStandardSynchronizerV4, offlineMode, }); } diff --git a/packages/synchronizer/src/common/constants.ts b/packages/synchronizer/src/common/constants.ts index 0e6c984..cf85c5c 100644 --- a/packages/synchronizer/src/common/constants.ts +++ b/packages/synchronizer/src/common/constants.ts @@ -20,10 +20,7 @@ export const HEALTHCHECK = Symbol('HealthCheck'); export const s3ConfigPath = 'dbs.s3'; -export const DB_CONFIGS = [ - { configKey: 'dbs.source', token: SOURCE_DATA_SOURCE_PROVIDER }, - { configKey: 'dbs.destination', token: DESTINATION_DATA_SOURCE_PROVIDER }, -] as const; +export const DESTINATION_DB_CONFIG_PATH = 'dbs.destination'; export const REPOSITORIES = [ { entity: Layer, token: LAYER_REPOSITORY_SYMBOL }, @@ -41,3 +38,7 @@ export const SERVICES = { CLEANUP_REGISTRY: Symbol('CleanupRegistry'), } satisfies Record; /* eslint-enable @typescript-eslint/naming-convention */ + +export { DESTINATION_DATA_SOURCE_PROVIDER }; + +export const NAMESPACE_HANDLES = Symbol('NamespaceHandles'); diff --git a/packages/synchronizer/src/common/db/connection.ts b/packages/synchronizer/src/common/db/connection.ts deleted file mode 100644 index 3ae1624..0000000 --- a/packages/synchronizer/src/common/db/connection.ts +++ /dev/null @@ -1,22 +0,0 @@ -import type { FactoryFunction, DependencyContainer } from 'tsyringe'; -import { DataSource } from 'typeorm'; -import { createDataSourceOptions, createDataSourceHealthCheck, type DbConfig } from '@db'; -import type { HealthCheck } from '@godaddy/terminus'; -import { DB_CONFIGS, SERVICE_NAME, SERVICES } from '../constants'; -import type { ConfigType } from '../config'; - -type DbConfigKey = (typeof DB_CONFIGS)[number]['configKey']; - -export const createDataSourceFactory = - (configKey: DbConfigKey): FactoryFunction => - (container: DependencyContainer): DataSource => { - const config = container.resolve(SERVICES.CONFIG); - const dbConfig: DbConfig = config.get(configKey); - return new DataSource(createDataSourceOptions(dbConfig, SERVICE_NAME)); - }; - -export const healthCheckFactory: FactoryFunction = (container: DependencyContainer): HealthCheck => - createDataSourceHealthCheck( - container, - DB_CONFIGS.map(({ token }) => token) - ); diff --git a/packages/synchronizer/src/common/interfaces.ts b/packages/synchronizer/src/common/interfaces.ts index 9dd65fd..997d12d 100644 --- a/packages/synchronizer/src/common/interfaces.ts +++ b/packages/synchronizer/src/common/interfaces.ts @@ -7,6 +7,12 @@ export interface LayerEnums { layerName: string; enums: string[]; excludeProperties?: string[]; + sourceKey?: string; } -export type EnrichmentConfig = ReturnType['enrichment']; +export type NamespacesConfig = ReturnType['namespaces']; +export type NamespaceConfig = NamespacesConfig[number]; +export type LayerSourceConfig = NamespaceConfig['layerSource']; +export type SharedLuaSourceConfig = Extract; +export type PerLayerJsonSourceConfig = Extract; +export type EnrichmentConfig = SharedLuaSourceConfig['enrichment']; diff --git a/packages/synchronizer/src/common/s3/interfaces.ts b/packages/synchronizer/src/common/s3/interfaces.ts index c3e0e53..7f1bb0d 100644 --- a/packages/synchronizer/src/common/s3/interfaces.ts +++ b/packages/synchronizer/src/common/s3/interfaces.ts @@ -1,6 +1,6 @@ import type { S3ClientConfig } from '@aws-sdk/client-s3'; -import { type vectorVectorStandardSynchronizerV1Type } from '@map-colonies/schemas'; +import { type vectorVectorStandardSynchronizerV4Type } from '@map-colonies/schemas'; -export type BaseS3Config = vectorVectorStandardSynchronizerV1Type['dbs']['s3']; +export type BaseS3Config = vectorVectorStandardSynchronizerV4Type['dbs']['s3']; export type S3Config = BaseS3Config & S3ClientConfig; diff --git a/packages/synchronizer/src/common/s3/s3Repository.ts b/packages/synchronizer/src/common/s3/s3Repository.ts index e14a4bd..610b2d5 100644 --- a/packages/synchronizer/src/common/s3/s3Repository.ts +++ b/packages/synchronizer/src/common/s3/s3Repository.ts @@ -8,8 +8,7 @@ import { GetObjectCommand } from '@aws-sdk/client-s3'; import { context as contextAPI } from '@opentelemetry/api'; import { startActivePromisifiedSpan } from '@common/tracing/util'; import { S3Attributes, S3SpanName } from '@common/tracing/s3'; -import { SERVICES, s3ConfigPath } from '@common/constants'; -import type { ConfigType } from '../config'; +import { SERVICES } from '@common/constants'; import { FsRepository } from '../fs/fsRepository'; @injectable() @@ -17,7 +16,6 @@ import { FsRepository } from '../fs/fsRepository'; export class S3Repository { public constructor( @inject(SERVICES.S3_CLIENT) private readonly s3Client: S3Client, - @inject(SERVICES.CONFIG) private readonly config: ConfigType, @inject(SERVICES.LOGGER) private readonly logger: Logger, private readonly fsRepository: FsRepository ) {} @@ -42,20 +40,18 @@ export class S3Repository { ); } - public async downloadFile(): Promise { - const { bucket, fileName } = this.config.get(s3ConfigPath); - + public async downloadFile(bucket: string, fileName: string): Promise { return startActivePromisifiedSpan( S3SpanName.S3_DOWNLOAD_FILE, { [S3Attributes.BUCKET]: bucket, [S3Attributes.FILE_NAME]: fileName }, contextAPI.active(), async () => { try { - this.logger.info(`Downloading ${fileName} file from S3`); + this.logger.info(`Downloading ${fileName} file from S3 bucket ${bucket}`); const body = await this.getObjectWrapper(bucket, fileName); - const filePath = path.join(__dirname, 'downloads', fileName); + const filePath = path.join(__dirname, 'downloads', bucket, fileName); await this.fsRepository.mkdir(path.dirname(filePath)); await this.fsRepository.writeFile(filePath, await buffer(body)); diff --git a/packages/synchronizer/src/common/tracing/sync.ts b/packages/synchronizer/src/common/tracing/sync.ts index 0458768..3209699 100644 --- a/packages/synchronizer/src/common/tracing/sync.ts +++ b/packages/synchronizer/src/common/tracing/sync.ts @@ -15,6 +15,7 @@ export type SyncSpanName = (typeof SyncSpanName)[keyof typeof SyncSpanName]; export const SyncAttributes = { LAYER_NAME: 'layer.name', LAYER_ID: 'layer.id', + NAMESPACE: 'namespace', LAYERS_COUNT: 'layers.count', LAYERS_CHANGED: 'layers.changed', PROPERTIES_AFFECTED: 'properties.affected', diff --git a/packages/synchronizer/src/containerConfig.ts b/packages/synchronizer/src/containerConfig.ts index 6fc99fb..9d7da06 100644 --- a/packages/synchronizer/src/containerConfig.ts +++ b/packages/synchronizer/src/containerConfig.ts @@ -4,19 +4,21 @@ import { Registry } from 'prom-client'; import type { DependencyContainer } from 'tsyringe/dist/typings/types'; import type { Logger } from '@map-colonies/js-logger'; import { jsLogger } from '@map-colonies/js-logger'; -import { instancePerContainerCachingFactory } from 'tsyringe'; +import { Lifecycle, instancePerContainerCachingFactory } from 'tsyringe'; import { CleanupRegistry } from '@map-colonies/cleanup-registry'; import type { DataSource, Repository } from 'typeorm'; -import { DATA_SOURCE_PROVIDER as DESTINATION_DATA_SOURCE_PROVIDER } from '@db'; +import type { HealthCheck } from '@godaddy/terminus'; +import { DATA_SOURCE_PROVIDER as DESTINATION_DATA_SOURCE_PROVIDER, createDataSource, createDataSourceHealthCheck } from '@db'; import type { S3Client } from '@aws-sdk/client-s3'; import { ListBucketsCommand } from '@aws-sdk/client-s3'; import { type ConfigType, getConfig } from '@common/config'; import { type InjectionObject, registerDependencies } from '@common/dependencyRegistration'; -import { DB_CONFIGS, HEALTHCHECK, ON_SIGNAL, REPOSITORIES, SERVICES, SERVICE_NAME } from '@common/constants'; +import { DESTINATION_DB_CONFIG_PATH, HEALTHCHECK, NAMESPACE_HANDLES, ON_SIGNAL, REPOSITORIES, SERVICES, SERVICE_NAME } from '@common/constants'; import { getTracing } from '@common/tracing'; -import { createDataSourceFactory, healthCheckFactory } from './common/db/connection'; import { CRON_MANAGER_SYMBOL, CronManager } from './sync/cron'; import { s3ClientFactory } from './common/s3'; +import { S3Repository } from './common/s3/s3Repository'; +import { createNamespaceHandles } from './sync/namespaceHandle'; export interface RegisterOptions { override?: InjectionObject[]; @@ -92,26 +94,34 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise }); }, }, - ...DB_CONFIGS.map(({ configKey, token }) => { - return { - token, - provider: { - useFactory: instancePerContainerCachingFactory(createDataSourceFactory(configKey)), - }, - postInjectionHook: async (deps: DependencyContainer): Promise => { - const dataSource = deps.resolve(token); + { + token: S3Repository, + provider: { useClass: S3Repository }, + options: { lifecycle: Lifecycle.Singleton }, + }, + { + token: DESTINATION_DATA_SOURCE_PROVIDER, + provider: { + useFactory: instancePerContainerCachingFactory((container) => { + const config = container.resolve(SERVICES.CONFIG); + return createDataSource(config.get(DESTINATION_DB_CONFIG_PATH), SERVICE_NAME); + }), + }, + postInjectionHook: async (deps: DependencyContainer): Promise => { + const dataSource = deps.resolve(DESTINATION_DATA_SOURCE_PROVIDER); - if (!dataSource.isInitialized) { - await dataSource.initialize(); - } + if (!dataSource.isInitialized) { + await dataSource.initialize(); + } - cleanupRegistry.register({ - id: token, - func: dataSource.destroy.bind(dataSource), - }); - }, - }; - }), + deps.register(DESTINATION_DATA_SOURCE_PROVIDER, { useValue: dataSource }); + + cleanupRegistry.register({ + id: DESTINATION_DATA_SOURCE_PROVIDER, + func: dataSource.destroy.bind(dataSource), + }); + }, + }, ...REPOSITORIES.map(({ token, entity }) => ({ token, provider: { @@ -121,6 +131,12 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise }, }, })), + { + token: NAMESPACE_HANDLES, + provider: { + useFactory: instancePerContainerCachingFactory((container) => createNamespaceHandles(container, cleanupRegistry)), + }, + }, { token: CRON_MANAGER_SYMBOL, provider: { useClass: CronManager }, @@ -136,7 +152,7 @@ export const registerExternalValues = async (options?: RegisterOptions): Promise { token: HEALTHCHECK, provider: { - useFactory: healthCheckFactory, + useFactory: (container: DependencyContainer): HealthCheck => createDataSourceHealthCheck(container, [DESTINATION_DATA_SOURCE_PROVIDER]), }, }, ]; diff --git a/packages/synchronizer/src/sync/SyncModel.ts b/packages/synchronizer/src/sync/SyncModel.ts index 7de50f4..c5e4627 100644 --- a/packages/synchronizer/src/sync/SyncModel.ts +++ b/packages/synchronizer/src/sync/SyncModel.ts @@ -1,16 +1,14 @@ -import { readFile } from 'node:fs/promises'; import { inject, injectable } from 'tsyringe'; import { DataSource, Repository } from 'typeorm'; -import { ENUMS_REPOSITORY_SYMBOL, EnumValue, Layer, LAYER_REPOSITORY_SYMBOL, Property, PROPERTY_REPOSITORY_SYMBOL } from '@db'; +import { ENUMS_REPOSITORY_SYMBOL, EnumValue, Layer, LAYER_REPOSITORY_SYMBOL, LayerSource, Property, PROPERTY_REPOSITORY_SYMBOL } from '@db'; import { Logger } from '@map-colonies/js-logger'; import { context as contextAPI } from '@opentelemetry/api'; import { startActivePromisifiedSpan } from '@common/tracing/util'; import { SyncAttributes, SyncSpanName } from '@common/tracing/sync'; -import { SERVICES, SOURCE_DATA_SOURCE_PROVIDER, s3ConfigPath } from '@common/constants'; +import { SERVICES, SOURCE_DATA_SOURCE_PROVIDER } from '@common/constants'; import { ConfigType } from '@common/config'; import { InsertPropertyDTO, LayerEnums } from '@src/common/interfaces'; -import { S3Repository } from '@src/common/s3/s3Repository'; import { CaseInsensitiveMap } from '@src/common/caseInsensitiveMap'; import { columnInfosToProperties, excludedPropertySet, schemaOf } from './helpers'; import { @@ -24,10 +22,8 @@ import { StalePropertiesDeletionError, TableColumnsQueryError, } from './errors'; -import { fetchPropertyAliases } from './aliasEnricher'; -import type { FileAliases } from './fileReader'; +import { resolveFileAliases, type FileAliases } from './fileReader'; import type { TypeMap } from './typeMap'; -import { LuaLayer, parseLuaLayers } from './luaParser'; export interface ColumnInfo { columnName: string; @@ -42,25 +38,24 @@ export class SyncModel { @inject(SOURCE_DATA_SOURCE_PROVIDER) private readonly sourceDataSource: DataSource, @inject(LAYER_REPOSITORY_SYMBOL) private readonly layerRepository: Repository, @inject(PROPERTY_REPOSITORY_SYMBOL) private readonly propertyRepository: Repository, - @inject(ENUMS_REPOSITORY_SYMBOL) private readonly enumsRepository: Repository, - private readonly luaRepository: S3Repository + @inject(ENUMS_REPOSITORY_SYMBOL) private readonly enumsRepository: Repository ) {} - public async syncLayer(layerName: string, layerId: number, alias?: string): Promise { + public async syncLayer(namespace: string, layerName: string, layerId: number | null, source: LayerSource, alias?: string): Promise { return startActivePromisifiedSpan( SyncSpanName.SYNC_LAYER, - { [SyncAttributes.LAYER_NAME]: layerName, [SyncAttributes.LAYER_ID]: layerId }, + { [SyncAttributes.LAYER_NAME]: layerName, [SyncAttributes.NAMESPACE]: namespace }, contextAPI.active(), async () => { try { await this.layerRepository .createQueryBuilder() .insert() - .values({ layerName, layerId, alias: alias ?? layerName }) - .orIgnore() + .values({ namespace, layerName, layerId, source, alias: alias ?? layerName }) + .orUpdate(['layer_id', 'source', 'alias'], ['namespace', 'layer_name']) .execute(); } catch (err) { - this.logger.error({ msg: `Failed to sync layer ${layerName} (${layerId})`, err }); + this.logger.error({ msg: `Failed to sync layer ${namespace}/${layerName}`, err }); throw new LayerSyncError(layerName, layerId, err); } } @@ -102,7 +97,6 @@ export class SyncModel { const schema = schemaOf(this.sourceDataSource); const qualifiedTable = `"${schema}"."${layerName}"`; - // Recursive CTE function const cteParts = enums.map( (col) => `cte_${col} AS ( SELECT MIN("${col}") AS value FROM ${qualifiedTable} @@ -136,7 +130,7 @@ export class SyncModel { public async upsertProperties(properties: InsertPropertyDTO[]): Promise { return startActivePromisifiedSpan(SyncSpanName.UPSERT_PROPERTIES, {}, contextAPI.active(), async (span) => { try { - const result = await this.propertyRepository.upsert(properties, ['layerName', 'property']); + const result = await this.propertyRepository.upsert(properties, ['namespace', 'layerName', 'property']); span.setAttribute(SyncAttributes.PROPERTIES_AFFECTED, result.identifiers.length); return result.identifiers.length; } catch (err) { @@ -146,10 +140,16 @@ export class SyncModel { }); } - public async syncProperties(layer: LayerEnums, layerId: number, typeMap: TypeMap, fileAliases: FileAliases): Promise { + public async syncProperties( + namespace: string, + layer: LayerEnums, + typeMap: TypeMap, + fileAliases: FileAliases, + sourceAliases: Map + ): Promise { return startActivePromisifiedSpan( SyncSpanName.SYNC_PROPERTIES, - { [SyncAttributes.LAYER_NAME]: layer.layerName, [SyncAttributes.LAYER_ID]: layerId }, + { [SyncAttributes.LAYER_NAME]: layer.layerName, [SyncAttributes.NAMESPACE]: namespace }, contextAPI.active(), async (span) => { const excluded = excludedPropertySet(layer.excludeProperties); @@ -158,25 +158,12 @@ export class SyncModel { this.logger.debug({ layerName: layer.layerName, excludeProperties: layer.excludeProperties }, 'Excluding properties from sync'); } - const properties = columnInfosToProperties(columns, layer.layerName, typeMap, (columnName, udtName) => { + const properties = columnInfosToProperties(columns, namespace, layer.layerName, typeMap, (columnName, udtName) => { this.logger.warn({ layerName: layer.layerName, columnName, udtName }, 'Unknown column type, skipping property'); }); - const enrichment = this.config.get('enrichment'); - let apiAliases = new Map(); - if (enrichment.enabled) { - try { - apiAliases = await fetchPropertyAliases(layer.layerName, layerId, enrichment); - } catch (err) { - this.logger.warn( - { layerName: layer.layerName, layerId, err }, - 'Failed to fetch property aliases from enrichment API, continuing without them' - ); - } - } - const globalAliases = fileAliases.get('*') ?? new Map(); - const layerAliases = fileAliases.get(layer.layerName) ?? new Map(); - const aliases = new CaseInsensitiveMap([...apiAliases, ...globalAliases, ...layerAliases]); + const fileLayerAliases = resolveFileAliases(fileAliases, namespace, layer.layerName); + const aliases = new CaseInsensitiveMap([...sourceAliases, ...fileLayerAliases]); for (const property of properties) { const alias = aliases.get(property.property); @@ -192,6 +179,7 @@ export class SyncModel { affected += await this.upsertProperties(withoutAlias); } await this.deleteStaleProperties( + namespace, layer.layerName, properties.map((p) => p.property) ); @@ -201,28 +189,42 @@ export class SyncModel { ); } - public async deleteStaleProperties(layerName: string, currentPropertyNames: string[]): Promise { - const qb = this.propertyRepository.createQueryBuilder().delete().where('"layer_name" = :layerName', { layerName }); + public async deleteStaleProperties(namespace: string, layerName: string, currentPropertyNames: string[]): Promise { + const qb = this.propertyRepository + .createQueryBuilder() + .delete() + .where('"namespace" = :namespace', { namespace }) + .andWhere('"layer_name" = :layerName', { layerName }); if (currentPropertyNames.length > 0) { qb.andWhere('"property" NOT IN (:...properties)', { properties: currentPropertyNames }); } try { await qb.execute(); } catch (err) { - this.logger.error({ msg: `Failed to delete stale properties for ${layerName}`, err }); + this.logger.error({ msg: `Failed to delete stale properties for ${namespace}/${layerName}`, err }); throw new StalePropertiesDeletionError(layerName, err); } } - public async deleteLayersNotIn(layerNames: string[]): Promise { + public async deleteLayersNotIn(namespace: string, layerNames: string[]): Promise { if (layerNames.length === 0) { return; } try { - await this.propertyRepository.createQueryBuilder().delete().where('"layer_name" NOT IN (:...layerNames)', { layerNames }).execute(); - await this.layerRepository.createQueryBuilder().delete().where('"layer_name" NOT IN (:...layerNames)', { layerNames }).execute(); + await this.propertyRepository + .createQueryBuilder() + .delete() + .where('"namespace" = :namespace', { namespace }) + .andWhere('"layer_name" NOT IN (:...layerNames)', { layerNames }) + .execute(); + await this.layerRepository + .createQueryBuilder() + .delete() + .where('"namespace" = :namespace', { namespace }) + .andWhere('"layer_name" NOT IN (:...layerNames)', { layerNames }) + .execute(); } catch (err) { - this.logger.error({ msg: `Failed to delete layers not in [${layerNames.join(', ')}]`, err }); + this.logger.error({ msg: `Failed to delete layers not in [${layerNames.join(', ')}] for namespace ${namespace}`, err }); throw new LayersDeletionError(layerNames, err); } } @@ -247,25 +249,19 @@ export class SyncModel { } } - public async luaFileData(): Promise> { - return startActivePromisifiedSpan(SyncSpanName.LOAD_LUA_DATA, {}, contextAPI.active(), async () => { - this.logger.debug('Loading lua query data'); - const filePath = await this.luaRepository.downloadFile(); - const content = await readFile(filePath, 'utf-8'); - const { layersVariable, idFieldName, nameFieldName, aliasFieldName } = this.config.get(s3ConfigPath); - return parseLuaLayers(content, layersVariable, idFieldName, nameFieldName, aliasFieldName); - }); - } - - public async deleteStaleEnumValues(layerName: string, currentEnums: string[]): Promise { - const qb = this.enumsRepository.createQueryBuilder().delete().where('"layer_name" = :layerName', { layerName }); + public async deleteStaleEnumValues(namespace: string, layerName: string, currentEnums: string[]): Promise { + const qb = this.enumsRepository + .createQueryBuilder() + .delete() + .where('"namespace" = :namespace', { namespace }) + .andWhere('"layer_name" = :layerName', { layerName }); if (currentEnums.length > 0) { qb.andWhere('"property" NOT IN (:...properties)', { properties: currentEnums }); } try { await qb.execute(); } catch (err) { - this.logger.error({ msg: `Failed to delete stale enum values for ${layerName}`, err }); + this.logger.error({ msg: `Failed to delete stale enum values for ${namespace}/${layerName}`, err }); throw new StaleEnumValuesDeletionError(layerName, err); } } @@ -286,7 +282,7 @@ export class SyncModel { return existing; } - public async syncEnum(layer: LayerEnums): Promise { + public async syncEnum(namespace: string, layer: LayerEnums): Promise { return startActivePromisifiedSpan(SyncSpanName.SYNC_ENUM, { [SyncAttributes.LAYER_NAME]: layer.layerName }, contextAPI.active(), async (span) => { const existingLayer: LayerEnums = { layerName: layer.layerName, enums: await this.existingEnumColumns(layer) }; @@ -294,14 +290,14 @@ export class SyncModel { const enumValuesByColumn = await this.getEnumDistinctValues(existingLayer); const entities = [...enumValuesByColumn.entries()].flatMap(([col, values]) => - values.map((value) => this.enumsRepository.create({ layerName: layer.layerName, property: col, value })) + values.map((value) => this.enumsRepository.create({ namespace, layerName: layer.layerName, property: col, value })) ); let affected: number; try { affected = (await this.enumsRepository.save(entities)).length; } catch (err) { - this.logger.error({ msg: `Failed to save enum values for ${layer.layerName}`, err }); + this.logger.error({ msg: `Failed to save enum values for ${namespace}/${layer.layerName}`, err }); throw new EnumSaveError(layer.layerName, err); } span.setAttribute(SyncAttributes.ENUMS_AFFECTED, affected); diff --git a/packages/synchronizer/src/sync/cron.ts b/packages/synchronizer/src/sync/cron.ts index a601e97..34ef1c7 100644 --- a/packages/synchronizer/src/sync/cron.ts +++ b/packages/synchronizer/src/sync/cron.ts @@ -2,83 +2,101 @@ import { inject, injectable } from 'tsyringe'; import type { Logger } from '@map-colonies/js-logger'; import { schedule, type ScheduledTask } from 'node-cron'; import { context as contextAPI } from '@opentelemetry/api'; -import { SERVICES } from '@common/constants'; +import { SERVICES, NAMESPACE_HANDLES } from '@common/constants'; import { ConfigType } from '@common/config'; import { startActivePromisifiedSpan } from '@common/tracing/util'; -import { SyncAttributes, SyncSpanName } from '@common/tracing/sync'; -import { SyncModel } from './SyncModel'; -import { FileReader } from './fileReader'; +import { SyncSpanName } from '@common/tracing/sync'; +import { FileReader, type FileAliases } from './fileReader'; +import type { TypeMap } from './typeMap'; +import type { NamespaceHandle } from './namespaceHandle'; export const CRON_MANAGER_SYMBOL = Symbol('cronManagerSymbol'); @injectable() export class CronManager { private task: ScheduledTask | undefined; - private isRunning = false; - private lastLayersChecksum: string | undefined; + private readonly lastLayersChecksum = new Map(); public constructor( @inject(SERVICES.LOGGER) private readonly logger: Logger, - @inject(SyncModel) private readonly dal: SyncModel, @inject(SERVICES.CONFIG) private readonly config: ConfigType, + @inject(NAMESPACE_HANDLES) private readonly namespaces: NamespaceHandle[], private readonly fileReader: FileReader ) {} public async tick(): Promise { - if (this.isRunning) { - this.logger.warn('previous sync tick still running, skipping'); - return; - } - this.isRunning = true; - try { - await startActivePromisifiedSpan(SyncSpanName.SYNC_TICK, {}, contextAPI.active(), async (span) => { - const { checksum, layers } = await this.fileReader.readLayersWithChecksum(this.config.get('layersFile')); - const layersChanged = checksum !== this.lastLayersChecksum; - span.setAttribute(SyncAttributes.LAYERS_COUNT, layers.length); - span.setAttribute(SyncAttributes.LAYERS_CHANGED, layersChanged); - - const typeMap = await this.fileReader.readTypeMap(this.config.get('typeMapFile')); - const fileAliases = await this.fileReader.readAliases(this.config.get('aliasesFile')); + await startActivePromisifiedSpan(SyncSpanName.SYNC_TICK, {}, contextAPI.active(), async () => { + const typeMap = await this.fileReader.readTypeMap(this.config.get('typeMapFile')); + const fileAliases = await this.fileReader.readAliases(this.config.get('aliasesFile')); - const luaLayersMap = await this.dal.luaFileData(); - for (const layer of layers) { - const luaLayer = luaLayersMap.get(layer.layerName); - if (luaLayer === undefined) { - this.logger.warn({ layerName: layer.layerName }, 'Layer not found in lua file, skipping'); - continue; - } - await this.dal.syncLayer(layer.layerName, luaLayer.layerId, luaLayer.alias); - try { - const affected = await this.dal.syncProperties(layer, luaLayer.layerId, typeMap, fileAliases); - this.logger.info({ layer, affected }, 'synced properties'); - } catch (err) { - this.logger.warn({ layerName: layer.layerName, err }, 'failed to sync properties, skipping to enum sync'); - } - const enumsAffected = await this.dal.syncEnum(layer); - this.logger.info({ layer, enumsAffected }, 'synced enums'); - if (layersChanged) { - await this.dal.deleteStaleEnumValues(layer.layerName, layer.enums); - } + for (const namespace of this.namespaces) { + try { + await this.tickNamespace(namespace, typeMap, fileAliases); + } catch (err) { + this.logger.error({ namespace: namespace.name, err }, 'namespace sync tick failed, continuing with next namespace'); } - if (layersChanged) { - await this.dal.deleteLayersNotIn(layers.map((l) => l.layerName)); - this.lastLayersChecksum = checksum; - } - }); - } catch (err) { - this.logger.error({ err }, 'sync tick failed, waiting for next tick'); - } finally { - this.isRunning = false; - } + } + }); } public start(): void { - this.task = schedule(this.config.get('schedule'), async () => { - await this.tick(); + this.task = schedule( + this.config.get('schedule'), + async () => { + await this.tick(); + }, + { noOverlap: true } + ); + this.task.on('execution:overlap', () => { + this.logger.warn('previous sync tick still running, skipping this execution'); }); } public async stop(): Promise { await this.task?.stop(); } + + private async tickNamespace(namespace: NamespaceHandle, typeMap: TypeMap, fileAliases: FileAliases): Promise { + if (!namespace.sourceDataSource.isInitialized) { + try { + await namespace.sourceDataSource.initialize(); + } catch (err) { + this.logger.warn({ namespace: namespace.name, err }, 'source database unreachable, skipping namespace for this tick'); + return; + } + } + + const { checksum, layers } = await this.fileReader.readLayersWithChecksum(namespace.layersFile); + const layersChanged = checksum !== this.lastLayersChecksum.get(namespace.name); + + const records = await namespace.layerSource.tick(layers); + + for (const layer of layers) { + const record = records.get(layer.layerName); + if (record === undefined) { + continue; + } + + await namespace.dal.syncLayer(namespace.name, layer.layerName, record.layerId ?? null, record.source, record.alias); + try { + const affected = await namespace.dal.syncProperties(namespace.name, layer, typeMap, fileAliases, record.propertyAliases); + this.logger.info({ namespace: namespace.name, layer, affected }, 'synced properties'); + } catch (err) { + this.logger.warn({ namespace: namespace.name, layerName: layer.layerName, err }, 'failed to sync properties, skipping to enum sync'); + } + const enumsAffected = await namespace.dal.syncEnum(namespace.name, layer); + this.logger.info({ namespace: namespace.name, layer, enumsAffected }, 'synced enums'); + if (layersChanged) { + await namespace.dal.deleteStaleEnumValues(namespace.name, layer.layerName, layer.enums); + } + } + + if (layersChanged) { + await namespace.dal.deleteLayersNotIn( + namespace.name, + layers.map((l) => l.layerName) + ); + this.lastLayersChecksum.set(namespace.name, checksum); + } + } } diff --git a/packages/synchronizer/src/sync/errors.ts b/packages/synchronizer/src/sync/errors.ts index 3d76d0b..54b33dd 100644 --- a/packages/synchronizer/src/sync/errors.ts +++ b/packages/synchronizer/src/sync/errors.ts @@ -7,7 +7,7 @@ abstract class SyncError extends Error { } export class LayerSyncError extends SyncError { - public constructor(layerName: string, layerId: number, cause?: unknown) { + public constructor(layerName: string, layerId: number | null, cause?: unknown) { super(`Failed to sync layer ${layerName} (${layerId})`, { cause }); } } @@ -65,3 +65,15 @@ export class EnumSaveError extends SyncError { super(`Failed to save enum values for ${layerName}`, { cause }); } } + +export class LayerJsonFetchError extends SyncError { + public constructor(layerName: string, key: string, cause?: unknown) { + super(`Failed to fetch layer JSON for ${layerName} at key ${key}`, { cause }); + } +} + +export class LayerJsonParseError extends SyncError { + public constructor(layerName: string, key: string, cause?: unknown) { + super(`Failed to parse layer JSON for ${layerName} at key ${key}`, { cause }); + } +} diff --git a/packages/synchronizer/src/sync/fileReader.ts b/packages/synchronizer/src/sync/fileReader.ts index cc1d368..b3709e4 100644 --- a/packages/synchronizer/src/sync/fileReader.ts +++ b/packages/synchronizer/src/sync/fileReader.ts @@ -6,9 +6,21 @@ import type { LayerEnums } from '@common/interfaces'; import { parseTypeMap, type TypeMap } from './typeMap'; import { TypeMapError } from './errors'; -type AliasesFile = Record>; +const ALL = '*'; -export type FileAliases = Map>; +type AliasesFile = Record>>; + +export type FileAliases = Map>>; + +export const resolveFileAliases = (fileAliases: FileAliases, namespace: string, layerName: string): Map => { + const tiers = [ + fileAliases.get(ALL)?.get(ALL), + fileAliases.get(ALL)?.get(layerName), + fileAliases.get(namespace)?.get(ALL), + fileAliases.get(namespace)?.get(layerName), + ]; + return new Map(tiers.filter((tier): tier is Map => tier !== undefined).flatMap((tier) => [...tier])); +}; @injectable() export class FileReader { @@ -38,12 +50,19 @@ export class FileReader { } public async readAliases(filePath: string): Promise { + let parsed: AliasesFile; try { const content = await this.fsRepository.readFile(filePath, 'utf-8'); - const parsed = JSON.parse(content.toString()) as AliasesFile; - return new Map(Object.entries(parsed).map(([layer, props]) => [layer, new Map(Object.entries(props))])); + parsed = JSON.parse(content.toString()) as AliasesFile; } catch (err) { throw new Error(`Failed to read aliases from ${filePath}`, { cause: err }); } + + return new Map( + Object.entries(parsed).map(([namespace, layers]) => [ + namespace, + new Map(Object.entries(layers).map(([layer, props]) => [layer, new Map(Object.entries(props))])), + ]) + ); } } diff --git a/packages/synchronizer/src/sync/helpers.ts b/packages/synchronizer/src/sync/helpers.ts index fad307c..516be7e 100644 --- a/packages/synchronizer/src/sync/helpers.ts +++ b/packages/synchronizer/src/sync/helpers.ts @@ -16,7 +16,6 @@ const normalizeUdtName = (rawUdtName: string, typeMap: TypeMap): ColumnType | un return plain; } - // Strip length/precision modifier e.g. "character varying(255)" → "character varying" const withoutModifier = udtName.replace(/\s*\(\d+(?:,\s*\d+)?\)$/, ''); if (withoutModifier !== udtName) { const stripped = typeMap.types.get(withoutModifier); @@ -52,6 +51,7 @@ export const schemaOf = (dataSource: DataSource): string => { export const columnInfosToProperties = ( columnInfos: ColumnInfo[], + namespace: string, layerName: string, typeMap: TypeMap, onUnknown?: (columnName: string, udtName: string) => void @@ -62,7 +62,7 @@ export const columnInfosToProperties = ( if (type === undefined) { onUnknown?.(columnName, udtName); } else { - result.push({ layerName, property: columnName, type }); + result.push({ namespace, layerName, property: columnName, type }); } } return result; diff --git a/packages/synchronizer/src/sync/layerSource/aliasJsonParser.ts b/packages/synchronizer/src/sync/layerSource/aliasJsonParser.ts new file mode 100644 index 0000000..71ac92f --- /dev/null +++ b/packages/synchronizer/src/sync/layerSource/aliasJsonParser.ts @@ -0,0 +1,39 @@ +import ajvCtor from 'ajv'; +import type { PerLayerJsonSourceConfig } from '@common/interfaces'; +import { CaseInsensitiveMap } from '@src/common/caseInsensitiveMap'; + +const ajv = new ajvCtor(); +const isRecord = ajv.compile>({ type: 'object' }); +const isString = ajv.compile({ type: 'string' }); +const isArray = ajv.compile({ type: 'array' }); + +const isRecordEntry = (value: unknown): value is Record => isRecord(value); + +const buildPropertyAliases = (rawFields: unknown[], config: PerLayerJsonSourceConfig): Map => + new CaseInsensitiveMap( + rawFields + .filter(isRecordEntry) + .map((field) => ({ fieldName: field[config.fieldNameField], aliasFieldName: field[config.aliasFieldNameField] })) + .filter((field): field is { fieldName: string; aliasFieldName: string } => isString(field.fieldName) && isString(field.aliasFieldName)) + .map(({ fieldName, aliasFieldName }): [string, string] => [fieldName, aliasFieldName]) + ); + +export interface ParsedDocument { + alias: string; + propertyAliases: Map; +} + +export const parseDocument = (content: string, config: PerLayerJsonSourceConfig): ParsedDocument => { + const raw: unknown = JSON.parse(content); + if (!isRecord(raw)) { + throw new Error('layer JSON must be an object'); + } + + const alias = raw[config.aliasLayerNameField]; + const fields = raw[config.fieldsField]; + if (!isString(alias) || !isArray(fields)) { + throw new Error(`layer JSON must have a string "${config.aliasLayerNameField}" and an array "${config.fieldsField}"`); + } + + return { alias, propertyAliases: buildPropertyAliases(fields, config) }; +}; diff --git a/packages/synchronizer/src/sync/layerSource/index.ts b/packages/synchronizer/src/sync/layerSource/index.ts new file mode 100644 index 0000000..3e11a8d --- /dev/null +++ b/packages/synchronizer/src/sync/layerSource/index.ts @@ -0,0 +1,20 @@ +import type { Logger } from '@map-colonies/js-logger'; +import type { NamespaceConfig } from '@common/interfaces'; +import type { S3Repository } from '@common/s3/s3Repository'; +import type { FsRepository } from '@common/fs/fsRepository'; +import { SharedLuaSource } from './sharedLuaSource'; +import { PerLayerJsonSource } from './perLayerJsonSource'; +import type { LayerSourceStrategy } from './types'; + +export const createLayerSourceStrategy = ( + namespaceConfig: NamespaceConfig, + s3Repository: S3Repository, + fsRepository: FsRepository, + logger: Logger +): LayerSourceStrategy => { + const { layerSource, bucket } = namespaceConfig; + if (layerSource.type === 'sharedLua') { + return new SharedLuaSource(s3Repository, fsRepository, logger, bucket, layerSource); + } + return new PerLayerJsonSource(s3Repository, fsRepository, logger, bucket, layerSource); +}; diff --git a/packages/synchronizer/src/sync/layerSource/perLayerJsonSource.ts b/packages/synchronizer/src/sync/layerSource/perLayerJsonSource.ts new file mode 100644 index 0000000..0c5c324 --- /dev/null +++ b/packages/synchronizer/src/sync/layerSource/perLayerJsonSource.ts @@ -0,0 +1,66 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { layerSource } from '@db'; +import type { LayerEnums, PerLayerJsonSourceConfig } from '@common/interfaces'; +import type { FsRepository } from '@common/fs/fsRepository'; +import { LayerJsonFetchError, LayerJsonParseError } from '../errors'; +import type { FileDownloader, LayerSourceRecord, LayerSourceStrategy } from './types'; +import { parseDocument } from './aliasJsonParser'; + +const resolveOne = async ( + layer: LayerEnums, + sourceKey: string, + s3Repository: FileDownloader, + fsRepository: FsRepository, + bucket: string, + config: PerLayerJsonSourceConfig, + logger: Logger +): Promise => { + const key = `${config.prefix}${sourceKey}`; + + let filePath: string; + try { + filePath = await s3Repository.downloadFile(bucket, key); + } catch (err) { + logger.warn({ err: new LayerJsonFetchError(layer.layerName, key, err) }, 'Failed to fetch layer JSON, skipping layer'); + return undefined; + } + + try { + const content = (await fsRepository.readFile(filePath, 'utf-8')).toString(); + const document = parseDocument(content, config); + return { + alias: document.alias, + propertyAliases: document.propertyAliases, + source: layerSource.perLayerJson, + }; + } catch (err) { + logger.warn({ err: new LayerJsonParseError(layer.layerName, key, err) }, 'Failed to read or parse layer JSON, skipping layer'); + return undefined; + } +}; + +export class PerLayerJsonSource implements LayerSourceStrategy { + public constructor( + private readonly s3Repository: FileDownloader, + private readonly fsRepository: FsRepository, + private readonly logger: Logger, + private readonly bucket: string, + private readonly config: PerLayerJsonSourceConfig + ) {} + + public async tick(layers: LayerEnums[]): Promise> { + const records = new Map(); + for (const layer of layers) { + if (layer.sourceKey === undefined) { + this.logger.warn({ layerName: layer.layerName }, 'No sourceKey configured for perLayerJson layer, skipping'); + continue; + } + + const record = await resolveOne(layer, layer.sourceKey, this.s3Repository, this.fsRepository, this.bucket, this.config, this.logger); + if (record !== undefined) { + records.set(layer.layerName, record); + } + } + return records; + } +} diff --git a/packages/synchronizer/src/sync/layerSource/sharedLuaSource.ts b/packages/synchronizer/src/sync/layerSource/sharedLuaSource.ts new file mode 100644 index 0000000..8d98c62 --- /dev/null +++ b/packages/synchronizer/src/sync/layerSource/sharedLuaSource.ts @@ -0,0 +1,62 @@ +import type { Logger } from '@map-colonies/js-logger'; +import { layerSource } from '@db'; +import type { LayerEnums, SharedLuaSourceConfig } from '@common/interfaces'; +import type { FsRepository } from '@common/fs/fsRepository'; +import { fetchPropertyAliases } from '../aliasEnricher'; +import { parseLuaLayers, type LuaLayer } from '../luaParser'; +import type { LayerSourceRecord, FileDownloader, LayerSourceStrategy } from './types'; + +const resolveOne = async (layer: LayerEnums, luaLayer: LuaLayer, config: SharedLuaSourceConfig, logger: Logger): Promise => { + let propertyAliases = new Map(); + if (config.enrichment.enabled) { + try { + propertyAliases = await fetchPropertyAliases(layer.layerName, luaLayer.layerId, config.enrichment); + } catch (err) { + logger.warn( + { layerName: layer.layerName, layerId: luaLayer.layerId, err }, + 'Failed to fetch property aliases from enrichment API, continuing without them' + ); + } + } + + return { + layerId: luaLayer.layerId, + alias: luaLayer.alias, + propertyAliases, + source: layerSource.sharedLua, + }; +}; + +export class SharedLuaSource implements LayerSourceStrategy { + public constructor( + private readonly s3Repository: FileDownloader, + private readonly fsRepository: FsRepository, + private readonly logger: Logger, + private readonly bucket: string, + private readonly config: SharedLuaSourceConfig + ) {} + + public async tick(layers: LayerEnums[]): Promise> { + this.logger.debug('Loading lua query data'); + const filePath = await this.s3Repository.downloadFile(this.bucket, this.config.fileName); + const content = (await this.fsRepository.readFile(filePath, 'utf-8')).toString(); + const luaLayers = parseLuaLayers( + content, + this.config.layersVariable, + this.config.idFieldName, + this.config.nameFieldName, + this.config.aliasFieldName + ); + + const records = new Map(); + for (const layer of layers) { + const luaLayer = luaLayers.get(layer.layerName); + if (luaLayer === undefined) { + this.logger.warn({ layerName: layer.layerName }, 'Layer not found in lua file, skipping'); + continue; + } + records.set(layer.layerName, await resolveOne(layer, luaLayer, this.config, this.logger)); + } + return records; + } +} diff --git a/packages/synchronizer/src/sync/layerSource/types.ts b/packages/synchronizer/src/sync/layerSource/types.ts new file mode 100644 index 0000000..693498f --- /dev/null +++ b/packages/synchronizer/src/sync/layerSource/types.ts @@ -0,0 +1,17 @@ +import type { LayerSource } from '@db'; +import type { LayerEnums } from '@common/interfaces'; + +export interface LayerSourceRecord { + layerId?: number; + alias: string; + propertyAliases: Map; + source: LayerSource; +} + +export interface LayerSourceStrategy { + tick: (layers: LayerEnums[]) => Promise>; +} + +export interface FileDownloader { + downloadFile: (bucket: string, key: string) => Promise; +} diff --git a/packages/synchronizer/src/sync/namespaceHandle.ts b/packages/synchronizer/src/sync/namespaceHandle.ts new file mode 100644 index 0000000..1efc3e8 --- /dev/null +++ b/packages/synchronizer/src/sync/namespaceHandle.ts @@ -0,0 +1,56 @@ +import type { CleanupRegistry } from '@map-colonies/cleanup-registry'; +import type { Logger } from '@map-colonies/js-logger'; +import type { DependencyContainer } from 'tsyringe/dist/typings/types'; +import type { DataSource, Repository } from 'typeorm'; +import { + createDataSource, + ENUMS_REPOSITORY_SYMBOL, + type EnumValue, + type Layer, + LAYER_REPOSITORY_SYMBOL, + type Property, + PROPERTY_REPOSITORY_SYMBOL, +} from '@db'; +import type { ConfigType } from '@common/config'; +import { SERVICES, SERVICE_NAME, SOURCE_DATA_SOURCE_PROVIDER } from '@common/constants'; +import { S3Repository } from '@common/s3/s3Repository'; +import { FsRepository } from '@common/fs/fsRepository'; +import { SyncModel } from './SyncModel'; +import { createLayerSourceStrategy } from './layerSource'; +import type { LayerSourceStrategy } from './layerSource/types'; + +export interface NamespaceHandle { + name: string; + sourceDataSource: DataSource; + layerSource: LayerSourceStrategy; + layersFile: string; + dal: SyncModel; +} + +export const createNamespaceHandles = (container: DependencyContainer, cleanupRegistry: CleanupRegistry): NamespaceHandle[] => { + const config = container.resolve(SERVICES.CONFIG); + const logger = container.resolve(SERVICES.LOGGER); + const s3Repository = container.resolve(S3Repository); + const fsRepository = container.resolve(FsRepository); + const layerRepository = container.resolve>(LAYER_REPOSITORY_SYMBOL); + const propertyRepository = container.resolve>(PROPERTY_REPOSITORY_SYMBOL); + const enumsRepository = container.resolve>(ENUMS_REPOSITORY_SYMBOL); + + return config.get('namespaces').map((namespaceConfig) => { + const sourceDataSource = createDataSource(namespaceConfig.db, `${SERVICE_NAME}-${namespaceConfig.name}`); + + cleanupRegistry.register({ + id: `${SOURCE_DATA_SOURCE_PROVIDER.toString()}:${namespaceConfig.name}`, + func: async () => { + if (sourceDataSource.isInitialized) { + await sourceDataSource.destroy(); + } + }, + }); + + const layerSource = createLayerSourceStrategy(namespaceConfig, s3Repository, fsRepository, logger); + const dal = new SyncModel(config, logger, sourceDataSource, layerRepository, propertyRepository, enumsRepository); + + return { name: namespaceConfig.name, sourceDataSource, layerSource, layersFile: namespaceConfig.layersFile, dal }; + }); +}; diff --git a/packages/synchronizer/tests/configurations/globalSetup.ts b/packages/synchronizer/tests/configurations/globalSetup.ts index dbd6b15..ac4afb3 100644 --- a/packages/synchronizer/tests/configurations/globalSetup.ts +++ b/packages/synchronizer/tests/configurations/globalSetup.ts @@ -4,11 +4,15 @@ import { createSslOptions } from '../../../db/dist/db/connection'; export default async function setup(): Promise { await initConfig(true); - const { source, destination } = getConfig().get('dbs'); + const config = getConfig(); + const destination = config.get('dbs.destination'); + const namespaces = config.get('namespaces'); const pgClient = new Client({ ...destination, user: destination.username, ssl: createSslOptions(destination.ssl) }); await pgClient.connect(); await pgClient.query(`DROP SCHEMA IF EXISTS "${destination.schema}" CASCADE`); - await pgClient.query(`DROP SCHEMA IF EXISTS "${source.schema}" CASCADE`); + for (const namespace of namespaces) { + await pgClient.query(`DROP SCHEMA IF EXISTS "${namespace.db.schema}" CASCADE`); + } await pgClient.end(); } diff --git a/packages/synchronizer/tests/integration/syncModel/syncModel.spec.ts b/packages/synchronizer/tests/integration/syncModel/syncModel.spec.ts index d93b778..1d49775 100644 --- a/packages/synchronizer/tests/integration/syncModel/syncModel.spec.ts +++ b/packages/synchronizer/tests/integration/syncModel/syncModel.spec.ts @@ -1,11 +1,10 @@ -/* eslint-disable @typescript-eslint/naming-convention */ import { writeFileSync, mkdtempSync, rmSync } from 'node:fs'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { jsLogger } from '@map-colonies/js-logger'; -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest'; import { trace } from '@opentelemetry/api'; -import nock, { disableNetConnect, enableNetConnect, cleanAll } from 'nock'; +import { disableNetConnect, enableNetConnect, cleanAll } from 'nock'; import type { DataSource, Repository } from 'typeorm'; import type { EnumValue, Layer, Property } from '@map-colonies/vector-standard-db'; import { @@ -14,54 +13,65 @@ import { LAYER_REPOSITORY_SYMBOL, PROPERTY_REPOSITORY_SYMBOL, columnType, + layerSource, } from '@map-colonies/vector-standard-db'; -import { initConfig, getConfig } from '@src/common/config'; +import { initConfig } from '@src/common/config'; import { registerExternalValues } from '@src/containerConfig'; -import { SERVICES, SOURCE_DATA_SOURCE_PROVIDER } from '@src/common/constants'; +import { SERVICES, NAMESPACE_HANDLES } from '@src/common/constants'; import { S3Repository } from '@src/common/s3/s3Repository'; -import { SyncModel } from '@src/sync/SyncModel'; +import type { SyncModel } from '@src/sync/SyncModel'; import { schemaOf } from '@src/sync/helpers'; import { FileReader, type FileAliases } from '@src/sync/fileReader'; import type { TypeMap } from '@src/sync/typeMap'; +import type { NamespaceHandle } from '@src/sync/namespaceHandle'; const TEST_LAYER = 'test_layer'; -const ENRICHMENT_ORIGIN = 'http://mock-api'; +const NAMESPACE = 'test'; const TYPE_MAP_FILE = './config/typeMap.json'; const noAliases: FileAliases = new Map(); - -const mockEnrichmentApi = (responseBody: object, statusCode = 200): nock.Scope => - nock(ENRICHMENT_ORIGIN).get(`/${TEST_LAYER}`).reply(statusCode, responseBody); +const noSourceAliases = new Map(); const createDestinationSchema = async (dataSource: DataSource, schema: string): Promise => { await dataSource.query(`CREATE SCHEMA IF NOT EXISTS "${schema}"`); await dataSource.query(` CREATE TYPE "${schema}"."column_type" AS ENUM('xsd:long', 'xsd:double', 'xsd:boolean', 'xsd:string', 'xsd:dateTime', 'gml:GeometryPropertyType', 'gml:PointPropertyType', 'gml:LineStringPropertyType', 'gml:PolygonPropertyType', 'gml:MultiPointPropertyType', 'gml:MultiLineStringPropertyType', 'gml:MultiPolygonPropertyType') `); + await dataSource.query(` + CREATE TYPE "${schema}"."layer_source" AS ENUM('sharedLua', 'perLayerJson') + `); await dataSource.query(` CREATE TABLE "${schema}"."layer" ( + "namespace" character varying NOT NULL, "layer_name" character varying NOT NULL, - "layer_id" integer NOT NULL, + "layer_id" integer, + "source" "${schema}"."layer_source" NOT NULL, "alias" character varying NOT NULL, - PRIMARY KEY ("layer_name") + PRIMARY KEY ("namespace", "layer_name"), + CONSTRAINT "CHK_layer_source_layer_id" CHECK (("source" = 'sharedLua' AND "layer_id" IS NOT NULL) OR ("source" = 'perLayerJson' AND "layer_id" IS NULL)) ) `); await dataSource.query(` CREATE TABLE "${schema}"."property" ( + "namespace" character varying NOT NULL, "layer_name" character varying NOT NULL, "property" character varying NOT NULL, "type" "${schema}"."column_type" NOT NULL, "alias" character varying, - PRIMARY KEY ("layer_name", "property") + PRIMARY KEY ("namespace", "layer_name", "property"), + FOREIGN KEY ("namespace", "layer_name") + REFERENCES "${schema}"."layer"("namespace", "layer_name") + ON DELETE NO ACTION ON UPDATE NO ACTION ) `); await dataSource.query(` CREATE TABLE "${schema}"."enum_value" ( "value" character varying NOT NULL, + "namespace" character varying NOT NULL, "layer_name" character varying NOT NULL, "property" character varying NOT NULL, - PRIMARY KEY ("value", "layer_name", "property"), - FOREIGN KEY ("layer_name", "property") - REFERENCES "${schema}"."property"("layer_name","property") + PRIMARY KEY ("value", "namespace", "layer_name", "property"), + FOREIGN KEY ("namespace", "layer_name", "property") + REFERENCES "${schema}"."property"("namespace", "layer_name", "property") ON DELETE CASCADE ON UPDATE NO ACTION ) `); @@ -69,7 +79,6 @@ const createDestinationSchema = async (dataSource: DataSource, schema: string): describe('DAL', function () { let dal: SyncModel; - let enrichedDal: SyncModel; let fileReader: FileReader; let typeMap: TypeMap; let sourceDataSource: DataSource; @@ -88,15 +97,23 @@ describe('DAL', function () { override: [ { token: SERVICES.LOGGER, provider: { useValue: await jsLogger({ enabled: false }) } }, { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, - { token: S3Repository, provider: { useValue: { downloadFile: vi.fn() } } }, + { token: S3Repository, provider: { useValue: { downloadFile: async () => Promise.reject(new Error('not used in this suite')) } } }, ], useChild: true, }); - dal = container.resolve(SyncModel); + const namespaceHandles = container.resolve(NAMESPACE_HANDLES); + const namespace = namespaceHandles.find((n) => n.name === NAMESPACE); + if (namespace === undefined) { + throw new Error(`Namespace ${NAMESPACE} was not configured for this test run`); + } + + dal = namespace.dal; + sourceDataSource = namespace.sourceDataSource; + await sourceDataSource.initialize(); + fileReader = container.resolve(FileReader); typeMap = await fileReader.readTypeMap(TYPE_MAP_FILE); - sourceDataSource = container.resolve(SOURCE_DATA_SOURCE_PROVIDER); destinationDataSource = container.resolve(DESTINATION_DATA_SOURCE_PROVIDER); layerRepository = container.resolve>(LAYER_REPOSITORY_SYMBOL); propertyRepository = container.resolve>(PROPERTY_REPOSITORY_SYMBOL); @@ -107,38 +124,6 @@ describe('DAL', function () { await sourceDataSource.query(`CREATE SCHEMA IF NOT EXISTS "${sourceSchema}"`); await createDestinationSchema(destinationDataSource, destinationSchema); - - const configWithEnrichment = new Proxy(getConfig(), { - get(target, prop) { - if (prop === 'get') { - return (key: string) => { - if (key === 'enrichment') { - return { - enabled: true as const, - api: 'http://mock-api/{layerName}', - propertiesPath: 'fields_list', - aliasField: 'display_name', - requestTimeoutMilliseconds: 10000, - }; - } - return target.get(key); - }; - } - return target[prop as keyof typeof target]; - }, - }); - - const enrichedContainer = await registerExternalValues({ - override: [ - { token: SERVICES.LOGGER, provider: { useValue: await jsLogger({ enabled: false }) } }, - { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, - { token: SERVICES.CONFIG, provider: { useValue: configWithEnrichment } }, - { token: S3Repository, provider: { useValue: { downloadFile: vi.fn() } } }, - ], - useChild: true, - }); - - enrichedDal = enrichedContainer.resolve(SyncModel); }); beforeEach(async function () { @@ -153,6 +138,7 @@ describe('DAL', function () { shape geometry ) `); + await dal.syncLayer(NAMESPACE, TEST_LAYER, 1, layerSource.sharedLua); }); afterEach(async function () { @@ -174,9 +160,9 @@ describe('DAL', function () { describe('Happy Path', function () { describe('syncLayer', function () { it('should insert a layer row with the provided alias', async function () { - await dal.syncLayer('buildings', 1, 'Buildings Layer'); + await dal.syncLayer(NAMESPACE, 'buildings', 1, layerSource.sharedLua, 'Buildings Layer'); - const layer = await layerRepository.findOne({ where: { layerName: 'buildings' } }); + const layer = await layerRepository.findOne({ where: { namespace: NAMESPACE, layerName: 'buildings' } }); expect(layer?.layerName).toBe('buildings'); expect(layer?.layerId).toBe(1); @@ -184,12 +170,21 @@ describe('DAL', function () { }); it('should use layerName as alias when none is provided', async function () { - await dal.syncLayer('buildings', 1); + await dal.syncLayer(NAMESPACE, 'buildings', 1, layerSource.sharedLua); - const layer = await layerRepository.findOne({ where: { layerName: 'buildings' } }); + const layer = await layerRepository.findOne({ where: { namespace: NAMESPACE, layerName: 'buildings' } }); expect(layer?.alias).toBe('buildings'); }); + + it('should accept a null layerId for a perLayerJson layer', async function () { + await dal.syncLayer(NAMESPACE, 'parks', null, layerSource.perLayerJson, 'Parks'); + + const layer = await layerRepository.findOne({ where: { namespace: NAMESPACE, layerName: 'parks' } }); + + expect(layer?.layerId).toBeNull(); + expect(layer?.source).toBe(layerSource.perLayerJson); + }); }); describe('getTableColumns', function () { @@ -245,9 +240,9 @@ describe('DAL', function () { describe('syncProperties', function () { it('should upsert all columns from the source table into the property repository', async function () { - const affected = await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); + const affected = await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: [] }, typeMap, noAliases, noSourceAliases); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); expect(affected).toBeGreaterThan(0); expect(properties).toEqual( @@ -263,9 +258,9 @@ describe('DAL', function () { }); it('should mark enum columns with columnType.enum', async function () { - await dal.syncProperties({ layerName: TEST_LAYER, enums: ['category', 'name'] }, 1, typeMap, noAliases); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: ['category', 'name'] }, typeMap, noAliases, noSourceAliases); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); const categoryProp = properties.find((p) => p.property === 'category'); const nameProp = properties.find((p) => p.property === 'name'); const heightProp = properties.find((p) => p.property === 'height'); @@ -276,19 +271,25 @@ describe('DAL', function () { }); it('should upsert on subsequent calls without duplicating rows', async function () { - await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); - await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: [] }, typeMap, noAliases, noSourceAliases); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: [] }, typeMap, noAliases, noSourceAliases); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); const uniqueProps = new Set(properties.map((p) => p.property)); expect(properties).toHaveLength(uniqueProps.size); }); it('should not sync properties listed in excludeProperties', async function () { - await dal.syncProperties({ layerName: TEST_LAYER, enums: [], excludeProperties: ['height', 'CREATED_AT'] }, 1, typeMap, noAliases); + await dal.syncProperties( + NAMESPACE, + { layerName: TEST_LAYER, enums: [], excludeProperties: ['height', 'CREATED_AT'] }, + typeMap, + noAliases, + noSourceAliases + ); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); const names = properties.map((p) => p.property); expect(names).not.toContain('height'); @@ -297,11 +298,11 @@ describe('DAL', function () { }); it('should delete properties that were synced before being excluded', async function () { - await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: [] }, typeMap, noAliases, noSourceAliases); - await dal.syncProperties({ layerName: TEST_LAYER, enums: [], excludeProperties: ['height'] }, 1, typeMap, noAliases); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: [], excludeProperties: ['height'] }, typeMap, noAliases, noSourceAliases); - const heightProp = await propertyRepository.findOne({ where: { layerName: TEST_LAYER, property: 'height' } }); + const heightProp = await propertyRepository.findOne({ where: { namespace: NAMESPACE, layerName: TEST_LAYER, property: 'height' } }); expect(heightProp).toBeNull(); }); @@ -311,9 +312,9 @@ describe('DAL', function () { ALTER TABLE "${sourceSchema}"."${TEST_LAYER}" ADD COLUMN geom_param geometry(Point,4326) `); - await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: [] }, typeMap, noAliases, noSourceAliases); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); const geomProp = properties.find((p) => p.property === 'geom_param'); expect(geomProp?.type).toBe(columnType.point); @@ -325,11 +326,11 @@ describe('DAL', function () { await sourceDataSource.query(` INSERT INTO "${sourceSchema}"."${TEST_LAYER}" (category) VALUES ('A'), ('B'), ('A'), ('C') `); - await dal.syncProperties({ layerName: TEST_LAYER, enums: ['category'] }, 1, typeMap, noAliases); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: ['category'] }, typeMap, noAliases, noSourceAliases); - const affected = await dal.syncEnum({ layerName: TEST_LAYER, enums: ['category'] }); + const affected = await dal.syncEnum(NAMESPACE, { layerName: TEST_LAYER, enums: ['category'] }); - const enums = await enumsRepository.find({ where: { layerName: TEST_LAYER, property: 'category' } }); + const enums = await enumsRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER, property: 'category' } }); const values = enums.map((e) => e.value); expect(affected).toBe(3); @@ -337,8 +338,8 @@ describe('DAL', function () { }); it('should create an index on the enum column in the source table', async function () { - await dal.syncProperties({ layerName: TEST_LAYER, enums: ['category'] }, 1, typeMap, noAliases); - await dal.syncEnum({ layerName: TEST_LAYER, enums: ['category'] }); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: ['category'] }, typeMap, noAliases, noSourceAliases); + await dal.syncEnum(NAMESPACE, { layerName: TEST_LAYER, enums: ['category'] }); const indexName = `${TEST_LAYER}_category_idx`; const rows = await sourceDataSource.query<{ indexname: string }[]>( @@ -350,18 +351,28 @@ describe('DAL', function () { }); }); - describe('syncProperties with enrichment', function () { - it('should write aliases from the enrichment API into the property table', async function () { - mockEnrichmentApi({ - fields_list: { - name: { display_name: 'Layer Name', type: 'TEXT' }, - height: { display_name: 'Building Height', type: 'REAL' }, - }, - }); + describe('syncProperties with source and file aliases', function () { + let tmpDir: string; + let aliasesFilePath: string; + + beforeAll(function () { + tmpDir = mkdtempSync(join(tmpdir(), 'dal-aliases-test-')); + aliasesFilePath = join(tmpDir, 'aliases.json'); + }); + + afterAll(function () { + rmSync(tmpDir, { recursive: true }); + }); + + it('should write aliases from the source aliases map into the property table', async function () { + const sourceAliases = new Map([ + ['name', 'Layer Name'], + ['height', 'Building Height'], + ]); - await enrichedDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); + await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: [] }, typeMap, noAliases, sourceAliases); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); const nameProp = properties.find((p) => p.property === 'name'); const heightProp = properties.find((p) => p.property === 'height'); const idProp = properties.find((p) => p.property === 'id'); @@ -372,127 +383,100 @@ describe('DAL', function () { expect(idProp?.alias).toBeUndefined(); }); - it('should call the enrichment API once with the resolved layer URL', async function () { - const scope = mockEnrichmentApi({ fields_list: {} }); - - await enrichedDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); - - expect(scope.isDone()).toBe(true); - }); - }); + it('should apply aliases from the file when there are no source aliases', async function () { + writeFileSync(aliasesFilePath, JSON.stringify({ [NAMESPACE]: { [TEST_LAYER]: { name: 'File Name', height: 'File Height' } } })); - describe('syncProperties with file aliases', function () { - let tmpDir: string; - let aliasesFilePath: string; - let aliasesFileDal: SyncModel; - let bothDal: SyncModel; + await dal.syncProperties( + NAMESPACE, + { layerName: TEST_LAYER, enums: [] }, + typeMap, + await fileReader.readAliases(aliasesFilePath), + noSourceAliases + ); - beforeAll(async function () { - tmpDir = mkdtempSync(join(tmpdir(), 'dal-aliases-test-')); - aliasesFilePath = join(tmpDir, 'aliases.json'); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); - const makeConfig = (enrichmentEnabled: boolean) => - new Proxy(getConfig(), { - get(target, prop) { - if (prop === 'get') { - return (key: string) => { - if (key === 'enrichment') { - return enrichmentEnabled - ? { - enabled: true as const, - api: 'http://mock-api/{layerName}', - propertiesPath: 'fields_list', - aliasField: 'display_name', - requestTimeoutMilliseconds: 10000, - } - : { enabled: false }; - } - return target.get(key); - }; - } - return target[prop as keyof typeof target]; - }, - }); - - const logger = await jsLogger({ enabled: false }); - const s3Mock = { token: S3Repository, provider: { useValue: { downloadFile: vi.fn() } } }; - const sharedOverrides = [ - { token: SERVICES.LOGGER, provider: { useValue: logger } }, - { token: SERVICES.TRACER, provider: { useValue: trace.getTracer('testTracer') } }, - s3Mock, - ]; - - aliasesFileDal = ( - await registerExternalValues({ - override: [...sharedOverrides, { token: SERVICES.CONFIG, provider: { useValue: makeConfig(false) } }], - useChild: true, - }) - ).resolve(SyncModel); - - bothDal = ( - await registerExternalValues({ - override: [...sharedOverrides, { token: SERVICES.CONFIG, provider: { useValue: makeConfig(true) } }], - useChild: true, - }) - ).resolve(SyncModel); + expect(properties.find((p) => p.property === 'name')?.alias).toBe('File Name'); + expect(properties.find((p) => p.property === 'height')?.alias).toBe('File Height'); + expect(properties.find((p) => p.property === 'id')?.alias).toBeUndefined(); }); - afterAll(function () { - rmSync(tmpDir, { recursive: true }); - }); + it('should override source aliases with file aliases for the same property', async function () { + writeFileSync(aliasesFilePath, JSON.stringify({ [NAMESPACE]: { [TEST_LAYER]: { name: 'File Name' } } })); - it('should apply aliases from the file when enrichment is disabled', async function () { - writeFileSync(aliasesFilePath, JSON.stringify({ [TEST_LAYER]: { name: 'File Name', height: 'File Height' } })); + const sourceAliases = new Map([ + ['name', 'Source Name'], + ['height', 'Source Height'], + ]); - await aliasesFileDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, await fileReader.readAliases(aliasesFilePath)); + await dal.syncProperties( + NAMESPACE, + { layerName: TEST_LAYER, enums: [] }, + typeMap, + await fileReader.readAliases(aliasesFilePath), + sourceAliases + ); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); expect(properties.find((p) => p.property === 'name')?.alias).toBe('File Name'); - expect(properties.find((p) => p.property === 'height')?.alias).toBe('File Height'); - expect(properties.find((p) => p.property === 'id')?.alias).toBeUndefined(); + expect(properties.find((p) => p.property === 'height')?.alias).toBe('Source Height'); }); - it('should override API aliases with file aliases for the same property', async function () { - writeFileSync(aliasesFilePath, JSON.stringify({ [TEST_LAYER]: { name: 'File Name' } })); + it('should fill in aliases from the file for properties absent from the source aliases', async function () { + writeFileSync(aliasesFilePath, JSON.stringify({ [NAMESPACE]: { [TEST_LAYER]: { height: 'File Height' } } })); - mockEnrichmentApi({ - fields_list: { name: { display_name: 'API Name', type: 'TEXT' }, height: { display_name: 'API Height', type: 'REAL' } }, - }); + const sourceAliases = new Map([['name', 'Source Name']]); - await bothDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, await fileReader.readAliases(aliasesFilePath)); + await dal.syncProperties( + NAMESPACE, + { layerName: TEST_LAYER, enums: [] }, + typeMap, + await fileReader.readAliases(aliasesFilePath), + sourceAliases + ); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); - expect(properties.find((p) => p.property === 'name')?.alias).toBe('File Name'); - expect(properties.find((p) => p.property === 'height')?.alias).toBe('API Height'); + expect(properties.find((p) => p.property === 'name')?.alias).toBe('Source Name'); + expect(properties.find((p) => p.property === 'height')?.alias).toBe('File Height'); }); - it('should fill in aliases from the file for properties absent from the API response', async function () { - writeFileSync(aliasesFilePath, JSON.stringify({ [TEST_LAYER]: { height: 'File Height' } })); - - mockEnrichmentApi({ fields_list: { name: { display_name: 'API Name', type: 'TEXT' } } }); + it('should prefer a namespace-specific file alias over a wildcard one', async function () { + writeFileSync( + aliasesFilePath, + JSON.stringify({ + '*': { '*': { name: 'Global Name' } }, + [NAMESPACE]: { [TEST_LAYER]: { name: 'Specific Name' } }, + }) + ); - await bothDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, await fileReader.readAliases(aliasesFilePath)); + await dal.syncProperties( + NAMESPACE, + { layerName: TEST_LAYER, enums: [] }, + typeMap, + await fileReader.readAliases(aliasesFilePath), + noSourceAliases + ); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); - expect(properties.find((p) => p.property === 'name')?.alias).toBe('API Name'); - expect(properties.find((p) => p.property === 'height')?.alias).toBe('File Height'); + expect(properties.find((p) => p.property === 'name')?.alias).toBe('Specific Name'); }); }); }); describe('Sad Path', function () { describe('syncLayer', function () { - it('should not throw and not update on duplicate layerName', async function () { - await dal.syncLayer('buildings', 1, 'First'); - await dal.syncLayer('buildings', 2, 'Second'); + it('should upsert layerId, source and alias on a duplicate namespace/layerName', async function () { + await dal.syncLayer(NAMESPACE, 'buildings', 1, layerSource.sharedLua, 'First'); + await dal.syncLayer(NAMESPACE, 'buildings', 2, layerSource.sharedLua, 'Second'); - const layers = await layerRepository.find({ where: { layerName: 'buildings' } }); + const layers = await layerRepository.find({ where: { namespace: NAMESPACE, layerName: 'buildings' } }); expect(layers).toHaveLength(1); - expect(layers[0]?.alias).toBe('First'); + expect(layers[0]?.layerId).toBe(2); + expect(layers[0]?.alias).toBe('Second'); }); }); @@ -518,44 +502,15 @@ describe('DAL', function () { describe('syncEnum', function () { it('should return 0 and skip when no enum columns are configured', async function () { - const affected = await dal.syncEnum({ layerName: TEST_LAYER, enums: [] }); + const affected = await dal.syncEnum(NAMESPACE, { layerName: TEST_LAYER, enums: [] }); expect(affected).toBe(0); - const enums = await enumsRepository.find({ where: { layerName: TEST_LAYER } }); + const enums = await enumsRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); expect(enums).toHaveLength(0); }); }); - - describe('syncProperties with enrichment', function () { - it('should leave alias unset for properties absent from the enrichment response', async function () { - mockEnrichmentApi({ fields_list: { name: { display_name: 'Layer Name', type: 'TEXT' } } }); - - await enrichedDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); - - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); - const idProp = properties.find((p) => p.property === 'id'); - - expect(idProp).toBeDefined(); - expect(idProp?.alias).toBeUndefined(); - }); - - it('should sync properties without aliases when the enrichment API returns an error response', async function () { - mockEnrichmentApi({}, 500); - - const affected = await enrichedDal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); - - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); - const idProp = properties.find((p) => p.property === 'id'); - const nameProp = properties.find((p) => p.property === 'name'); - - expect(affected).toBeGreaterThan(0); - expect(idProp).toBeDefined(); - expect(idProp?.alias).toBeUndefined(); - expect(nameProp?.alias).toBeUndefined(); - }); - }); }); describe('Bad Path', function () { @@ -565,8 +520,8 @@ describe('DAL', function () { ALTER TABLE "${sourceSchema}"."${TEST_LAYER}" ADD COLUMN lsn pg_lsn `); - const affected = await dal.syncProperties({ layerName: TEST_LAYER, enums: [] }, 1, typeMap, noAliases); - const properties = await propertyRepository.find({ where: { layerName: TEST_LAYER } }); + const affected = await dal.syncProperties(NAMESPACE, { layerName: TEST_LAYER, enums: [] }, typeMap, noAliases, noSourceAliases); + const properties = await propertyRepository.find({ where: { namespace: NAMESPACE, layerName: TEST_LAYER } }); expect(affected).toBeGreaterThan(0); expect(properties.find((p) => p.property === 'lsn')).toBeUndefined(); @@ -574,4 +529,35 @@ describe('DAL', function () { }); }); }); + + describe('Multi-namespace isolation', function () { + it('should not delete layers belonging to another namespace', async function () { + await dal.syncLayer(NAMESPACE, 'buildings', 1, layerSource.sharedLua); + await layerRepository.insert({ namespace: 'other', layerName: 'buildings', layerId: 1, source: layerSource.sharedLua, alias: 'buildings' }); + + await dal.deleteLayersNotIn(NAMESPACE, ['roads']); + + const otherLayer = await layerRepository.findOne({ where: { namespace: 'other', layerName: 'buildings' } }); + const ownLayer = await layerRepository.findOne({ where: { namespace: NAMESPACE, layerName: 'buildings' } }); + + expect(otherLayer).not.toBeNull(); + expect(ownLayer).toBeNull(); + + await layerRepository.delete({ namespace: 'other', layerName: 'buildings' }); + }); + + it('should not delete stale properties belonging to another namespace', async function () { + await layerRepository.insert({ namespace: 'other', layerName: TEST_LAYER, layerId: 1, source: layerSource.sharedLua, alias: TEST_LAYER }); + await propertyRepository.insert({ namespace: 'other', layerName: TEST_LAYER, property: 'legacy', type: columnType.text }); + + await dal.deleteStaleProperties(NAMESPACE, TEST_LAYER, []); + + const otherProperty = await propertyRepository.findOne({ where: { namespace: 'other', layerName: TEST_LAYER, property: 'legacy' } }); + + expect(otherProperty).not.toBeNull(); + + await propertyRepository.delete({ namespace: 'other', layerName: TEST_LAYER }); + await layerRepository.delete({ namespace: 'other', layerName: TEST_LAYER }); + }); + }); }); diff --git a/packages/synchronizer/tests/unit/synchronizer/aliasEnricher.spec.ts b/packages/synchronizer/tests/unit/synchronizer/aliasEnricher.spec.ts index 0d5ae3f..38ba574 100644 --- a/packages/synchronizer/tests/unit/synchronizer/aliasEnricher.spec.ts +++ b/packages/synchronizer/tests/unit/synchronizer/aliasEnricher.spec.ts @@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach, beforeAll, afterEach, afterAll } from 'vitest'; import nock, { disableNetConnect, enableNetConnect, cleanAll } from 'nock'; import { fetchPropertyAliases } from '@src/sync/aliasEnricher'; -import { type ConfigType, getConfig, initConfig } from '@src/common/config'; +import type { EnrichmentConfig } from '@src/common/interfaces'; const ENRICHMENT_ORIGIN = 'https://example.com'; @@ -13,11 +13,15 @@ const makeBody = (fields: Record }); describe('fetchPropertyAliases', function () { - let enrichmentConfig: ReturnType['enrichment'] & { enabled: true }; - - beforeAll(async function () { - await initConfig(true); - enrichmentConfig = getConfig().get('enrichmentApi'); + const enrichmentConfig: Extract = { + enabled: true, + api: `${ENRICHMENT_ORIGIN}/{layerName}`, + propertiesPath: 'fields_list', + aliasField: 'display_name', + requestTimeoutMilliseconds: 10000, + }; + + beforeAll(function () { disableNetConnect(); }); @@ -37,7 +41,7 @@ describe('fetchPropertyAliases', function () { it('should call the API once with the layer name substituted into the URL', async function () { const scope = nock(ENRICHMENT_ORIGIN).get('/buildings').reply(200, makeBody({})); - await fetchPropertyAliases('buildings', 1, getConfig().get('enrichmentApi')); + await fetchPropertyAliases('buildings', 1, enrichmentConfig); expect(scope.isDone()).toBe(true); }); diff --git a/packages/synchronizer/tests/unit/synchronizer/helpers.spec.ts b/packages/synchronizer/tests/unit/synchronizer/helpers.spec.ts index ce520ba..d5d9ec3 100644 --- a/packages/synchronizer/tests/unit/synchronizer/helpers.spec.ts +++ b/packages/synchronizer/tests/unit/synchronizer/helpers.spec.ts @@ -10,6 +10,7 @@ const typeMapFile = './config/typeMap.json'; const typeMap = parseTypeMap(JSON.parse(readFileSync(typeMapFile, 'utf-8')) as JsonValue, typeMapFile); describe('columnInfosToProperties', function () { + const namespace = 'test_namespace'; const layer = 'test_layer'; describe('Happy Path', function () { @@ -53,7 +54,7 @@ describe('columnInfosToProperties', function () { ])('should map udt "%s" to %s', function (udtName, expected) { const columnInfos: ColumnInfo[] = [{ columnName: 'col', udtName }]; - const [result] = columnInfosToProperties(columnInfos, layer, typeMap); + const [result] = columnInfosToProperties(columnInfos, namespace, layer, typeMap); expect(result.type).toBe(expected); }); @@ -72,19 +73,19 @@ describe('columnInfosToProperties', function () { ['geography(MultiPolygon,4326)', columnType.multiPolygon], ['geometry(GeometryCollection,4326)', columnType.geom], ])('should map "%s" to its specific GML type', function (udtName, expected) { - const [result] = columnInfosToProperties([{ columnName: 'shape', udtName }], layer, typeMap); + const [result] = columnInfosToProperties([{ columnName: 'shape', udtName }], namespace, layer, typeMap); expect(result.type).toBe(expected); }); it('should map unqualified geometry to gml:GeometryPropertyType', function () { - const [result] = columnInfosToProperties([{ columnName: 'shape', udtName: 'geometry' }], layer, typeMap); + const [result] = columnInfosToProperties([{ columnName: 'shape', udtName: 'geometry' }], namespace, layer, typeMap); expect(result.type).toBe(columnType.geom); }); it('should match udt names case insensitively', function () { - const [result] = columnInfosToProperties([{ columnName: 'name', udtName: 'CHARACTER VARYING(255)' }], layer, typeMap); + const [result] = columnInfosToProperties([{ columnName: 'name', udtName: 'CHARACTER VARYING(255)' }], namespace, layer, typeMap); expect(result.type).toBe(columnType.text); }); @@ -95,30 +96,30 @@ describe('columnInfosToProperties', function () { { columnName: 'height', udtName: 'real' }, ]; - const result = columnInfosToProperties(columnInfos, layer, typeMap); + const result = columnInfosToProperties(columnInfos, namespace, layer, typeMap); expect(result).toEqual([ - { layerName: layer, property: 'name', type: columnType.text }, - { layerName: layer, property: 'height', type: columnType.real }, + { namespace, layerName: layer, property: 'name', type: columnType.text }, + { namespace, layerName: layer, property: 'height', type: columnType.real }, ]); }); }); describe('Sad Path', function () { it('should return an empty array for no column infos', function () { - expect(columnInfosToProperties([], layer, typeMap)).toEqual([]); + expect(columnInfosToProperties([], namespace, layer, typeMap)).toEqual([]); }); }); describe('Bad Path', function () { it('should map unknown geometry sub-type to gml:GeometryPropertyType', function () { - const [result] = columnInfosToProperties([{ columnName: 'shape', udtName: 'geometry(Curve,4326)' }], layer, typeMap); + const [result] = columnInfosToProperties([{ columnName: 'shape', udtName: 'geometry(Curve,4326)' }], namespace, layer, typeMap); expect(result.type).toBe(columnType.geom); }); it('should return no properties for a udt name not in the type map', function () { - const result = columnInfosToProperties([{ columnName: 'data', udtName: 'pg_lsn' }], layer, typeMap); + const result = columnInfosToProperties([{ columnName: 'data', udtName: 'pg_lsn' }], namespace, layer, typeMap); expect(result).toHaveLength(0); }); @@ -131,7 +132,7 @@ describe('columnInfosToProperties', function () { { columnName: 'height', udtName: 'real' }, ]; - const result = columnInfosToProperties(columnInfos, layer, typeMap, onUnknown); + const result = columnInfosToProperties(columnInfos, namespace, layer, typeMap, onUnknown); expect(result).toHaveLength(2); expect(result.map((p) => p.property)).toEqual(['name', 'height']); diff --git a/packages/synchronizer/tests/unit/synchronizer/perLayerJsonSource.spec.ts b/packages/synchronizer/tests/unit/synchronizer/perLayerJsonSource.spec.ts new file mode 100644 index 0000000..293505d --- /dev/null +++ b/packages/synchronizer/tests/unit/synchronizer/perLayerJsonSource.spec.ts @@ -0,0 +1,141 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { writeFileSync, mkdtempSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { describe, it, expect, vi, beforeEach, afterEach, beforeAll } from 'vitest'; +import { jsLogger } from '@map-colonies/js-logger'; +import type { Logger } from '@map-colonies/js-logger'; +import { PerLayerJsonSource } from '@src/sync/layerSource/perLayerJsonSource'; +import type { FileDownloader } from '@src/sync/layerSource/types'; +import { FsRepository } from '@src/common/fs/fsRepository'; +import type { PerLayerJsonSourceConfig } from '@src/common/interfaces'; + +const BUCKET = 'test-bucket'; + +const config: PerLayerJsonSourceConfig = { + type: 'perLayerJson', + prefix: '', + aliasLayerNameField: 'alias_layer_name', + fieldsField: 'fields', + fieldNameField: 'field_name', + aliasFieldNameField: 'alias_field_name', +}; + +describe('PerLayerJsonSource', function () { + let tmpDir: string; + let logger: Logger; + let fsRepository: FsRepository; + + const makeS3Repository = (key: string, content: string): { s3Repository: FileDownloader; downloadFile: ReturnType } => { + const filePath = join(tmpDir, 'downloaded.json'); + writeFileSync(filePath, content); + const downloadFile = vi.fn().mockImplementation((_bucket: string, requestedKey: string) => { + if (requestedKey !== key) { + throw new Error('unexpected key'); + } + return filePath; + }); + return { s3Repository: { downloadFile }, downloadFile }; + }; + + beforeAll(async function () { + logger = await jsLogger({ enabled: false }); + fsRepository = new FsRepository(logger); + }); + + beforeEach(function () { + tmpDir = mkdtempSync(join(tmpdir(), 'per-layer-json-source-test-')); + }); + + afterEach(function () { + rmSync(tmpDir, { recursive: true }); + }); + + describe('Happy Path', function () { + it('should resolve a layer alias and case-insensitive property aliases', async function () { + const { s3Repository } = makeS3Repository( + 'BUILDINGS.json', + JSON.stringify({ + alias_layer_name: 'Buildings', + fields: [{ field_name: 'PROPERTY', alias_field_name: 'Property Alias' }], + }) + ); + const source = new PerLayerJsonSource(s3Repository, fsRepository, logger, BUCKET, config); + + const records = await source.tick([{ layerName: 'buildings', enums: [], sourceKey: 'BUILDINGS.json' }]); + const record = records.get('buildings'); + + expect(record?.alias).toBe('Buildings'); + expect(record?.layerId).toBeUndefined(); + expect(record?.source).toBe('perLayerJson'); + expect(record?.propertyAliases.get('property')).toBe('Property Alias'); + }); + + it('should prepend the configured prefix to the sourceKey', async function () { + const prefixedConfig: PerLayerJsonSourceConfig = { ...config, prefix: 'layers/' }; + const { s3Repository } = makeS3Repository('layers/BUILDINGS.json', JSON.stringify({ alias_layer_name: 'Buildings', fields: [] })); + const source = new PerLayerJsonSource(s3Repository, fsRepository, logger, BUCKET, prefixedConfig); + + const records = await source.tick([{ layerName: 'buildings', enums: [], sourceKey: 'BUILDINGS.json' }]); + + expect(records.get('buildings')?.alias).toBe('Buildings'); + }); + + it('should ignore fields with no matching field_name or alias_field_name', async function () { + const { s3Repository } = makeS3Repository( + 'BUILDINGS.json', + JSON.stringify({ + alias_layer_name: 'Buildings', + fields: [{ field_name: 'PROPERTY' }, { alias_field_name: 'Orphan Alias' }, 'not-an-object'], + }) + ); + const source = new PerLayerJsonSource(s3Repository, fsRepository, logger, BUCKET, config); + + const records = await source.tick([{ layerName: 'buildings', enums: [], sourceKey: 'BUILDINGS.json' }]); + + expect(records.get('buildings')?.propertyAliases.size).toBe(0); + }); + }); + + describe('Sad Path', function () { + it('should skip and warn when the layer has no sourceKey', async function () { + const { s3Repository, downloadFile } = makeS3Repository('BUILDINGS.json', '{}'); + const source = new PerLayerJsonSource(s3Repository, fsRepository, logger, BUCKET, config); + + const records = await source.tick([{ layerName: 'buildings', enums: [] }]); + + expect(records.has('buildings')).toBe(false); + expect(downloadFile).not.toHaveBeenCalled(); + }); + + it('should skip a layer whose object is missing from S3', async function () { + const downloadFile = vi.fn().mockRejectedValue(new Error('NoSuchKey')); + const s3Repository: FileDownloader = { downloadFile }; + const source = new PerLayerJsonSource(s3Repository, fsRepository, logger, BUCKET, config); + + const records = await source.tick([{ layerName: 'buildings', enums: [], sourceKey: 'MISSING.json' }]); + + expect(records.has('buildings')).toBe(false); + }); + }); + + describe('Bad Path', function () { + it('should skip a layer with unparseable JSON', async function () { + const { s3Repository } = makeS3Repository('BUILDINGS.json', 'not json'); + const source = new PerLayerJsonSource(s3Repository, fsRepository, logger, BUCKET, config); + + const records = await source.tick([{ layerName: 'buildings', enums: [], sourceKey: 'BUILDINGS.json' }]); + + expect(records.has('buildings')).toBe(false); + }); + + it('should skip a layer missing required fields', async function () { + const { s3Repository } = makeS3Repository('BUILDINGS.json', JSON.stringify({ fields: [] })); + const source = new PerLayerJsonSource(s3Repository, fsRepository, logger, BUCKET, config); + + const records = await source.tick([{ layerName: 'buildings', enums: [], sourceKey: 'BUILDINGS.json' }]); + + expect(records.has('buildings')).toBe(false); + }); + }); +}); diff --git a/packages/synchronizer/tests/unit/synchronizer/sharedLuaSource.spec.ts b/packages/synchronizer/tests/unit/synchronizer/sharedLuaSource.spec.ts new file mode 100644 index 0000000..c0004b2 --- /dev/null +++ b/packages/synchronizer/tests/unit/synchronizer/sharedLuaSource.spec.ts @@ -0,0 +1,141 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { writeFileSync, mkdtempSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { describe, it, expect, vi, beforeEach, afterEach, beforeAll, afterAll } from 'vitest'; +import nock, { disableNetConnect, enableNetConnect, cleanAll } from 'nock'; +import { jsLogger } from '@map-colonies/js-logger'; +import type { Logger } from '@map-colonies/js-logger'; +import { SharedLuaSource } from '@src/sync/layerSource/sharedLuaSource'; +import type { FileDownloader } from '@src/sync/layerSource/types'; +import { FsRepository } from '@src/common/fs/fsRepository'; +import type { SharedLuaSourceConfig } from '@src/common/interfaces'; + +const ID_FIELD = 'layer_id'; +const NAME_FIELD = 'layer_name'; +const ALIAS_FIELD = 'layer_alias'; +const BUCKET = 'test-bucket'; +const FILE_NAME = 'query.lua'; +const ENRICHMENT_ORIGIN = 'http://mock-api'; + +const makeLua = (entries: string): string => `local layers = {\n${entries}\n}`; + +const makeEntry = (layerId: string, layerName: string, alias: string): string => + ` { ${ID_FIELD} = '${layerId}', ${NAME_FIELD} = '${layerName}', ${ALIAS_FIELD} = '${alias}' },`; + +describe('SharedLuaSource', function () { + let tmpDir: string; + let logger: Logger; + let fsRepository: FsRepository; + + const makeS3Repository = (content: string): FileDownloader => { + const filePath = join(tmpDir, FILE_NAME); + writeFileSync(filePath, content); + return { downloadFile: vi.fn().mockResolvedValue(filePath) }; + }; + + const disabledEnrichmentConfig: SharedLuaSourceConfig = { + type: 'sharedLua', + fileName: FILE_NAME, + layersVariable: 'layers', + aliasFieldName: ALIAS_FIELD, + idFieldName: ID_FIELD, + nameFieldName: NAME_FIELD, + enrichment: { enabled: false }, + }; + + beforeAll(async function () { + logger = await jsLogger({ enabled: false }); + fsRepository = new FsRepository(logger); + disableNetConnect(); + }); + + beforeEach(function () { + tmpDir = mkdtempSync(join(tmpdir(), 'shared-lua-source-test-')); + cleanAll(); + }); + + afterEach(function () { + rmSync(tmpDir, { recursive: true }); + cleanAll(); + }); + + afterAll(function () { + enableNetConnect(); + }); + + describe('Happy Path', function () { + it('should resolve a known layer with its id and alias', async function () { + const s3Repository = makeS3Repository(makeLua(makeEntry('1', 'buildings', 'Buildings'))); + const source = new SharedLuaSource(s3Repository, fsRepository, logger, BUCKET, disabledEnrichmentConfig); + + const records = await source.tick([{ layerName: 'buildings', enums: [] }]); + const record = records.get('buildings'); + + expect(record?.layerId).toBe(1); + expect(record?.alias).toBe('Buildings'); + expect(record?.source).toBe('sharedLua'); + expect(record?.propertyAliases.size).toBe(0); + }); + + it('should fetch property aliases from the enrichment API when enabled', async function () { + const enrichmentConfig: SharedLuaSourceConfig = { + ...disabledEnrichmentConfig, + enrichment: { + enabled: true, + api: `${ENRICHMENT_ORIGIN}/{layerName}`, + propertiesPath: 'fields_list', + aliasField: 'display_name', + requestTimeoutMilliseconds: 5000, + }, + }; + nock(ENRICHMENT_ORIGIN) + .get('/buildings') + .reply(200, { fields_list: { name: { display_name: 'Name' } } }); + + const s3Repository = makeS3Repository(makeLua(makeEntry('1', 'buildings', 'Buildings'))); + const source = new SharedLuaSource(s3Repository, fsRepository, logger, BUCKET, enrichmentConfig); + + const records = await source.tick([{ layerName: 'buildings', enums: [] }]); + const record = records.get('buildings'); + + expect(record?.propertyAliases.get('name')).toBe('Name'); + }); + }); + + describe('Sad Path', function () { + it('should not include a layer absent from the lua file', async function () { + const s3Repository = makeS3Repository(makeLua(makeEntry('1', 'buildings', 'Buildings'))); + const source = new SharedLuaSource(s3Repository, fsRepository, logger, BUCKET, disabledEnrichmentConfig); + + const records = await source.tick([{ layerName: 'roads', enums: [] }]); + + expect(records.has('roads')).toBe(false); + }); + }); + + describe('Bad Path', function () { + it('should resolve without property aliases when the enrichment API fails', async function () { + const enrichmentConfig: SharedLuaSourceConfig = { + ...disabledEnrichmentConfig, + enrichment: { + enabled: true, + api: `${ENRICHMENT_ORIGIN}/{layerName}`, + propertiesPath: 'fields_list', + aliasField: 'display_name', + requestTimeoutMilliseconds: 5000, + }, + }; + nock(ENRICHMENT_ORIGIN).get('/buildings').reply(500); + + const s3Repository = makeS3Repository(makeLua(makeEntry('1', 'buildings', 'Buildings'))); + const source = new SharedLuaSource(s3Repository, fsRepository, logger, BUCKET, enrichmentConfig); + + const records = await source.tick([{ layerName: 'buildings', enums: [] }]); + const record = records.get('buildings'); + + expect(record?.layerId).toBe(1); + expect(record?.propertyAliases.size).toBe(0); + }); + }); +});