diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7715614..f373d65 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,6 +68,9 @@ jobs: - name: Run tests run: npm run test:ci + - name: Check data health + run: npm run data-health:check + spell-check: name: Spell Check runs-on: ubuntu-latest diff --git a/data/data-health.json b/data/data-health.json new file mode 100644 index 0000000..ae307d0 --- /dev/null +++ b/data/data-health.json @@ -0,0 +1,1348 @@ +{ + "asOf": "2026-07-18", + "thresholds": { + "models": 30, + "providers": 30, + "ides": 60, + "clis": 60, + "extensions": 60, + "vendors": 90 + }, + "summary": { + "totalRecords": 134, + "recordsWithSources": 0, + "verifiedRecords": 36, + "provenanceComplete": 0, + "staleVerifiedRecords": 0, + "translationPlaceholderValues": 1411, + "danglingRelationships": 5, + "modelBenchmarkCoverage": 28.1, + "productsWithPricing": 48, + "productRecords": 48, + "errors": 0, + "warnings": 41, + "info": 134 + }, + "byCategory": { + "ides": { + "total": 13, + "verified": 0, + "provenanceComplete": 0, + "stale": 0 + }, + "clis": { + "total": 20, + "verified": 2, + "provenanceComplete": 0, + "stale": 0 + }, + "extensions": { + "total": 15, + "verified": 3, + "provenanceComplete": 0, + "stale": 0 + }, + "models": { + "total": 33, + "verified": 28, + "provenanceComplete": 0, + "stale": 0 + }, + "providers": { + "total": 13, + "verified": 1, + "provenanceComplete": 0, + "stale": 0 + }, + "vendors": { + "total": 40, + "verified": 2, + "provenanceComplete": 0, + "stale": 0 + } + }, + "translationsByLocale": { + "de": { + "totalStrings": 329, + "matchingEnglish": 120, + "matchingEnglishPercent": 36.5 + }, + "es": { + "totalStrings": 329, + "matchingEnglish": 116, + "matchingEnglishPercent": 35.3 + }, + "fr": { + "totalStrings": 329, + "matchingEnglish": 126, + "matchingEnglishPercent": 38.3 + }, + "id": { + "totalStrings": 329, + "matchingEnglish": 160, + "matchingEnglishPercent": 48.6 + }, + "ja": { + "totalStrings": 329, + "matchingEnglish": 117, + "matchingEnglishPercent": 35.6 + }, + "ko": { + "totalStrings": 329, + "matchingEnglish": 133, + "matchingEnglishPercent": 40.4 + }, + "pt": { + "totalStrings": 329, + "matchingEnglish": 126, + "matchingEnglishPercent": 38.3 + }, + "ru": { + "totalStrings": 329, + "matchingEnglish": 142, + "matchingEnglishPercent": 43.2 + }, + "tr": { + "totalStrings": 329, + "matchingEnglish": 142, + "matchingEnglishPercent": 43.2 + }, + "zh-Hans": { + "totalStrings": 329, + "matchingEnglish": 116, + "matchingEnglishPercent": 35.3 + }, + "zh-Hant": { + "totalStrings": 329, + "matchingEnglish": 113, + "matchingEnglishPercent": 34.3 + } + }, + "issues": [ + { + "severity": "warning", + "code": "dangling-related-product", + "category": "extensions", + "id": "droid", + "message": "References missing clis/factory." + }, + { + "severity": "warning", + "code": "dangling-related-product", + "category": "extensions", + "id": "qoder", + "message": "References missing ides/qoder-ide." + }, + { + "severity": "warning", + "code": "dangling-related-product", + "category": "ides", + "id": "intellij-idea", + "message": "References missing clis/idea." + }, + { + "severity": "warning", + "code": "dangling-related-product", + "category": "ides", + "id": "vscode", + "message": "References missing clis/code." + }, + { + "severity": "warning", + "code": "dangling-related-product", + "category": "ides", + "id": "zed", + "message": "References missing clis/zed." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "clis", + "id": "claude-code-cli", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "clis", + "id": "kilo-code-cli", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "extensions", + "id": "claude-code", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "extensions", + "id": "gemini-code-assist", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "extensions", + "id": "kilo-code", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "claude-haiku-4-5", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "claude-opus-4", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "claude-opus-4-1", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "claude-opus-4-5", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "claude-sonnet-4", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "claude-sonnet-4-5", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "gemini-2-5-flash", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "gemini-2-5-pro", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "gemini-3-flash", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "gemini-3-pro", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "glm-4-6", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "glm-4-6v", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "glm-4-7", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "gpt-4-1", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "gpt-4o", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "gpt-5", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "gpt-5-1", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "gpt-5-1-codex", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "gpt-5-2", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "gpt-5-codex", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "grok-code-fast-1", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "kimi-k2-0905", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "kimi-k2-thinking", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "minimax-m2", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "minimax-m2-1", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "qwen3-coder-30b-a3b", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "qwen3-coder-480b-a35b", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "models", + "id": "qwen3-coder-plus", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "providers", + "id": "siliconflow", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "vendors", + "id": "kilo", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "warning", + "code": "verified-without-provenance", + "category": "vendors", + "id": "siliconflow", + "message": "Verified record is missing sources, review date, reviewer, or confidence." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "clis", + "id": "amazon-q-developer-cli", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "clis", + "id": "amp-cli", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "clis", + "id": "augment-code-cli", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "clis", + "id": "claude-code-cli", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "clis", + "id": "cline-cli", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "clis", + "id": "codebuddy-cli", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "clis", + "id": "codex-cli", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "clis", + "id": "continue-cli", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "clis", + "id": "cursor-cli", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "clis", + "id": "deepv-code", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "clis", + "id": "droid-cli", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "clis", + "id": "gemini-cli", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "clis", + "id": "github-copilot-cli", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "clis", + "id": "kilo-code-cli", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "clis", + "id": "kimi-cli", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "clis", + "id": "kiro-cli", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "clis", + "id": "kode", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "clis", + "id": "neovate-code", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "clis", + "id": "opencode", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "clis", + "id": "qoder-cli", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "extensions", + "id": "amp", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "extensions", + "id": "augment-code", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "extensions", + "id": "claude-code", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "extensions", + "id": "cline", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "extensions", + "id": "codex", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "extensions", + "id": "continue", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "extensions", + "id": "droid", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "extensions", + "id": "gemini-code-assist", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "extensions", + "id": "github-copilot", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "extensions", + "id": "jetbrains-junie", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "extensions", + "id": "kilo-code", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "extensions", + "id": "qoder", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "extensions", + "id": "roo-code", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "extensions", + "id": "tabnine", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "extensions", + "id": "verdent", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "ides", + "id": "air", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "ides", + "id": "antigravity", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "ides", + "id": "codebuddy", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "ides", + "id": "codeflicker", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "ides", + "id": "cursor", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "ides", + "id": "intellij-idea", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "ides", + "id": "kiro", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "ides", + "id": "qoder", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "ides", + "id": "trae", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "ides", + "id": "verdent-deck", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "ides", + "id": "vscode", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "ides", + "id": "windsurf", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "ides", + "id": "zed", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "claude-haiku-4-5", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "claude-opus-4", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "claude-opus-4-1", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "claude-opus-4-5", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "claude-sonnet-4", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "claude-sonnet-4-5", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "composer", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "deepseek-3-2", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "deepseek-r1", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "deepseek-v3-terminus", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "gemini-2-5-flash", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "gemini-2-5-pro", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "gemini-3-flash", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "gemini-3-pro", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "glm-4-6", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "glm-4-6v", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "glm-4-7", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "gpt-4-1", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "gpt-4o", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "gpt-5", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "gpt-5-1", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "gpt-5-1-codex", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "gpt-5-2", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "gpt-5-codex", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "grok-code-fast-1", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "kat-coder-pro-v1", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "kimi-k2-0905", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "kimi-k2-thinking", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "minimax-m2", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "minimax-m2-1", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "qwen3-coder-30b-a3b", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "qwen3-coder-480b-a35b", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "models", + "id": "qwen3-coder-plus", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "providers", + "id": "alibaba", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "providers", + "id": "anthropic", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "providers", + "id": "deepseek", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "providers", + "id": "google", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "providers", + "id": "kwaikatonai", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "providers", + "id": "meta", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "providers", + "id": "minimax", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "providers", + "id": "moonshot", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "providers", + "id": "openai", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "providers", + "id": "openrouter", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "providers", + "id": "siliconflow", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "providers", + "id": "xai", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "providers", + "id": "z-ai", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "alibaba", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "ant-group", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "anthropic", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "anysphere", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "augment", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "aws", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "bytedance", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "cline", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "cline-bot", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "cognition", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "continue", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "continue-dev", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "deepseek", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "factory-ai", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "github", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "google", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "jetbrains", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "kilo", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "kwai", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "kwaikat", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "meta", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "microsoft", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "minimax", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "moonshot", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "moonshot-ai", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "openai", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "openrouter", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "orionstar", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "roo-code", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "shareai-lab", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "siliconflow", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "sourcegraph", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "sst", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "streamlake", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "tabnine", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "tencent", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "verdent-ai", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "xai", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "z-ai", + "message": "No structured source references are recorded." + }, + { + "severity": "info", + "code": "missing-sources", + "category": "vendors", + "id": "zed-industries", + "message": "No structured source references are recorded." + } + ] +} diff --git a/docs/DATA-HEALTH.md b/docs/DATA-HEALTH.md new file mode 100644 index 0000000..63f21df --- /dev/null +++ b/docs/DATA-HEALTH.md @@ -0,0 +1,111 @@ +# Data Health Report + +Snapshot date: 2026-07-18. Regenerate with `npm run data-health:report`. + +## Scorecard + +| Metric | Value | +| --- | ---: | +| Manifest records | 134 | +| Records with structured sources | 0 | +| Verified records | 36 | +| Verified with complete provenance | 0 | +| Stale verified records | 0 | +| Non-English values identical to English | 1411 | +| Dangling product relationships | 5 | +| Model benchmark coverage | 28.1% | +| Products with pricing | 48/48 | +| Errors / warnings / info | 0 / 41 / 134 | + +## Category Breakdown + +| Category | Total | Verified | Provenance complete | Stale | +| --- | ---: | ---: | ---: | ---: | +| ides | 13 | 0 | 0 | 0 | +| clis | 20 | 2 | 0 | 0 | +| extensions | 15 | 3 | 0 | 0 | +| models | 33 | 28 | 0 | 0 | +| providers | 13 | 1 | 0 | 0 | +| vendors | 40 | 2 | 0 | 0 | + +## Translation Placeholder Proxy + +Exact English matches are a triage signal; product names and technical terms can be intentional. + +| Locale | Comparable strings | Exact English matches | Match rate | +| --- | ---: | ---: | ---: | +| de | 329 | 120 | 36.5% | +| es | 329 | 116 | 35.3% | +| fr | 329 | 126 | 38.3% | +| id | 329 | 160 | 48.6% | +| ja | 329 | 117 | 35.6% | +| ko | 329 | 133 | 40.4% | +| pt | 329 | 126 | 38.3% | +| ru | 329 | 142 | 43.2% | +| tr | 329 | 142 | 43.2% | +| zh-Hans | 329 | 116 | 35.3% | +| zh-Hant | 329 | 113 | 34.3% | + +## Backlog by Issue Type + +| Issue | Count | +| --- | ---: | +| missing-sources | 134 | +| verified-without-provenance | 36 | +| dangling-related-product | 5 | + +## Priority Queue + +Only errors and warnings are listed here. Source migration inventory and translation metrics remain +visible in the scorecards and `data/data-health.json`. + +| Severity | Issue | Record | Detail | +| --- | --- | --- | --- | +| warning | dangling-related-product | extensions/droid | References missing clis/factory. | +| warning | dangling-related-product | extensions/qoder | References missing ides/qoder-ide. | +| warning | dangling-related-product | ides/intellij-idea | References missing clis/idea. | +| warning | dangling-related-product | ides/vscode | References missing clis/code. | +| warning | dangling-related-product | ides/zed | References missing clis/zed. | +| warning | verified-without-provenance | clis/claude-code-cli | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | clis/kilo-code-cli | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | extensions/claude-code | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | extensions/gemini-code-assist | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | extensions/kilo-code | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/claude-haiku-4-5 | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/claude-opus-4 | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/claude-opus-4-1 | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/claude-opus-4-5 | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/claude-sonnet-4 | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/claude-sonnet-4-5 | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/gemini-2-5-flash | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/gemini-2-5-pro | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/gemini-3-flash | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/gemini-3-pro | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/glm-4-6 | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/glm-4-6v | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/glm-4-7 | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/gpt-4-1 | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/gpt-4o | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/gpt-5 | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/gpt-5-1 | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/gpt-5-1-codex | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/gpt-5-2 | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/gpt-5-codex | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/grok-code-fast-1 | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/kimi-k2-0905 | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/kimi-k2-thinking | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/minimax-m2 | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/minimax-m2-1 | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/qwen3-coder-30b-a3b | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/qwen3-coder-480b-a35b | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | models/qwen3-coder-plus | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | providers/siliconflow | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | vendors/kilo | Verified record is missing sources, review date, reviewer, or confidence. | +| warning | verified-without-provenance | vendors/siliconflow | Verified record is missing sources, review date, reviewer, or confidence. | + +## Freshness Thresholds + +Models and providers: 30 days. IDEs, CLIs, and extensions: 60 days. Vendors: 90 days. + +This snapshot is an operational backlog, not a claim that records without findings are independently audited. +Network reachability remains covered by the separate scheduled URL validation workflow. diff --git a/docs/PROJECT-REVIEW-2026-07-18.md b/docs/PROJECT-REVIEW-2026-07-18.md index 6581979..d7e1304 100644 --- a/docs/PROJECT-REVIEW-2026-07-18.md +++ b/docs/PROJECT-REVIEW-2026-07-18.md @@ -70,9 +70,9 @@ Baseline captured on 2026-07-18: - Candidate fields: `sources`, `lastVerifiedAt`, `verifiedBy`, `confidence`, and field-level effective dates where pricing or benchmarks change independently. - [ ] Backfill provenance for all verified entries before treating verification as authoritative. -- [ ] Generate a data-health report covering freshness, missing sources, broken URLs, relationship +- [x] Generate a data-health report covering freshness, missing sources, broken URLs, relationship integrity, translation placeholders, and missing pricing/benchmark values. -- [ ] Add freshness policies (for example 30/60/90-day review thresholds by field/category). +- [x] Add freshness policies (for example 30/60/90-day review thresholds by field/category). - [ ] Automate change discovery from official sources, but keep human review before merge. - [ ] Populate a real changelog from manifest diffs instead of keeping an empty static file. @@ -160,3 +160,6 @@ evidence; completed items are reflected in the checklists and implementation log `null`. - Added route/search regression tests and completed production verification: Biome, TypeScript, 25 tests, manifest validation, generation, Next.js build, and OpenNext Cloudflare bundling. +- Added a reproducible data-health scorecard and JSON snapshot covering provenance, freshness, + relationship integrity, translation placeholders, pricing, and benchmark coverage; CI now rejects + invalid or stale snapshots while leaving legacy migration debt visible as warnings and inventory. diff --git a/package.json b/package.json index 780da4f..42906bb 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,9 @@ "validate:urls": "npx tsx scripts/validate/visit-urls.ts", "validate:urls:all": "npx tsx scripts/validate/visit-urls.ts --locales all --slugs all", "validate:urls:quick": "npx tsx scripts/validate/visit-urls.ts", + "data-health": "npx tsx scripts/validate/data-health.ts", + "data-health:check": "npx tsx scripts/validate/data-health.ts --check-snapshot --fail-on=error", + "data-health:report": "npx tsx scripts/validate/data-health.ts --write", "generate": "npx tsx scripts/generate/index.ts", "generate:manifests": "npx tsx scripts/generate/index.ts manifest-indexes", "generate:metadata": "npx tsx scripts/generate/index.ts metadata", diff --git a/scripts/README.md b/scripts/README.md index a0e18a2..4b511c0 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -12,6 +12,8 @@ Repository automation is written in TypeScript and executed with `tsx`. Prefer t | Sort manifest fields | `npm run refactor:sort-fields` | Reorders manifest JSON using schema order | | Fetch GitHub stars | `npm run fetch:github-stars` | Updates `data/github-stars.json` | | Validate manifests and data | `npm run test:validate` | Runs the validation test suite | +| Check data health | `npm run data-health:check` | Fails on invalid health-report data | +| Refresh data-health snapshot | `npm run data-health:report` | Writes `data/data-health.json` and `docs/DATA-HEALTH.md` | | Validate i18n structure | `npm run validate:i18n` | Checks locale alignment and translation shape | | Validate i18n usage | `npm run validate:i18n-usage` | Checks translation keys referenced by source | | Validate duplicate i18n values | `npm run validate:i18n-duplicates` | Reports duplicated translation content | diff --git a/scripts/validate/data-health.ts b/scripts/validate/data-health.ts new file mode 100644 index 0000000..8afef29 --- /dev/null +++ b/scripts/validate/data-health.ts @@ -0,0 +1,547 @@ +#!/usr/bin/env node + +import fs from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +export const MANIFEST_CATEGORIES = [ + 'ides', + 'clis', + 'extensions', + 'models', + 'providers', + 'vendors', +] as const + +export type ManifestCategory = (typeof MANIFEST_CATEGORIES)[number] +export type IssueSeverity = 'error' | 'warning' | 'info' + +const FRESHNESS_THRESHOLDS: Record = { + models: 30, + providers: 30, + ides: 60, + clis: 60, + extensions: 60, + vendors: 90, +} + +const RELATED_PRODUCT_CATEGORIES: Record = { + ide: 'ides', + cli: 'clis', + extension: 'extensions', +} + +const BENCHMARK_KEYS = [ + 'sweBench', + 'terminalBench', + 'mmmu', + 'mmmuPro', + 'webDevArena', + 'sciCode', + 'liveCodeBench', +] as const + +export interface ManifestRecord { + category: ManifestCategory + filePath: string + data: Record +} + +export interface TranslationCatalog { + locale: string + values: Record +} + +export interface DataHealthIssue { + severity: IssueSeverity + code: string + category: ManifestCategory + id: string + message: string +} + +interface CategoryHealth { + total: number + verified: number + provenanceComplete: number + stale: number +} + +interface TranslationHealth { + totalStrings: number + matchingEnglish: number + matchingEnglishPercent: number +} + +export interface DataHealthReport { + asOf: string + thresholds: Record + summary: { + totalRecords: number + recordsWithSources: number + verifiedRecords: number + provenanceComplete: number + staleVerifiedRecords: number + translationPlaceholderValues: number + danglingRelationships: number + modelBenchmarkCoverage: number + productsWithPricing: number + productRecords: number + errors: number + warnings: number + info: number + } + byCategory: Record + translationsByLocale: Record + issues: DataHealthIssue[] +} + +function getString(value: unknown): string | null { + return typeof value === 'string' && value.trim() ? value.trim() : null +} + +function getId(record: ManifestRecord): string { + return getString(record.data.id) ?? path.basename(record.filePath, '.json') +} + +function hasCompleteProvenance(data: Record): boolean { + return ( + Array.isArray(data.sources) && + data.sources.length > 0 && + Boolean(getString(data.lastVerifiedAt)) && + Boolean(getString(data.verifiedBy)) && + ['high', 'medium', 'low'].includes(getString(data.confidence) ?? '') + ) +} + +function hasSources(data: Record): boolean { + return Array.isArray(data.sources) && data.sources.length > 0 +} + +function parseIsoDate(value: string): Date | null { + if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) return null + const date = new Date(`${value}T00:00:00.000Z`) + return !Number.isNaN(date.getTime()) && date.toISOString().slice(0, 10) === value ? date : null +} + +function differenceInDays(asOf: Date, date: Date): number { + return Math.floor((asOf.getTime() - date.getTime()) / 86_400_000) +} + +function countBenchmarkCoverage(records: ManifestRecord[]): number { + let populated = 0 + let possible = 0 + for (const record of records.filter(item => item.category === 'models')) { + const benchmarks = record.data.benchmarks + if (!benchmarks || typeof benchmarks !== 'object' || Array.isArray(benchmarks)) continue + const values = benchmarks as Record + for (const key of BENCHMARK_KEYS) { + possible++ + if (typeof values[key] === 'number') populated++ + } + } + return possible === 0 ? 0 : Number(((populated / possible) * 100).toFixed(1)) +} + +function countPricingCoverage(records: ManifestRecord[]): { + covered: number + total: number +} { + const products = records.filter(record => + ['ides', 'clis', 'extensions'].includes(record.category) + ) + return { + covered: products.filter( + record => Array.isArray(record.data.pricing) && record.data.pricing.length > 0 + ).length, + total: products.length, + } +} + +export function loadManifestRecords(rootDir: string): ManifestRecord[] { + const records: ManifestRecord[] = [] + for (const category of MANIFEST_CATEGORIES) { + const directory = path.join(rootDir, 'manifests', category) + for (const fileName of fs + .readdirSync(directory) + .filter(file => file.endsWith('.json')) + .sort()) { + const filePath = path.join(directory, fileName) + records.push({ + category, + filePath, + data: JSON.parse(fs.readFileSync(filePath, 'utf8')) as Record, + }) + } + } + return records +} + +function flattenStrings(value: unknown, prefix = ''): Record { + if (typeof value === 'string') return { [prefix]: value } + if (!value || typeof value !== 'object' || Array.isArray(value)) return {} + + return Object.fromEntries( + Object.entries(value).flatMap(([key, child]) => + Object.entries(flattenStrings(child, prefix ? `${prefix}.${key}` : key)) + ) + ) +} + +function listJsonFiles(directory: string): string[] { + return fs.readdirSync(directory, { withFileTypes: true }).flatMap(entry => { + const entryPath = path.join(directory, entry.name) + if (entry.isDirectory()) return listJsonFiles(entryPath) + return entry.isFile() && entry.name.endsWith('.json') ? [entryPath] : [] + }) +} + +export function loadTranslationCatalogs(rootDir: string): TranslationCatalog[] { + const translationsDir = path.join(rootDir, 'translations') + return fs + .readdirSync(translationsDir, { withFileTypes: true }) + .filter(entry => entry.isDirectory()) + .map(entry => { + const localeDir = path.join(translationsDir, entry.name) + const values = Object.fromEntries( + listJsonFiles(localeDir).flatMap(filePath => { + const relativePath = path.relative(localeDir, filePath).replaceAll(path.sep, '/') + const parsed = JSON.parse(fs.readFileSync(filePath, 'utf8')) as unknown + return Object.entries(flattenStrings(parsed)).map(([key, value]) => [ + `${relativePath}:${key}`, + value, + ]) + }) + ) + return { locale: entry.name, values } + }) + .sort((a, b) => a.locale.localeCompare(b.locale)) +} + +export function analyzeDataHealth( + records: ManifestRecord[], + asOfInput: string, + translationCatalogs: TranslationCatalog[] = [] +): DataHealthReport { + const asOf = parseIsoDate(asOfInput) + if (!asOf) throw new Error(`Invalid --as-of date: ${asOfInput}`) + + const issues: DataHealthIssue[] = [] + const idsByCategory = new Map>() + const byCategory = Object.fromEntries( + MANIFEST_CATEGORIES.map(category => [ + category, + { total: 0, verified: 0, provenanceComplete: 0, stale: 0 }, + ]) + ) as Record + + for (const category of MANIFEST_CATEGORIES) { + idsByCategory.set( + category, + new Set(records.filter(record => record.category === category).map(getId)) + ) + } + + for (const record of records) { + const id = getId(record) + const categoryHealth = byCategory[record.category] + categoryHealth.total++ + const verified = record.data.verified === true + const provenanceComplete = hasCompleteProvenance(record.data) + + if (!hasSources(record.data)) { + issues.push({ + severity: 'info', + code: 'missing-sources', + category: record.category, + id, + message: 'No structured source references are recorded.', + }) + } + + if (verified) { + categoryHealth.verified++ + if (provenanceComplete) categoryHealth.provenanceComplete++ + else { + issues.push({ + severity: 'warning', + code: 'verified-without-provenance', + category: record.category, + id, + message: 'Verified record is missing sources, review date, reviewer, or confidence.', + }) + } + } + + const lastVerifiedAt = getString(record.data.lastVerifiedAt) + if (lastVerifiedAt) { + const verifiedDate = parseIsoDate(lastVerifiedAt) + if (!verifiedDate) { + issues.push({ + severity: 'error', + code: 'invalid-verification-date', + category: record.category, + id, + message: `lastVerifiedAt is not a valid date: ${lastVerifiedAt}`, + }) + } else { + const age = differenceInDays(asOf, verifiedDate) + if (age < 0) { + issues.push({ + severity: 'error', + code: 'future-verification-date', + category: record.category, + id, + message: `lastVerifiedAt is ${Math.abs(age)} days after the report date.`, + }) + } else if (verified && age > FRESHNESS_THRESHOLDS[record.category]) { + categoryHealth.stale++ + issues.push({ + severity: 'warning', + code: 'stale-verification', + category: record.category, + id, + message: `Last reviewed ${age} days ago; threshold is ${FRESHNESS_THRESHOLDS[record.category]} days.`, + }) + } + } + } + + const relatedProducts = record.data.relatedProducts + if (Array.isArray(relatedProducts)) { + for (const relation of relatedProducts) { + if (!relation || typeof relation !== 'object' || Array.isArray(relation)) continue + const related = relation as Record + const targetCategory = RELATED_PRODUCT_CATEGORIES[getString(related.type) ?? ''] + const productId = getString(related.productId) + if (targetCategory && productId && !idsByCategory.get(targetCategory)?.has(productId)) { + issues.push({ + severity: 'warning', + code: 'dangling-related-product', + category: record.category, + id, + message: `References missing ${targetCategory}/${productId}.`, + }) + } + } + } + } + + issues.sort((a, b) => { + const order: Record = { error: 0, warning: 1, info: 2 } + return ( + order[a.severity] - order[b.severity] || + a.code.localeCompare(b.code) || + a.category.localeCompare(b.category) || + a.id.localeCompare(b.id) + ) + }) + + const english = translationCatalogs.find(catalog => catalog.locale === 'en')?.values ?? {} + const translationsByLocale = Object.fromEntries( + translationCatalogs + .filter(catalog => catalog.locale !== 'en') + .map(catalog => { + const comparable = Object.entries(catalog.values).filter(([key]) => key in english) + const matchingEnglish = comparable.filter(([key, value]) => english[key] === value).length + return [ + catalog.locale, + { + totalStrings: comparable.length, + matchingEnglish, + matchingEnglishPercent: + comparable.length === 0 + ? 0 + : Number(((matchingEnglish / comparable.length) * 100).toFixed(1)), + }, + ] + }) + ) + const pricing = countPricingCoverage(records) + return { + asOf: asOfInput, + thresholds: { ...FRESHNESS_THRESHOLDS }, + summary: { + totalRecords: records.length, + recordsWithSources: records.filter(record => hasSources(record.data)).length, + verifiedRecords: records.filter(record => record.data.verified === true).length, + provenanceComplete: records.filter( + record => record.data.verified === true && hasCompleteProvenance(record.data) + ).length, + staleVerifiedRecords: MANIFEST_CATEGORIES.reduce( + (total, category) => total + byCategory[category].stale, + 0 + ), + translationPlaceholderValues: Object.values(translationsByLocale).reduce( + (total, locale) => total + locale.matchingEnglish, + 0 + ), + danglingRelationships: issues.filter(issue => issue.code === 'dangling-related-product') + .length, + modelBenchmarkCoverage: countBenchmarkCoverage(records), + productsWithPricing: pricing.covered, + productRecords: pricing.total, + errors: issues.filter(issue => issue.severity === 'error').length, + warnings: issues.filter(issue => issue.severity === 'warning').length, + info: issues.filter(issue => issue.severity === 'info').length, + }, + byCategory, + translationsByLocale, + issues, + } +} + +export function renderDataHealthMarkdown(report: DataHealthReport): string { + const issueCounts = new Map() + for (const issue of report.issues) + issueCounts.set(issue.code, (issueCounts.get(issue.code) ?? 0) + 1) + + const categoryRows = MANIFEST_CATEGORIES.map(category => { + const health = report.byCategory[category] + return `| ${category} | ${health.total} | ${health.verified} | ${health.provenanceComplete} | ${health.stale} |` + }).join('\n') + const translationRows = Object.entries(report.translationsByLocale) + .map( + ([locale, health]) => + `| ${locale} | ${health.totalStrings} | ${health.matchingEnglish} | ${health.matchingEnglishPercent}% |` + ) + .join('\n') + const backlogRows = [...issueCounts.entries()] + .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) + .map(([code, count]) => `| ${code} | ${count} |`) + .join('\n') + const priorityIssues = report.issues + .filter(issue => issue.severity !== 'info') + .slice(0, 100) + .map( + issue => + `| ${issue.severity} | ${issue.code} | ${issue.category}/${issue.id} | ${issue.message.replaceAll('|', '\\|')} |` + ) + .join('\n') + + return `# Data Health Report + +Snapshot date: ${report.asOf}. Regenerate with \`npm run data-health:report\`. + +## Scorecard + +| Metric | Value | +| --- | ---: | +| Manifest records | ${report.summary.totalRecords} | +| Records with structured sources | ${report.summary.recordsWithSources} | +| Verified records | ${report.summary.verifiedRecords} | +| Verified with complete provenance | ${report.summary.provenanceComplete} | +| Stale verified records | ${report.summary.staleVerifiedRecords} | +| Non-English values identical to English | ${report.summary.translationPlaceholderValues} | +| Dangling product relationships | ${report.summary.danglingRelationships} | +| Model benchmark coverage | ${report.summary.modelBenchmarkCoverage}% | +| Products with pricing | ${report.summary.productsWithPricing}/${report.summary.productRecords} | +| Errors / warnings / info | ${report.summary.errors} / ${report.summary.warnings} / ${report.summary.info} | + +## Category Breakdown + +| Category | Total | Verified | Provenance complete | Stale | +| --- | ---: | ---: | ---: | ---: | +${categoryRows} + +## Translation Placeholder Proxy + +Exact English matches are a triage signal; product names and technical terms can be intentional. + +| Locale | Comparable strings | Exact English matches | Match rate | +| --- | ---: | ---: | ---: | +${translationRows || '| None | 0 | 0 | 0% |'} + +## Backlog by Issue Type + +| Issue | Count | +| --- | ---: | +${backlogRows || '| None | 0 |'} + +## Priority Queue + +Only errors and warnings are listed here. Source migration inventory and translation metrics remain +visible in the scorecards and \`data/data-health.json\`. + +| Severity | Issue | Record | Detail | +| --- | --- | --- | --- | +${priorityIssues || '| — | — | — | No priority issues. |'} + +## Freshness Thresholds + +Models and providers: 30 days. IDEs, CLIs, and extensions: 60 days. Vendors: 90 days. + +This snapshot is an operational backlog, not a claim that records without findings are independently audited. +Network reachability remains covered by the separate scheduled URL validation workflow. +` +} + +function renderConsoleSummary(report: DataHealthReport): string { + return [ + `Data health as of ${report.asOf}`, + `Records: ${report.summary.totalRecords}`, + `Structured sources: ${report.summary.recordsWithSources}/${report.summary.totalRecords}`, + `Verified provenance: ${report.summary.provenanceComplete}/${report.summary.verifiedRecords}`, + `English-identical translations: ${report.summary.translationPlaceholderValues}`, + `Dangling relationships: ${report.summary.danglingRelationships}`, + `Model benchmark coverage: ${report.summary.modelBenchmarkCoverage}%`, + `Issues: ${report.summary.errors} errors, ${report.summary.warnings} warnings, ${report.summary.info} info`, + ].join('\n') +} + +function getArgument(name: string): string | null { + const prefix = `--${name}=` + return process.argv.find(argument => argument.startsWith(prefix))?.slice(prefix.length) ?? null +} + +async function main(): Promise { + const rootDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..') + const snapshotPath = path.join(rootDir, 'data', 'data-health.json') + const markdownPath = path.join(rootDir, 'docs', 'DATA-HEALTH.md') + const checkSnapshot = process.argv.includes('--check-snapshot') + const snapshotAsOf = checkSnapshot + ? getString((JSON.parse(fs.readFileSync(snapshotPath, 'utf8')) as Record).asOf) + : null + const asOf = getArgument('as-of') ?? snapshotAsOf ?? new Date().toISOString().slice(0, 10) + const report = analyzeDataHealth( + loadManifestRecords(rootDir), + asOf, + loadTranslationCatalogs(rootDir) + ) + console.log(renderConsoleSummary(report)) + const jsonOutput = `${JSON.stringify(report, null, 2)}\n` + const markdownOutput = renderDataHealthMarkdown(report) + + if (process.argv.includes('--write')) { + fs.writeFileSync(snapshotPath, jsonOutput, 'utf8') + fs.writeFileSync(markdownPath, markdownOutput, 'utf8') + console.log('Wrote data/data-health.json and docs/DATA-HEALTH.md') + } + + if ( + checkSnapshot && + (fs.readFileSync(snapshotPath, 'utf8') !== jsonOutput || + fs.readFileSync(markdownPath, 'utf8') !== markdownOutput) + ) { + throw new Error('Data-health snapshot is stale; run npm run data-health:report') + } + + const failOn = getArgument('fail-on') ?? 'never' + if (!['error', 'warning', 'never'].includes(failOn)) { + throw new Error(`Invalid --fail-on value: ${failOn}`) + } + if ( + (failOn === 'error' && report.summary.errors > 0) || + (failOn === 'warning' && report.summary.errors + report.summary.warnings > 0) + ) { + process.exitCode = 1 + } +} + +const currentFile = fileURLToPath(import.meta.url) +if (process.argv[1] && path.resolve(process.argv[1]) === currentFile) { + main().catch(error => { + console.error(error) + process.exitCode = 1 + }) +} diff --git a/tests/data-health.test.ts b/tests/data-health.test.ts new file mode 100644 index 0000000..3622a5e --- /dev/null +++ b/tests/data-health.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from 'vitest' + +import { + analyzeDataHealth, + type ManifestRecord, + renderDataHealthMarkdown, +} from '../scripts/validate/data-health' + +function record( + category: ManifestRecord['category'], + id: string, + data: Record = {} +): ManifestRecord { + return { + category, + filePath: `/repo/manifests/${category}/${id}.json`, + data: { id, name: id, description: `${id} description`, ...data }, + } +} + +describe('data health reporting', () => { + it('recognizes complete provenance and fresh verification', () => { + const report = analyzeDataHealth( + [ + record('models', 'trusted-model', { + verified: true, + sources: [{ url: 'https://example.com/model', title: 'Model card' }], + lastVerifiedAt: '2026-07-10', + verifiedBy: 'maintainers', + confidence: 'high', + }), + ], + '2026-07-18' + ) + + expect(report.summary).toMatchObject({ + totalRecords: 1, + recordsWithSources: 1, + verifiedRecords: 1, + provenanceComplete: 1, + staleVerifiedRecords: 0, + errors: 0, + warnings: 0, + }) + }) + + it('reports missing provenance, stale reviews, and missing relationships', () => { + const report = analyzeDataHealth( + [ + record('ides', 'editor', { + verified: true, + lastVerifiedAt: '2026-01-01', + relatedProducts: [{ type: 'cli', productId: 'missing-cli' }], + }), + ], + '2026-07-18' + ) + + expect(report.issues.map(issue => issue.code)).toEqual([ + 'dangling-related-product', + 'stale-verification', + 'verified-without-provenance', + 'missing-sources', + ]) + expect(report.summary).toMatchObject({ + recordsWithSources: 0, + staleVerifiedRecords: 1, + danglingRelationships: 1, + warnings: 3, + info: 1, + }) + }) + + it('treats impossible and future verification dates as errors', () => { + const report = analyzeDataHealth( + [ + record('providers', 'invalid-date', { lastVerifiedAt: '2026-02-30' }), + record('providers', 'future-date', { lastVerifiedAt: '2026-07-19' }), + ], + '2026-07-18' + ) + + expect( + report.issues.filter(issue => issue.severity === 'error').map(issue => issue.code) + ).toEqual(['future-verification-date', 'invalid-verification-date']) + }) + + it('captures translation placeholders and renders a durable summary', () => { + const report = analyzeDataHealth([record('vendors', 'example')], '2026-07-18', [ + { locale: 'en', values: { greeting: 'Hello', product: 'Codex' } }, + { locale: 'zh-Hans', values: { greeting: '你好', product: 'Codex' } }, + ]) + const markdown = renderDataHealthMarkdown(report) + + expect(report.summary.translationPlaceholderValues).toBe(1) + expect(markdown).toContain('| Records with structured sources | 0 |') + expect(markdown).toContain('| zh-Hans | 2 | 1 | 50% |') + }) + + it('rejects a malformed report date', () => { + expect(() => analyzeDataHealth([], '2026-02-30')).toThrow('Invalid --as-of date') + }) +})