Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 8 additions & 71 deletions data/data-health.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,18 @@
},
"summary": {
"totalRecords": 134,
"recordsWithSources": 0,
"verifiedRecords": 36,
"provenanceComplete": 0,
"recordsWithSources": 6,
"verifiedRecords": 39,
"provenanceComplete": 6,
"staleVerifiedRecords": 0,
"translationPlaceholderValues": 1411,
"danglingRelationships": 5,
"modelBenchmarkCoverage": 28.1,
"productsWithPricing": 48,
"productRecords": 48,
"errors": 0,
"warnings": 41,
"info": 134
"warnings": 38,
"info": 128
},
"byCategory": {
"ides": {
Expand All @@ -45,13 +45,13 @@
"models": {
"total": 33,
"verified": 28,
"provenanceComplete": 0,
"provenanceComplete": 3,
"stale": 0
},
"providers": {
"total": 13,
"verified": 1,
"provenanceComplete": 0,
"verified": 4,
"provenanceComplete": 3,
"stale": 0
},
"vendors": {
Expand Down Expand Up @@ -189,13 +189,6 @@
"id": "kilo-code",
"message": "Verified record is missing sources, review date, reviewer, or confidence."
},
{
"severity": "warning",
"code": "verified-without-provenance",
"category": "models",
"id": "claude-haiku-4-5",
"message": "Verified record is missing sources, review date, reviewer, or confidence."
},
{
"severity": "warning",
"code": "verified-without-provenance",
Expand Down Expand Up @@ -245,13 +238,6 @@
"id": "gemini-2-5-pro",
"message": "Verified record is missing sources, review date, reviewer, or confidence."
},
{
"severity": "warning",
"code": "verified-without-provenance",
"category": "models",
"id": "gemini-3-flash",
"message": "Verified record is missing sources, review date, reviewer, or confidence."
},
{
"severity": "warning",
"code": "verified-without-provenance",
Expand Down Expand Up @@ -315,13 +301,6 @@
"id": "gpt-5-1-codex",
"message": "Verified record is missing sources, review date, reviewer, or confidence."
},
{
"severity": "warning",
"code": "verified-without-provenance",
"category": "models",
"id": "gpt-5-2",
"message": "Verified record is missing sources, review date, reviewer, or confidence."
},
{
"severity": "warning",
"code": "verified-without-provenance",
Expand Down Expand Up @@ -742,13 +721,6 @@
"id": "zed",
"message": "No structured source references are recorded."
},
{
"severity": "info",
"code": "missing-sources",
"category": "models",
"id": "claude-haiku-4-5",
"message": "No structured source references are recorded."
},
{
"severity": "info",
"code": "missing-sources",
Expand Down Expand Up @@ -826,13 +798,6 @@
"id": "gemini-2-5-pro",
"message": "No structured source references are recorded."
},
{
"severity": "info",
"code": "missing-sources",
"category": "models",
"id": "gemini-3-flash",
"message": "No structured source references are recorded."
},
{
"severity": "info",
"code": "missing-sources",
Expand Down Expand Up @@ -896,13 +861,6 @@
"id": "gpt-5-1-codex",
"message": "No structured source references are recorded."
},
{
"severity": "info",
"code": "missing-sources",
"category": "models",
"id": "gpt-5-2",
"message": "No structured source references are recorded."
},
{
"severity": "info",
"code": "missing-sources",
Expand Down Expand Up @@ -980,27 +938,13 @@
"id": "alibaba",
"message": "No structured source references are recorded."
},
{
"severity": "info",
"code": "missing-sources",
"category": "providers",
"id": "anthropic",
"message": "No structured source references are recorded."
},
{
"severity": "info",
"code": "missing-sources",
"category": "providers",
"id": "deepseek",
"message": "No structured source references are recorded."
},
{
"severity": "info",
"code": "missing-sources",
"category": "providers",
"id": "google",
"message": "No structured source references are recorded."
},
{
"severity": "info",
"code": "missing-sources",
Expand Down Expand Up @@ -1029,13 +973,6 @@
"id": "moonshot",
"message": "No structured source references are recorded."
},
{
"severity": "info",
"code": "missing-sources",
"category": "providers",
"id": "openai",
"message": "No structured source references are recorded."
},
{
"severity": "info",
"code": "missing-sources",
Expand Down
19 changes: 8 additions & 11 deletions docs/DATA-HEALTH.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,15 @@ Snapshot date: 2026-07-18. Regenerate with `npm run data-health:report`.
| Metric | Value |
| --- | ---: |
| Manifest records | 134 |
| Records with structured sources | 0 |
| Verified records | 36 |
| Verified with complete provenance | 0 |
| Records with structured sources | 6 |
| Verified records | 39 |
| Verified with complete provenance | 6 |
| Stale verified records | 0 |
| Non-English values identical to English | 1411 |
| Dangling product relationships | 5 |
| Model benchmark coverage | 28.1% |
| Products with pricing | 48/48 |
| Errors / warnings / info | 0 / 41 / 134 |
| Errors / warnings / info | 0 / 38 / 128 |

## Category Breakdown

Expand All @@ -24,8 +24,8 @@ Snapshot date: 2026-07-18. Regenerate with `npm run data-health:report`.
| ides | 13 | 0 | 0 | 0 |
| clis | 20 | 2 | 0 | 0 |
| extensions | 15 | 3 | 0 | 0 |
| models | 33 | 28 | 0 | 0 |
| providers | 13 | 1 | 0 | 0 |
| models | 33 | 28 | 3 | 0 |
| providers | 13 | 4 | 3 | 0 |
| vendors | 40 | 2 | 0 | 0 |

## Translation Placeholder Proxy
Expand All @@ -50,8 +50,8 @@ Exact English matches are a triage signal; product names and technical terms can

| Issue | Count |
| --- | ---: |
| missing-sources | 134 |
| verified-without-provenance | 36 |
| missing-sources | 128 |
| verified-without-provenance | 33 |
| dangling-related-product | 5 |

## Priority Queue
Expand All @@ -71,15 +71,13 @@ visible in the scorecards and `data/data-health.json`.
| warning | verified-without-provenance | extensions/claude-code | Verified record is missing sources, review date, reviewer, or confidence. |
| warning | verified-without-provenance | extensions/gemini-code-assist | Verified record is missing sources, review date, reviewer, or confidence. |
| warning | verified-without-provenance | extensions/kilo-code | Verified record is missing sources, review date, reviewer, or confidence. |
| warning | verified-without-provenance | models/claude-haiku-4-5 | Verified record is missing sources, review date, reviewer, or confidence. |
| warning | verified-without-provenance | models/claude-opus-4 | Verified record is missing sources, review date, reviewer, or confidence. |
| warning | verified-without-provenance | models/claude-opus-4-1 | Verified record is missing sources, review date, reviewer, or confidence. |
| warning | verified-without-provenance | models/claude-opus-4-5 | Verified record is missing sources, review date, reviewer, or confidence. |
| warning | verified-without-provenance | models/claude-sonnet-4 | Verified record is missing sources, review date, reviewer, or confidence. |
| warning | verified-without-provenance | models/claude-sonnet-4-5 | Verified record is missing sources, review date, reviewer, or confidence. |
| warning | verified-without-provenance | models/gemini-2-5-flash | Verified record is missing sources, review date, reviewer, or confidence. |
| warning | verified-without-provenance | models/gemini-2-5-pro | Verified record is missing sources, review date, reviewer, or confidence. |
| warning | verified-without-provenance | models/gemini-3-flash | Verified record is missing sources, review date, reviewer, or confidence. |
| warning | verified-without-provenance | models/gemini-3-pro | Verified record is missing sources, review date, reviewer, or confidence. |
| warning | verified-without-provenance | models/glm-4-6 | Verified record is missing sources, review date, reviewer, or confidence. |
| warning | verified-without-provenance | models/glm-4-6v | Verified record is missing sources, review date, reviewer, or confidence. |
Expand All @@ -89,7 +87,6 @@ visible in the scorecards and `data/data-health.json`.
| warning | verified-without-provenance | models/gpt-5 | Verified record is missing sources, review date, reviewer, or confidence. |
| warning | verified-without-provenance | models/gpt-5-1 | Verified record is missing sources, review date, reviewer, or confidence. |
| warning | verified-without-provenance | models/gpt-5-1-codex | Verified record is missing sources, review date, reviewer, or confidence. |
| warning | verified-without-provenance | models/gpt-5-2 | Verified record is missing sources, review date, reviewer, or confidence. |
| warning | verified-without-provenance | models/gpt-5-codex | Verified record is missing sources, review date, reviewer, or confidence. |
| warning | verified-without-provenance | models/grok-code-fast-1 | Verified record is missing sources, review date, reviewer, or confidence. |
| warning | verified-without-provenance | models/kimi-k2-0905 | Verified record is missing sources, review date, reviewer, or confidence. |
Expand Down
8 changes: 5 additions & 3 deletions docs/DATA-TRUST.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,12 @@ Example:

## Current rollout state

Legacy records may have `verified: true` without structured provenance because the boolean predates these fields. Treat those badges as evidence of a prior team review, not as a current freshness guarantee. They will be backfilled category by category; no source or review date should be invented merely to complete the fields.
Legacy records may have `verified: true` without structured provenance because the boolean predates these fields. Treat those badges as evidence of a prior team review, not as a current freshness guarantee. They are being backfilled category by category; no source or review date should be invented merely to complete the fields. The first official-source batch covers OpenAI, Anthropic, and Google providers plus one representative model from each.

For new entries and material updates, a `verified: true` change should include at least one authoritative source, `lastVerifiedAt`, `verifiedBy`, and `confidence`. Official product documentation and pricing pages are preferred over third-party summaries. Benchmark fields should cite the benchmark owner or an official model report.

## Freshness direction
## Freshness policy

The next data-health automation should report verified records without provenance, records older than their category threshold, broken source URLs, and fields such as pricing or benchmarks with no field-level source. Until that automation is in place, freshness review remains a pull-request responsibility.
Run `npm run data-health:report` after manifest or translation changes and commit both generated snapshots. CI runs `npm run data-health:check` to reject invalid or stale snapshots.

The current review thresholds are 30 days for models and providers, 60 days for IDEs, CLIs, and extensions, and 90 days for vendors. Threshold findings remain warnings so that legacy debt stays visible without blocking unrelated work. Network reachability is checked by the separate scheduled URL workflow; source authority and field coverage remain pull-request review responsibilities.
4 changes: 4 additions & 0 deletions docs/PROJECT-REVIEW-2026-07-18.md
Original file line number Diff line number Diff line change
Expand Up @@ -163,3 +163,7 @@ evidence; completed items are reflected in the checklists and implementation log
- Added a reproducible data-health scorecard and JSON snapshot covering provenance, freshness,
relationship integrity, translation placeholders, pricing, and benchmark coverage; CI now rejects
invalid or stale snapshots while leaving legacy migration debt visible as warnings and inventory.
- Completed the first official-source provenance batch for the OpenAI, Anthropic, and Google
providers and representative GPT-5.2, Claude Haiku 4.5, and Gemini 3 Flash model records.
- Corrected GPT-5.2 from `latest` to `maintained` after its official model page identified it as a
previous frontier model that remains available.
31 changes: 30 additions & 1 deletion manifests/models/claude-haiku-4-5.json
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,37 @@
}
},
"verified": true,
"sources": [
{
"url": "https://platform.claude.com/docs/en/about-claude/models/overview",
"title": "Claude models overview",
"fields": [
"docsUrl",
"contextWindow",
"maxOutput",
"lifecycle",
"knowledgeCutoff",
"inputModalities",
"outputModalities",
"capabilities"
]
},
{
"url": "https://platform.claude.com/docs/en/about-claude/pricing",
"title": "Claude API pricing",
"fields": ["tokenPricing"]
},
{
"url": "https://www.anthropic.com/news/claude-haiku-4-5",
"title": "Introducing Claude Haiku 4.5",
"fields": ["name", "description", "releaseDate"]
}
],
"lastVerifiedAt": "2026-07-18",
"verifiedBy": "codex-agent",
"confidence": "medium",
"websiteUrl": "https://www.anthropic.com",
"docsUrl": "https://docs.anthropic.com/claude/docs/models-overview",
"docsUrl": "https://platform.claude.com/docs/en/about-claude/models/overview",
"vendor": "Anthropic",
"size": "Unknown",
"contextWindow": 200000,
Expand Down
27 changes: 26 additions & 1 deletion manifests/models/gemini-3-flash.json
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,33 @@
}
},
"verified": true,
"sources": [
{
"url": "https://ai.google.dev/gemini-api/docs/gemini-3",
"title": "Gemini 3 developer guide",
"fields": [
"docsUrl",
"contextWindow",
"maxOutput",
"tokenPricing",
"lifecycle",
"knowledgeCutoff",
"inputModalities",
"outputModalities",
"capabilities"
]
},
{
"url": "https://blog.google/products-and-platforms/products/gemini/gemini-3-flash/",
"title": "Introducing Gemini 3 Flash",
"fields": ["name", "description", "releaseDate"]
}
],
"lastVerifiedAt": "2026-07-18",
"verifiedBy": "codex-agent",
"confidence": "medium",
"websiteUrl": "https://deepmind.google",
"docsUrl": "https://ai.google.dev/gemini-api/docs/models#gemini-3-flash",
"docsUrl": "https://ai.google.dev/gemini-api/docs/gemini-3",
"vendor": "Google",
"size": "Unknown",
"contextWindow": 1048576,
Expand Down
29 changes: 27 additions & 2 deletions manifests/models/gpt-5-2.json
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,33 @@
}
},
"verified": true,
"sources": [
{
"url": "https://developers.openai.com/api/docs/models/gpt-5.2",
"title": "GPT-5.2 model documentation",
"fields": [
"docsUrl",
"contextWindow",
"maxOutput",
"tokenPricing",
"lifecycle",
"knowledgeCutoff",
"inputModalities",
"outputModalities",
"capabilities"
]
},
{
"url": "https://openai.com/index/introducing-gpt-5-2/",
"title": "Introducing GPT-5.2",
"fields": ["name", "description", "releaseDate"]
}
],
"lastVerifiedAt": "2026-07-18",
"verifiedBy": "codex-agent",
"confidence": "medium",
"websiteUrl": "https://openai.com",
"docsUrl": "https://platform.openai.com/docs/models/gpt-5.2",
"docsUrl": "https://developers.openai.com/api/docs/models/gpt-5.2",
"vendor": "OpenAI",
"size": "Unknown",
"contextWindow": 400000,
Expand All @@ -51,7 +76,7 @@
"cache": 0.175
},
"releaseDate": "2025-12-11",
"lifecycle": "latest",
"lifecycle": "maintained",
"knowledgeCutoff": "2025-08-31",
"inputModalities": ["text", "image"],
"outputModalities": ["text"],
Expand Down
Loading
Loading