diff --git a/CHANGELOG.md b/CHANGELOG.md index 967257521..a2ec8a84d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,16 @@ ones are marked like "v1.0.0-fork". ### Added +* **Import an Anki deck to seed known words** (#228): point LWT at a deck you + already study in Anki and it creates the terms for you, working out how well + you know each word from Anki's own scheduling — mature cards become *well + known*, younger ones get a learning status, suspended ones become *ignored*. + You pick the note type, which field holds the term, and the language, since an + `.apkg` records none of that. Importing is create-only, so running it twice is + safe. This is the direction #259 did **not** cover: that one round-trips LWT's + own exports and silently matches nothing in a deck built in Anki. See + `docs/reference/anki-deck-import`. + * **FSRS scheduling groundwork** (#238, phase 2a): LWT now records FSRS-6 memory state per term — stability, difficulty, due date and a review history. Nothing user-visible changes yet: the legacy scoring still drives the review diff --git a/docs-src/.vitepress/config.ts b/docs-src/.vitepress/config.ts index 829a0746f..7422adaab 100644 --- a/docs-src/.vitepress/config.ts +++ b/docs-src/.vitepress/config.ts @@ -81,6 +81,7 @@ export default defineConfig({ { text: 'Text Parsers', link: '/reference/parsers' }, { text: 'Lemmatization', link: '/reference/lemmatization' }, { text: 'Anki Export & Import', link: '/reference/anki-export-import' }, + { text: 'Import an Anki Deck', link: '/reference/anki-deck-import' }, { text: 'Term Scores', link: '/reference/term-scores' }, { text: 'Export Templates', link: '/reference/export-templates' }, { text: 'Database Schema', link: '/reference/database-schema' }, diff --git a/docs-src/reference/anki-deck-import.md b/docs-src/reference/anki-deck-import.md new file mode 100644 index 000000000..d4d83a8a2 --- /dev/null +++ b/docs-src/reference/anki-deck-import.md @@ -0,0 +1,118 @@ +--- +title: "Import an Anki Deck" +description: Seed your LWT vocabulary from a deck you already study in Anki, using Anki's own scheduling data to work out which words you already know. +--- + +# Import an Anki Deck + +If you already study a language in Anki, you probably know thousands of words +that LWT does not know about yet. Importing your deck marks those words as known +in one step, instead of you reclassifying each one as you meet it while reading. + +**Where:** *Terms → Import an Anki deck*, or `/vocabulary/anki-deck/import`. + +::: tip This is not the same as Anki Export & Import +[Anki Export & Import](/reference/anki-export-import) is a **round trip** for +terms that started life in LWT: export them, study in Anki, import the file back +to update them. It matches notes by an `lwt-` identifier that only LWT-exported +files carry. + +This page is the opposite direction: a deck **you built in Anki**, which has no +such identifier, imported to **create new terms**. Feeding an Anki-built deck to +the round-trip importer does nothing at all — it finds nothing to match. +::: + +## Getting the file out of Anki + +In Anki: **File → Export**, choose **Anki Deck Package (.apkg)**, pick the deck, +and keep **Include scheduling information** ticked. The scheduling data is what +lets LWT tell an word you have known for a year from one you saw yesterday. + +## The import in two steps + +**1. Upload.** LWT reads the file and lists the note types it contains, with +their field names and note counts. + +**2. Map.** An `.apkg` records neither a language nor which field means what, so +you choose: + +| Choice | Why LWT has to ask | +| --- | --- | +| **Note type** | A deck can contain several; you may only want one | +| **Term field** | Field names are arbitrary — `Front`, `Expression`, `Word`, `Vocab`… | +| **Translation field** | Optional. Pick `(none)` to import words only | +| **Language** | An `.apkg` does not record one | +| **Word status** | Derived from Anki, or one fixed status for everything | +| **Import tags** | Anki tags become LWT term tags | + +## How word status is decided + +With **Derive from Anki** (the default), each note's status comes from how well +Anki thinks you know it — specifically the card's interval: + +| In Anki | Becomes in LWT | +| --- | --- | +| Suspended | **98** — Ignored | +| Never studied | **1** — Learning, level 1 | +| Interval 1–6 days | **2** | +| Interval 7–13 days | **3** | +| Interval 14–20 days | **4** | +| Interval 21+ days | **99** — Well known | + +The 21-day line is Anki's own definition of a *mature* card, the same threshold +its statistics screen uses, so it should match your intuition about the deck. + +A note with several cards (forward and reverse, say) is judged by its **strongest** +card: knowing the word in one direction is enough to count. A note only counts as +suspended if *every* one of its cards is suspended. + +If that mapping does not suit your deck, choose **Give every word the same +status** instead — useful for a deck you have fully mastered (set everything to +*Well known*) or one you are only starting. + +## What gets cleaned up + +Anki fields are HTML, and real decks lean on it. Before storing a term LWT +removes formatting tags, converts `
` and block boundaries to spaces, decodes +entities like ` ` and `é`, drops `[sound:…]` media references, and +keeps the answer out of `{{c1::…}}` cloze markers. Without that, markup would end +up in the reading view and terms would never match the words in your texts. + +## Importing twice is safe + +The importer only ever **creates**. It never modifies or deletes a term you +already have, so re-importing the same deck — or a bigger version of it later — +adds only what is genuinely new. Words already in LWT are reported as +"already in LWT" and left alone. + +Duplicates *within* one file are collapsed case-insensitively, so forward and +reverse notes for the same word produce one term. + +## Reading the summary + +The summary accounts for every note read: + +- **Terms created** — new terms now in your vocabulary +- **Already in LWT** — the word existed, so it was left untouched +- **Skipped (empty term field)** — the note had nothing in the chosen field +- **Skipped (too long to store)** — the term exceeded 250 characters + +If *everything* was skipped as empty, you almost certainly picked the wrong term +field. Go back and choose another; nothing was written. + +## Limits + +- **Only `.apkg` files.** Anki's newer `.colpkg` whole-collection format and + compressed exports are not supported yet. +- **Requires `pdo_sqlite`**, since an `.apkg` is a zipped SQLite database. See + [Anki Export & Import](/reference/anki-export-import#requirements). +- **No sentences or images.** Only the term, its translation and its tags are + imported. +- **Scheduling is read, not carried over.** Anki's intervals decide the starting + status, but LWT then schedules the term with its own system — the two do not + stay in sync. Live sync is tracked separately and is deliberately deferred. + +## See also + +- [Anki Export & Import](/reference/anki-export-import) — the LWT→Anki→LWT round trip +- [Term Scores](/reference/term-scores) — how LWT schedules reviews afterwards diff --git a/docs-src/reference/anki-export-import.md b/docs-src/reference/anki-export-import.md index c2b6b3b43..dc5194bcc 100644 --- a/docs-src/reference/anki-export-import.md +++ b/docs-src/reference/anki-export-import.md @@ -2,6 +2,16 @@ LWT can export your vocabulary to a real Anki package (`.apkg`) and read changes back from a `.apkg` you re-export from Anki. The flow is **manual round-trip** — not a live sync — but the file format is universal, so any Anki client (desktop, AnkiMobile, AnkiDroid, AnkiWeb) can study the deck. +::: warning Importing a deck you built in Anki? +This page covers the **round trip** for terms that started in LWT: export them, +study in Anki, import the file back to update them. Notes are matched by an +`lwt-` identifier that only LWT-exported files carry, so a deck built in Anki or +downloaded from AnkiWeb has nothing to match and **nothing would change**. + +To seed LWT from a deck you already study, use +[Import an Anki Deck](/reference/anki-deck-import) instead. +::: + ## When to use this - You already have an Anki workflow and want LWT to seed it from your reading. diff --git a/src/Modules/Vocabulary/Application/Services/Anki/AnkiDeckImportService.php b/src/Modules/Vocabulary/Application/Services/Anki/AnkiDeckImportService.php new file mode 100644 index 000000000..1c5e2573f --- /dev/null +++ b/src/Modules/Vocabulary/Application/Services/Anki/AnkiDeckImportService.php @@ -0,0 +1,137 @@ +reader->notes($apkgPath, $settings->notetypeId); + + $created = 0; + $skippedExisting = 0; + $skippedEmpty = 0; + $skippedTooLong = 0; + $statusCounts = []; + $samples = []; + + // Terms already created in this run, so a deck containing the same word + // twice (common with forward/reverse notes split across notetypes) + // does not report a spurious "already existed". + $seen = []; + + foreach ($notes as $note) { + $term = $this->cleanFieldValue($note->field($settings->termField)); + + if ($term === '') { + $skippedEmpty++; + continue; + } + + if (mb_strlen($term) > self::MAX_TERM_LENGTH) { + $skippedTooLong++; + continue; + } + + $termLc = mb_strtolower($term, 'UTF-8'); + if (isset($seen[$termLc])) { + $skippedExisting++; + continue; + } + $seen[$termLc] = true; + + $status = $settings->statusFor($note); + + $result = $this->words->create([ + 'WoLgID' => $settings->languageId, + 'WoText' => $term, + 'WoStatus' => $status, + 'WoTranslation' => $this->translationFor($note, $settings), + 'WoSentence' => '', + 'WoNotes' => '', + 'WoRomanization' => '', + ]); + + if ($result['success'] !== true) { + // WordCrudService reports a duplicate as a failed create; the + // unique key on (WoTextLC, WoLgID) is what actually enforces it, + // so this is the authoritative "already had it" signal. + $skippedExisting++; + continue; + } + + $created++; + $statusCounts[$status] = ($statusCounts[$status] ?? 0) + 1; + + if (count($samples) < self::SAMPLE_SIZE) { + $samples[] = $term; + } + + if ($settings->importTags && $note->tags !== []) { + TermTagService::saveWordTags($result['id'], $note->tags); + } + } + + ksort($statusCounts); + + return new DeckImportResult( + totalNotes: count($notes), + created: $created, + skippedExisting: $skippedExisting, + skippedEmpty: $skippedEmpty, + skippedTooLong: $skippedTooLong, + statusCounts: $statusCounts, + samples: $samples, + ); + } + + private function translationFor(ForeignNote $note, DeckImportSettings $settings): string + { + if ($settings->translationField === null) { + return ''; + } + + return $this->cleanFieldValue($note->field($settings->translationField)); + } + + private function cleanFieldValue(string $value): string + { + return AnkiFieldText::toPlainText($value); + } +} diff --git a/src/Modules/Vocabulary/Application/Services/Anki/DeckImportResult.php b/src/Modules/Vocabulary/Application/Services/Anki/DeckImportResult.php new file mode 100644 index 000000000..355354b9d --- /dev/null +++ b/src/Modules/Vocabulary/Application/Services/Anki/DeckImportResult.php @@ -0,0 +1,34 @@ + $statusCounts status => number of terms created + * @param list $samples A few created terms, for the summary + */ + public function __construct( + public readonly int $totalNotes, + public readonly int $created, + public readonly int $skippedExisting, + public readonly int $skippedEmpty, + public readonly int $skippedTooLong, + public readonly array $statusCounts, + public readonly array $samples, + ) { + } +} diff --git a/src/Modules/Vocabulary/Application/Services/Anki/DeckImportSettings.php b/src/Modules/Vocabulary/Application/Services/Anki/DeckImportSettings.php new file mode 100644 index 000000000..f01125435 --- /dev/null +++ b/src/Modules/Vocabulary/Application/Services/Anki/DeckImportSettings.php @@ -0,0 +1,97 @@ + 98 (ignored) — the user parked it in Anki + * never studied -> 1 (learning) — no evidence of knowledge + * < 21 days -> 2..4 — young, scaled by interval + * >= 21 days -> 99 (well-known) — Anki's own "mature" threshold + */ + public function statusFor(ForeignNote $note): int + { + if (!$this->deriveStatus) { + return $this->fixedStatus; + } + + if ($note->suspended) { + return 98; + } + + if ($note->isNew || $note->interval <= 0) { + return 1; + } + + if ($note->interval >= self::MATURE_INTERVAL_DAYS) { + return 99; + } + + // 1..20 days spread across the middle learning statuses. + if ($note->interval >= 14) { + return 4; + } + if ($note->interval >= 7) { + return 3; + } + + return 2; + } +} diff --git a/src/Modules/Vocabulary/Http/AnkiDeckImportController.php b/src/Modules/Vocabulary/Http/AnkiDeckImportController.php new file mode 100644 index 000000000..c886e325d --- /dev/null +++ b/src/Modules/Vocabulary/Http/AnkiDeckImportController.php @@ -0,0 +1,389 @@ + configure, or configure -> import + * + * The uploaded file is parked in a temp path between the two steps, keyed by a + * token held in the session so the second request cannot be pointed at an + * arbitrary path. + */ +class AnkiDeckImportController extends VocabularyBaseController +{ + private const SESSION_KEY = 'lwt_anki_deck_import'; + + private LanguageFacade $languageFacade; + private ForeignApkgReader $reader; + private ?AnkiDeckImportService $importService; + + public function __construct( + ?LanguageFacade $languageFacade = null, + ?ForeignApkgReader $reader = null, + ?AnkiDeckImportService $importService = null, + ) { + parent::__construct(); + $this->languageFacade = $languageFacade ?? new LanguageFacade(); + $this->reader = $reader ?? new ForeignApkgReader(); + $this->importService = $importService; + } + + /** + * @param array $params Route params (unused). + */ + public function index(array $params): void + { + PageLayoutHelper::renderPageStart('Import an Anki deck', true); + + try { + if (($_SERVER['REQUEST_METHOD'] ?? 'GET') !== 'POST') { + $this->renderUploadForm(null); + } elseif (InputValidator::getString('step') === 'import') { + $this->handleImport(); + } else { + $this->handleUpload(); + } + } catch (Throwable $e) { + $this->renderUploadForm($e->getMessage()); + } + + PageLayoutHelper::renderPageEnd(); + } + + /** + * Step 1: accept the upload, park it, show the mapping form. + */ + private function handleUpload(): void + { + $file = InputValidator::getUploadedFile('apkg'); + if ($file === null) { + $this->renderUploadForm('No file was uploaded.'); + return; + } + + if (!str_ends_with(strtolower($file['name']), '.apkg')) { + $this->renderUploadForm('Only .apkg files are accepted.'); + return; + } + + $parked = tempnam(sys_get_temp_dir(), 'lwt_deck_'); + if ($parked === false) { + $this->renderUploadForm('Could not store the uploaded file.'); + return; + } + if (!move_uploaded_file($file['tmp_name'], $parked)) { + // move_uploaded_file only accepts a genuine HTTP upload; fall back + // to a copy so the flow stays exercisable outside a web request. + if (!copy($file['tmp_name'], $parked)) { + @unlink($parked); + $this->renderUploadForm('Could not store the uploaded file.'); + return; + } + } + + $notetypes = $this->reader->notetypes($parked); + if ($notetypes === []) { + @unlink($parked); + $this->renderUploadForm('No notetypes found in this file — it may not be an Anki deck.'); + return; + } + + $this->rememberParkedFile($parked); + $this->renderMappingForm($notetypes, null); + } + + /** + * Step 2: apply the chosen mapping. + */ + private function handleImport(): void + { + $parked = $this->parkedFile(); + if ($parked === null) { + $this->renderUploadForm('That upload has expired. Please choose the file again.'); + return; + } + + $notetypes = $this->reader->notetypes($parked); + + try { + $settings = new DeckImportSettings( + notetypeId: InputValidator::getInt('notetype') ?? 0, + termField: InputValidator::getString('term_field'), + translationField: $this->optionalField('translation_field'), + languageId: InputValidator::getInt('language') ?? 0, + deriveStatus: InputValidator::getString('status_mode') !== 'fixed', + fixedStatus: InputValidator::getInt('fixed_status') ?? 1, + importTags: InputValidator::getString('import_tags') !== '', + ); + } catch (\InvalidArgumentException $e) { + $this->renderMappingForm($notetypes, $e->getMessage()); + return; + } + + $result = $this->importSvc()->import($parked, $settings); + + $this->forgetParkedFile(); + $this->renderSummary($result, $settings); + } + + private function optionalField(string $key): ?string + { + $value = InputValidator::getString($key); + + return $value === '' ? null : $value; + } + + private function renderUploadForm(?string $error): void + { + echo '

Import an Anki deck

'; + + if ($error !== null) { + echo '
' . $this->esc($error) . '
'; + } + + echo '

Already study this language in Anki? Import the deck and LWT will ' + . 'mark those words as known, so you do not have to reclassify them while reading. ' + . 'Export your deck from Anki with File → Export → Anki Deck Package ' + . '(.apkg), including scheduling information.

'; + + echo '
'; + echo $this->csrfField(); + echo '
' + . '
'; + echo '
' + . '' + . '
'; + echo '
'; + + echo '

This creates new terms. It never changes terms you already ' + . 'have — importing the same deck twice is safe.

'; + } + + /** + * @param list $notetypes + */ + private function renderMappingForm(array $notetypes, ?string $error): void + { + echo '

Import an Anki deck

'; + + if ($error !== null) { + echo '
' . $this->esc($error) . '
'; + } + + echo '

LWT cannot tell which field holds the word, or what language the ' + . 'deck is in — an .apkg does not record either. Choose them below.

'; + + echo '
'; + echo $this->csrfField(); + echo ''; + + // Notetype, with its fields listed so the choice is informed. + echo '
' + . '
'; + + // Field pickers list every field across every notetype; the user picks + // the notetype above and the matching names from the same vocabulary. + $allFields = []; + foreach ($notetypes as $nt) { + foreach ($nt->fields as $field) { + $allFields[$field] = true; + } + } + $fieldNames = array_keys($allFields); + + echo $this->fieldSelect('term_field', 'Field holding the term', $fieldNames, false); + echo $this->fieldSelect('translation_field', 'Field holding the translation', $fieldNames, true); + + // Language. + echo '
' + . '
'; + + // Status rule. + echo '
'; + echo '
'; + echo '

Cards you have known for ' . DeckImportSettings::MATURE_INTERVAL_DAYS + . ' days or more become well known; younger cards get a learning ' + . 'status based on their interval; unstudied cards start at level 1; suspended cards ' + . 'become ignored.

'; + echo '
'; + echo '
' + . '
'; + echo '
'; + + echo '
'; + + echo '
' + . '' + . '
'; + echo '
'; + } + + /** + * @param list $fieldNames + */ + private function fieldSelect(string $name, string $label, array $fieldNames, bool $optional): string + { + $out = '
' + . '
'; + } + + private function renderSummary(DeckImportResult $result, DeckImportSettings $settings): void + { + echo '

Import an Anki deck

'; + + echo '

Imported ' + . $result->created . ' new term' . ($result->created === 1 ? '' : 's') . '.

'; + + echo ''; + echo ''; + echo ''; + echo ''; + if ($result->skippedEmpty > 0) { + echo ''; + } + if ($result->skippedTooLong > 0) { + echo ''; + } + echo '
Notes read' . $result->totalNotes . '
Terms created' . $result->created . '
Already in LWT' . $result->skippedExisting . '
Skipped (empty term field)' . $result->skippedEmpty . '
Skipped (too long to store)' . $result->skippedTooLong . '
'; + + if ($result->statusCounts !== []) { + echo '

Status breakdown

    '; + foreach ($result->statusCounts as $status => $count) { + echo '
  • ' . $this->esc($this->statusLabel($status)) . ': ' . $count . '
    • '; + } + echo '
'; + } + + if ($result->samples !== []) { + echo '

For example: ' + . $this->esc(implode(', ', $result->samples)) . '…

'; + } + + if ($result->created === 0 && $result->skippedEmpty === $result->totalNotes) { + echo '
Every note had an empty term field. ' + . 'The chosen field is probably not the one holding the word — go back and pick ' + . 'a different one.
'; + } + + echo '

View the imported terms ' + . 'Import another deck

'; + } + + private function statusLabel(int $status): string + { + return match ($status) { + 98 => 'Ignored', + 99 => 'Well known', + default => 'Level ' . $status, + }; + } + + private function csrfField(): string + { + return ''; + } + + private function esc(string $value): string + { + return htmlspecialchars($value, ENT_QUOTES, 'UTF-8'); + } + + private function rememberParkedFile(string $path): void + { + if (session_status() === PHP_SESSION_ACTIVE) { + $_SESSION[self::SESSION_KEY] = $path; + } + } + + /** + * The parked upload for this session, if it is still there. + * + * Reading the path from the session rather than the request is what stops + * step 2 being pointed at an arbitrary file on disk. + */ + private function parkedFile(): ?string + { + if (session_status() !== PHP_SESSION_ACTIVE) { + return null; + } + + $path = $_SESSION[self::SESSION_KEY] ?? null; + if (!is_string($path) || !is_file($path)) { + return null; + } + + return $path; + } + + private function forgetParkedFile(): void + { + $path = $this->parkedFile(); + if ($path !== null) { + @unlink($path); + } + unset($_SESSION[self::SESSION_KEY]); + } + + private function importSvc(): AnkiDeckImportService + { + return $this->importService ??= AnkiDeckImportService::default(); + } +} diff --git a/src/Modules/Vocabulary/Http/ApkgController.php b/src/Modules/Vocabulary/Http/ApkgController.php index 34d41b721..5ba1f35f8 100644 --- a/src/Modules/Vocabulary/Http/ApkgController.php +++ b/src/Modules/Vocabulary/Http/ApkgController.php @@ -148,7 +148,7 @@ private function renderImportForm(?string $error, ?ImportResult $result): void . '
  • Updated: ' . $result->updated . '
    • ' . '
  • Unchanged: ' . $result->unchanged . '
    • ' . '
  • Skipped (term not found): ' . $result->skippedMissing . '
    • ' - . '
  • Skipped (no LWT id): ' . $result->skippedUnknown . '
    • ' + . '
  • Not created by LWT: ' . $result->skippedUnknown . '
    • ' . '
  • Demoted to Ignored from suspended: ' . $result->statusSetToIgnored . '
    • ' . '
  • Tag changes applied: ' . $result->tagsChanged . '
    • ' . ''; @@ -173,6 +173,17 @@ private function renderImportForm(?string $error, ?ImportResult $result): void . 'Translations, romanizations, notes, and tags are updated. ' . 'Cards suspended in Anki demote learning-status terms to Ignored.' . '

    '; + + // The commonest wrong turn: arriving here with a deck built in Anki, + // which has no LWT guids and so updates nothing at all. + echo '
    ' + . 'Importing a deck you built in Anki? This page only updates terms ' + . 'that LWT exported in the first place — a deck from Anki or AnkiWeb has nothing here ' + . 'to match against, so nothing would change. Use ' + . 'Import an Anki deck instead, which ' + . 'creates new terms and works out how well you know each word from Anki\'s own ' + . 'scheduling.' + . '
    '; } private function streamDownload(string $path, string $filename): never diff --git a/src/Modules/Vocabulary/Infrastructure/Anki/AnkiFieldText.php b/src/Modules/Vocabulary/Infrastructure/Anki/AnkiFieldText.php new file mode 100644 index 000000000..f9df3f038 --- /dev/null +++ b/src/Modules/Vocabulary/Infrastructure/Anki/AnkiFieldText.php @@ -0,0 +1,38 @@ +` + * wrappers, ``/`` emphasis, `
    ` line breaks, ` `, + * `[sound:...]` media references and `{{c1::...}}` cloze markers. Storing any + * of it raw would put markup into the reading view and stop terms matching the + * words in a text, so it is all reduced to text here. + */ +final class AnkiFieldText +{ + public static function toPlainText(string $value): string + { + // Anki media syntax — meaningless outside Anki, and never part of the word. + $value = preg_replace('/\[sound:[^\]]*\]/u', '', $value) ?? $value; + + // Cloze deletions: {{c1::answer}} and {{c1::answer::hint}} keep the answer. + $value = preg_replace('/\{\{c\d+::(.*?)(?:::[^}]*)?\}\}/u', '$1', $value) ?? $value; + + // Structural breaks carry a word boundary; drop the rest of the markup. + $value = preg_replace('#<\s*(br|/div|/p|/li|/tr|/td)\s*/?\s*>#iu', ' ', $value) ?? $value; + $value = strip_tags($value); + + $value = html_entity_decode($value, ENT_QUOTES | ENT_HTML5, 'UTF-8'); + + // Anki emits non-breaking spaces liberally; treat them as whitespace. + $value = str_replace("\u{00A0}", ' ', $value); + $value = preg_replace('/\s+/u', ' ', $value) ?? $value; + + return trim($value); + } +} diff --git a/src/Modules/Vocabulary/Infrastructure/Anki/ForeignApkgReader.php b/src/Modules/Vocabulary/Infrastructure/Anki/ForeignApkgReader.php new file mode 100644 index 000000000..f74691dd3 --- /dev/null +++ b/src/Modules/Vocabulary/Infrastructure/Anki/ForeignApkgReader.php @@ -0,0 +1,280 @@ + + */ + public function notetypes(string $apkgPath): array + { + return $this->withCollection($apkgPath, function (PDO $pdo): array { + $models = $this->decodeModels($pdo); + $counts = $this->noteCountsByModel($pdo); + + $out = []; + foreach ($models as $mid => $model) { + $out[] = new ForeignNotetype( + id: $mid, + name: $model['name'], + fields: $model['fields'], + noteCount: $counts[$mid] ?? 0, + ); + } + + usort($out, static fn(ForeignNotetype $a, ForeignNotetype $b) => $b->noteCount <=> $a->noteCount); + + return $out; + }); + } + + /** + * Every note using one notetype, with fields unmapped. + * + * @return list + */ + public function notes(string $apkgPath, int $notetypeId): array + { + return $this->withCollection($apkgPath, function (PDO $pdo) use ($notetypeId): array { + $models = $this->decodeModels($pdo); + if (!isset($models[$notetypeId])) { + throw new RuntimeException("Notetype {$notetypeId} not found in this file"); + } + + $fieldNames = $models[$notetypeId]['fields']; + $cardsByNote = $this->cardStateByNote($pdo); + + $stmt = $pdo->prepare('SELECT id, tags, flds FROM notes WHERE mid = ?'); + $stmt->execute([$notetypeId]); + + $out = []; + foreach ($stmt as $row) { + if (!is_array($row)) { + continue; + } + + $values = explode(AnkiSchema::FIELD_SEPARATOR, (string) ($row['flds'] ?? '')); + + $fields = []; + foreach ($fieldNames as $ord => $name) { + $fields[$name] = $values[$ord] ?? ''; + } + + $noteId = (int) ($row['id'] ?? 0); + $state = $cardsByNote[$noteId] ?? ['interval' => 0, 'suspended' => false, 'new' => true]; + + $out[] = new ForeignNote( + fields: $fields, + tags: $this->decodeTags((string) ($row['tags'] ?? '')), + interval: $state['interval'], + suspended: $state['suspended'], + isNew: $state['new'], + ); + } + + return $out; + }); + } + + /** + * Extract the collection to a temp file and hand a PDO handle to $work. + * + * @template T + * @param callable(PDO): T $work + * @return T + */ + private function withCollection(string $apkgPath, callable $work): mixed + { + AnkiSchema::assertSqliteAvailable(); + + if (!is_file($apkgPath)) { + throw new RuntimeException("APKG file not found: {$apkgPath}"); + } + + $zip = new ZipArchive(); + if ($zip->open($apkgPath) !== true) { + throw new RuntimeException("Could not open APKG: {$apkgPath}"); + } + + $collectionName = null; + foreach (['collection.anki21', 'collection.anki2'] as $candidate) { + if ($zip->locateName($candidate) !== false) { + $collectionName = $candidate; + break; + } + } + if ($collectionName === null) { + $zip->close(); + throw new RuntimeException( + 'This file contains no Anki collection. Newer Anki exports using the ' + . '.colpkg format or a compressed collection are not supported yet.' + ); + } + + $contents = $zip->getFromName($collectionName); + $zip->close(); + if ($contents === false) { + throw new RuntimeException("Failed to extract {$collectionName} from APKG"); + } + + $tmpDb = tempnam(sys_get_temp_dir(), 'lwt_apkg_foreign_'); + if ($tmpDb === false) { + throw new RuntimeException('Could not allocate temp file'); + } + file_put_contents($tmpDb, $contents); + + try { + $pdo = new PDO('sqlite:' . $tmpDb); + $pdo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); + + return $work($pdo); + } finally { + unlink($tmpDb); + } + } + + /** + * Notetype id => name + ordered field names. + * + * @return array}> + */ + private function decodeModels(PDO $pdo): array + { + $stmt = $pdo->query('SELECT models FROM col LIMIT 1'); + $modelsJson = $stmt === false ? null : $stmt->fetchColumn(); + if (!is_string($modelsJson) || $modelsJson === '') { + throw new RuntimeException('Could not read col.models'); + } + + /** @var mixed $decoded */ + $decoded = json_decode($modelsJson, true); + if (!is_array($decoded)) { + throw new RuntimeException('Could not decode col.models'); + } + + $out = []; + /** @var mixed $model */ + foreach ($decoded as $mid => $model) { + if (!is_array($model) || !isset($model['flds']) || !is_array($model['flds'])) { + continue; + } + + // Anki stores fields with an explicit ordinal; sort by it rather + // than trusting array order, which shared decks do not guarantee. + $byOrd = []; + /** @var mixed $fld */ + foreach ($model['flds'] as $fld) { + if (!is_array($fld) || !isset($fld['name'], $fld['ord'])) { + continue; + } + $byOrd[(int) $fld['ord']] = (string) $fld['name']; + } + ksort($byOrd); + + if ($byOrd === []) { + continue; + } + + $out[(int) $mid] = [ + 'name' => isset($model['name']) ? (string) $model['name'] : 'Unnamed notetype', + 'fields' => array_values($byOrd), + ]; + } + + return $out; + } + + /** + * @return array notetype id => note count + */ + private function noteCountsByModel(PDO $pdo): array + { + $stmt = $pdo->query('SELECT mid, COUNT(*) AS c FROM notes GROUP BY mid'); + if ($stmt === false) { + return []; + } + + $out = []; + /** @var mixed $row */ + foreach ($stmt as $row) { + if (is_array($row)) { + $out[(int) ($row['mid'] ?? 0)] = (int) ($row['c'] ?? 0); + } + } + + return $out; + } + + /** + * Collapse each note's cards into the facts the status mapping needs. + * + * A note can have several cards (forward/reverse). The strongest card wins: + * if the user knows the word in one direction it is worth importing as + * known, and a note only counts as suspended when every card is. + * + * @return array + */ + private function cardStateByNote(PDO $pdo): array + { + $stmt = $pdo->query('SELECT nid, ivl, queue, type FROM cards'); + if ($stmt === false) { + return []; + } + + $out = []; + foreach ($stmt as $row) { + if (!is_array($row)) { + continue; + } + + $nid = (int) ($row['nid'] ?? 0); + $ivl = (int) ($row['ivl'] ?? 0); + $queue = (int) ($row['queue'] ?? 0); + $type = (int) ($row['type'] ?? 0); + + // Anki stores sub-day learning intervals as negative seconds. + $intervalDays = $ivl < 0 ? 0 : $ivl; + + if (!isset($out[$nid])) { + $out[$nid] = ['interval' => 0, 'suspended' => true, 'new' => true]; + } + + $out[$nid]['interval'] = max($out[$nid]['interval'], $intervalDays); + $out[$nid]['suspended'] = $out[$nid]['suspended'] && $queue === -1; + $out[$nid]['new'] = $out[$nid]['new'] && $type === 0; + } + + return $out; + } + + /** + * @return list + */ + private function decodeTags(string $tags): array + { + $parts = preg_split('/\s+/', trim($tags)) ?: []; + + return array_values(array_filter($parts, static fn(string $t): bool => $t !== '')); + } +} diff --git a/src/Modules/Vocabulary/Infrastructure/Anki/ForeignNote.php b/src/Modules/Vocabulary/Infrastructure/Anki/ForeignNote.php new file mode 100644 index 000000000..3b9752b31 --- /dev/null +++ b/src/Modules/Vocabulary/Infrastructure/Anki/ForeignNote.php @@ -0,0 +1,40 @@ + $fields Field name => value, as stored in Anki + * @param list $tags Anki tags on the note + * @param int $interval Largest `cards.ivl` across the note's + * cards, in days. 0 for unseen cards. + * @param bool $suspended Whether every card is suspended + * @param bool $isNew Whether the note has never been studied + */ + public function __construct( + public readonly array $fields, + public readonly array $tags, + public readonly int $interval, + public readonly bool $suspended, + public readonly bool $isNew, + ) { + } + + /** + * Value of one field, or '' when the notetype has no such field. + */ + public function field(string $name): string + { + return $this->fields[$name] ?? ''; + } +} diff --git a/src/Modules/Vocabulary/Infrastructure/Anki/ForeignNotetype.php b/src/Modules/Vocabulary/Infrastructure/Anki/ForeignNotetype.php new file mode 100644 index 000000000..fcb3f25cc --- /dev/null +++ b/src/Modules/Vocabulary/Infrastructure/Anki/ForeignNotetype.php @@ -0,0 +1,30 @@ + $fields Field names, in Anki's ordinal order + * @param int $noteCount How many notes use this notetype + */ + public function __construct( + public readonly int $id, + public readonly string $name, + public readonly array $fields, + public readonly int $noteCount, + ) { + } +} diff --git a/src/Shared/Infrastructure/Routing/routes.php b/src/Shared/Infrastructure/Routing/routes.php index 4e6c31fc1..4800c32e2 100644 --- a/src/Shared/Infrastructure/Routing/routes.php +++ b/src/Shared/Infrastructure/Routing/routes.php @@ -334,6 +334,20 @@ function registerRoutes(Router $router): void AUTH_MIDDLEWARE ); + // Import a deck built in Anki as new terms (AnkiDeckImportController). + // Distinct from the apkg routes above: those round-trip LWT's own export, + // this one seeds terms from a foreign deck. + $router->get( + '/vocabulary/anki-deck/import', + 'Lwt\\Modules\\Vocabulary\\Http\\AnkiDeckImportController@index', + AUTH_MIDDLEWARE + ); + $router->post( + '/vocabulary/anki-deck/import', + 'Lwt\\Modules\\Vocabulary\\Http\\AnkiDeckImportController@index', + AUTH_MIDDLEWARE + ); + // Review status change during review (iframe/ajax view). // NB: the dead text-reader frame routes (set-status, delete-term, // delete-multi, insert-wellknown, insert-ignore) are unregistered here — diff --git a/tests/backend/Modules/Vocabulary/Application/Services/Anki/AnkiDeckImportIntegrationTest.php b/tests/backend/Modules/Vocabulary/Application/Services/Anki/AnkiDeckImportIntegrationTest.php new file mode 100644 index 000000000..733ee2c87 --- /dev/null +++ b/tests/backend/Modules/Vocabulary/Application/Services/Anki/AnkiDeckImportIntegrationTest.php @@ -0,0 +1,293 @@ +markTestSkipped('pdo_sqlite required to read an .apkg collection'); + } + if (!defined('LWT_TEST_DB_AVAILABLE') || !LWT_TEST_DB_AVAILABLE) { + $this->markTestSkipped('Database connection required'); + } + if (!self::$dbConnected) { + $this->markTestSkipped('Test database setup failed'); + } + + $tmp = tempnam(sys_get_temp_dir(), 'lwt_deckimport_'); + $this->deckPath = $tmp !== false ? $tmp . '.apkg' : ''; + } + + protected function tearDown(): void + { + if ($this->deckPath !== '') { + ForeignDeckBuilder::cleanup($this->deckPath); + } + if (self::$dbConnected && self::$languageId > 0) { + Connection::query( + 'DELETE FROM word_tag_map WHERE WtWoID IN (' + . 'SELECT WoID FROM words WHERE WoLgID = ' . self::$languageId . ')' + ); + Connection::query('DELETE FROM words WHERE WoLgID = ' . self::$languageId); + } + } + + private function settings(bool $derive = true, ?string $translation = 'Back'): DeckImportSettings + { + return new DeckImportSettings( + notetypeId: 1500000000000, + termField: 'Front', + translationField: $translation, + languageId: self::$languageId, + deriveStatus: $derive, + ); + } + + public function testImportsAForeignDeckAndDerivesStatusFromMaturity(): void + { + (new ForeignDeckBuilder()) + ->addMatureNote(['hund', 'the dog'], 200) + ->addYoungNote(['katze', 'the cat'], 8) + ->addNote(['pferd', 'the horse']) + ->addSuspendedNote(['maus', 'the mouse']) + ->build($this->deckPath); + + $result = AnkiDeckImportService::default()->import($this->deckPath, $this->settings()); + + $this->assertSame(4, $result->totalNotes); + $this->assertSame(4, $result->created); + $this->assertSame(0, $result->skippedExisting); + + $rows = Connection::preparedFetchAll( + 'SELECT WoTextLC, WoStatus, WoTranslation FROM words WHERE WoLgID = ? ORDER BY WoTextLC', + [self::$languageId] + ); + + $byTerm = []; + foreach ($rows as $row) { + $byTerm[(string) $row['WoTextLC']] = $row; + } + + $this->assertSame(99, (int) $byTerm['hund']['WoStatus'], 'a mature card is well known'); + $this->assertSame(3, (int) $byTerm['katze']['WoStatus'], '8-day interval lands mid-learning'); + $this->assertSame(1, (int) $byTerm['pferd']['WoStatus'], 'an unstudied card starts at 1'); + $this->assertSame(98, (int) $byTerm['maus']['WoStatus'], 'a suspended card is ignored'); + + $this->assertSame('the dog', (string) $byTerm['hund']['WoTranslation']); + } + + public function testReimportingTheSameDeckCreatesNothingNew(): void + { + (new ForeignDeckBuilder()) + ->addMatureNote(['wiederholung', 'repetition'], 60) + ->build($this->deckPath); + + $service = AnkiDeckImportService::default(); + + $first = $service->import($this->deckPath, $this->settings()); + $this->assertSame(1, $first->created); + + $second = $service->import($this->deckPath, $this->settings()); + $this->assertSame(0, $second->created, 'importing twice must not duplicate'); + $this->assertSame(1, $second->skippedExisting); + + $count = (int) Connection::preparedFetchValue( + 'SELECT COUNT(*) AS value FROM words WHERE WoLgID = ? AND WoTextLC = ?', + [self::$languageId, 'wiederholung'] + ); + $this->assertSame(1, $count); + } + + public function testDuplicateWordsWithinOneDeckAreCollapsed(): void + { + // Forward and reverse notes for the same word are common. + (new ForeignDeckBuilder()) + ->addMatureNote(['gleich', 'same'], 40) + ->addMatureNote(['Gleich', 'same (capitalised)'], 40) + ->build($this->deckPath); + + $result = AnkiDeckImportService::default()->import($this->deckPath, $this->settings()); + + $this->assertSame(1, $result->created, 'case-insensitive duplicates collapse'); + $this->assertSame(1, $result->skippedExisting); + } + + public function testHtmlInFieldsIsStrippedBeforeStoring(): void + { + (new ForeignDeckBuilder()) + ->addMatureNote(['
    der Baum
    [sound:baum.mp3]
    ', 'the tree'], 50) + ->build($this->deckPath); + + AnkiDeckImportService::default()->import($this->deckPath, $this->settings()); + + $row = Connection::preparedFetchOne( + 'SELECT WoText, WoTranslation FROM words WHERE WoLgID = ?', + [self::$languageId] + ); + + $this->assertNotNull($row); + $this->assertSame('der Baum', (string) $row['WoText']); + $this->assertSame('the tree', (string) $row['WoTranslation']); + } + + public function testEmptyTermFieldsAreSkippedNotStored(): void + { + (new ForeignDeckBuilder()) + ->addMatureNote(['', 'orphan translation'], 30) + ->addMatureNote(['gut', 'good'], 30) + ->build($this->deckPath); + + $result = AnkiDeckImportService::default()->import($this->deckPath, $this->settings()); + + $this->assertSame(1, $result->created); + $this->assertSame(1, $result->skippedEmpty); + } + + public function testFixedStatusAppliesToEveryTerm(): void + { + (new ForeignDeckBuilder()) + ->addMatureNote(['eins', 'one'], 200) + ->addNote(['zwei', 'two']) + ->build($this->deckPath); + + $settings = new DeckImportSettings( + notetypeId: 1500000000000, + termField: 'Front', + translationField: 'Back', + languageId: self::$languageId, + deriveStatus: false, + fixedStatus: 99, + ); + + AnkiDeckImportService::default()->import($this->deckPath, $settings); + + $statuses = Connection::preparedFetchAll( + 'SELECT DISTINCT WoStatus FROM words WHERE WoLgID = ?', + [self::$languageId] + ); + + $this->assertCount(1, $statuses); + $this->assertSame(99, (int) $statuses[0]['WoStatus']); + } + + public function testTagsAreImportedWhenRequested(): void + { + (new ForeignDeckBuilder()) + ->addMatureNote(['getaggt', 'tagged'], 40, 'noun core1000') + ->build($this->deckPath); + + AnkiDeckImportService::default()->import($this->deckPath, $this->settings()); + + $wordId = (int) Connection::preparedFetchValue( + 'SELECT WoID AS value FROM words WHERE WoLgID = ? AND WoTextLC = ?', + [self::$languageId, 'getaggt'] + ); + + $tagCount = (int) Connection::preparedFetchValue( + 'SELECT COUNT(*) AS value FROM word_tag_map WHERE WtWoID = ?', + [$wordId] + ); + + $this->assertSame(2, $tagCount); + } + + public function testTranslationIsOptional(): void + { + (new ForeignDeckBuilder()) + ->addMatureNote(['nurwort', 'ignored'], 40) + ->build($this->deckPath); + + $result = AnkiDeckImportService::default() + ->import($this->deckPath, $this->settings(translation: null)); + + $this->assertSame(1, $result->created); + + $row = Connection::preparedFetchOne( + 'SELECT WoTranslation FROM words WHERE WoLgID = ?', + [self::$languageId] + ); + $this->assertNotNull($row); + // WordCrudService normalises an empty translation to the '*' placeholder. + $this->assertContains((string) $row['WoTranslation'], ['', '*']); + } +} diff --git a/tests/backend/Modules/Vocabulary/Application/Services/Anki/DeckImportSettingsTest.php b/tests/backend/Modules/Vocabulary/Application/Services/Anki/DeckImportSettingsTest.php new file mode 100644 index 000000000..df97df363 --- /dev/null +++ b/tests/backend/Modules/Vocabulary/Application/Services/Anki/DeckImportSettingsTest.php @@ -0,0 +1,122 @@ + 'w', 'Back' => 't'], + tags: [], + interval: $interval, + suspended: $suspended, + isNew: $isNew, + ); + } + + public function testMatureCardsBecomeWellKnown(): void + { + $settings = $this->settings(); + + $this->assertSame(99, $settings->statusFor($this->note(21)), 'exactly at the threshold'); + $this->assertSame(99, $settings->statusFor($this->note(365))); + } + + public function testYoungCardsScaleWithInterval(): void + { + $settings = $this->settings(); + + $this->assertSame(2, $settings->statusFor($this->note(1))); + $this->assertSame(2, $settings->statusFor($this->note(6))); + $this->assertSame(3, $settings->statusFor($this->note(7))); + $this->assertSame(3, $settings->statusFor($this->note(13))); + $this->assertSame(4, $settings->statusFor($this->note(14))); + $this->assertSame(4, $settings->statusFor($this->note(20))); + } + + public function testUnstudiedCardsStartAtTheLowestStatus(): void + { + $settings = $this->settings(); + + $this->assertSame(1, $settings->statusFor($this->note(0, isNew: true))); + $this->assertSame(1, $settings->statusFor($this->note(0))); + } + + public function testSuspendedCardsBecomeIgnored(): void + { + $settings = $this->settings(); + + // Suspension wins over a long interval: the user parked it deliberately. + $this->assertSame(98, $settings->statusFor($this->note(200, suspended: true))); + } + + public function testFixedStatusOverridesTheDerivation(): void + { + $settings = $this->settings(derive: false, fixed: 99); + + $this->assertSame(99, $settings->statusFor($this->note(0, isNew: true))); + $this->assertSame(99, $settings->statusFor($this->note(200, suspended: true))); + } + + public function testDerivedStatusIsAlwaysAValidTermStatus(): void + { + $settings = $this->settings(); + $valid = [1, 2, 3, 4, 5, 98, 99]; + + foreach ([0, 1, 5, 7, 13, 14, 20, 21, 100, 10000] as $interval) { + foreach ([true, false] as $suspended) { + foreach ([true, false] as $isNew) { + $this->assertContains( + $settings->statusFor($this->note($interval, $suspended, $isNew)), + $valid, + "interval={$interval}" + ); + } + } + } + } + + public function testRejectsMissingTermField(): void + { + $this->expectException(InvalidArgumentException::class); + new DeckImportSettings(1, '', 'Back', 3); + } + + public function testRejectsMissingLanguage(): void + { + $this->expectException(InvalidArgumentException::class); + new DeckImportSettings(1, 'Front', 'Back', 0); + } + + public function testRejectsInvalidFixedStatus(): void + { + $this->expectException(InvalidArgumentException::class); + new DeckImportSettings(1, 'Front', 'Back', 3, false, 42); + } +} diff --git a/tests/backend/Modules/Vocabulary/Infrastructure/Anki/AnkiFieldTextTest.php b/tests/backend/Modules/Vocabulary/Infrastructure/Anki/AnkiFieldTextTest.php new file mode 100644 index 000000000..799d0c26b --- /dev/null +++ b/tests/backend/Modules/Vocabulary/Infrastructure/Anki/AnkiFieldTextTest.php @@ -0,0 +1,61 @@ + + */ + public static function fieldProvider(): array + { + return [ + 'plain text is untouched' => ['der Hund', 'der Hund'], + 'strips bold' => ['der Hund', 'der Hund'], + 'strips styled spans' => ['Katze', 'Katze'], + 'strips div wrappers' => ['
    das Pferd
    ', 'das Pferd'], + 'br becomes a space' => ['eins
    zwei', 'eins zwei'], + 'self-closing br becomes a space' => ['eins
    zwei', 'eins zwei'], + 'div boundaries separate words' => ['
    eins
    zwei
    ', 'eins zwei'], + 'removes sound references' => ['neko [sound:neko.mp3]', 'neko'], + 'removes sound-only content' => ['[sound:a.mp3]', ''], + 'keeps cloze answers' => ['{{c1::Hund}}', 'Hund'], + 'keeps cloze answers with hints' => ['{{c1::Hund::animal}}', 'Hund'], + 'decodes entities' => ['café', 'café'], + 'decodes ampersands' => ['this & that', 'this & that'], + 'non-breaking space becomes a space' => ["a b", 'a b'], + 'collapses whitespace' => ["a \n\t b", 'a b'], + 'trims' => [' Hund ', 'Hund'], + 'empty stays empty' => ['', ''], + 'markup-only becomes empty' => ['

    ', ''], + 'combined real-world field' => [ + '
    der Hund
    [sound:hund.mp3]
    ', + 'der Hund', + ], + ]; + } + + #[DataProvider('fieldProvider')] + public function testToPlainText(string $input, string $expected): void + { + $this->assertSame($expected, AnkiFieldText::toPlainText($input)); + } + + public function testNonLatinScriptsSurvive(): void + { + $this->assertSame('日本語', AnkiFieldText::toPlainText('
    日本語
    ')); + $this->assertSame('привет', AnkiFieldText::toPlainText('привет')); + $this->assertSame('مرحبا', AnkiFieldText::toPlainText('مرحبا')); + } +} diff --git a/tests/backend/Modules/Vocabulary/Infrastructure/Anki/ForeignApkgReaderTest.php b/tests/backend/Modules/Vocabulary/Infrastructure/Anki/ForeignApkgReaderTest.php new file mode 100644 index 000000000..06658e799 --- /dev/null +++ b/tests/backend/Modules/Vocabulary/Infrastructure/Anki/ForeignApkgReaderTest.php @@ -0,0 +1,136 @@ +markTestSkipped('pdo_sqlite required to read an .apkg collection'); + } + + $tmp = tempnam(sys_get_temp_dir(), 'lwt_foreign_test_'); + $this->tmpFile = $tmp !== false ? $tmp . '.apkg' : ''; + } + + protected function tearDown(): void + { + if ($this->tmpFile !== '') { + ForeignDeckBuilder::cleanup($this->tmpFile); + } + } + + public function testReportsNotetypesWithTheirFieldNames(): void + { + (new ForeignDeckBuilder('Basic', ['Front', 'Back'])) + ->addMatureNote(['der Hund', 'the dog']) + ->addMatureNote(['die Katze', 'the cat']) + ->build($this->tmpFile); + + $notetypes = (new ForeignApkgReader())->notetypes($this->tmpFile); + + $this->assertCount(1, $notetypes); + $this->assertSame('Basic', $notetypes[0]->name); + $this->assertSame(['Front', 'Back'], $notetypes[0]->fields); + $this->assertSame(2, $notetypes[0]->noteCount); + } + + public function testHandlesArbitraryFieldNames(): void + { + // A real shared deck: nothing here is called Front/Back or Term. + (new ForeignDeckBuilder('Japanese Core', ['Expression', 'Reading', 'Meaning'])) + ->addMatureNote(['日本語', 'にほんご', 'Japanese language']) + ->build($this->tmpFile); + + $reader = new ForeignApkgReader(); + $notetypes = $reader->notetypes($this->tmpFile); + + $this->assertSame(['Expression', 'Reading', 'Meaning'], $notetypes[0]->fields); + + $notes = $reader->notes($this->tmpFile, $notetypes[0]->id); + + $this->assertCount(1, $notes); + $this->assertSame('日本語', $notes[0]->field('Expression')); + $this->assertSame('にほんご', $notes[0]->field('Reading')); + $this->assertSame('Japanese language', $notes[0]->field('Meaning')); + } + + public function testExposesSchedulingStatePerNote(): void + { + (new ForeignDeckBuilder()) + ->addMatureNote(['mature', 'x'], 120) + ->addYoungNote(['young', 'x'], 5) + ->addNote(['brandnew', 'x']) + ->addSuspendedNote(['parked', 'x']) + ->build($this->tmpFile); + + $reader = new ForeignApkgReader(); + $notes = $reader->notes($this->tmpFile, $reader->notetypes($this->tmpFile)[0]->id); + + $byTerm = []; + foreach ($notes as $note) { + $byTerm[$note->field('Front')] = $note; + } + + $this->assertSame(120, $byTerm['mature']->interval); + $this->assertFalse($byTerm['mature']->isNew); + + $this->assertSame(5, $byTerm['young']->interval); + + $this->assertTrue($byTerm['brandnew']->isNew); + $this->assertSame(0, $byTerm['brandnew']->interval); + + $this->assertTrue($byTerm['parked']->suspended); + } + + public function testNegativeIntervalsAreTreatedAsUnlearned(): void + { + // Anki stores sub-day learning steps as negative seconds in `ivl`. + (new ForeignDeckBuilder()) + ->addNote(['subday', 'x'], '', -600, 1, 1) + ->build($this->tmpFile); + + $reader = new ForeignApkgReader(); + $notes = $reader->notes($this->tmpFile, $reader->notetypes($this->tmpFile)[0]->id); + + $this->assertSame(0, $notes[0]->interval, 'a negative ivl must not become a huge interval'); + } + + public function testReadsAnkiTags(): void + { + (new ForeignDeckBuilder()) + ->addMatureNote(['tagged', 'x'], 30, ' noun german ') + ->build($this->tmpFile); + + $reader = new ForeignApkgReader(); + $notes = $reader->notes($this->tmpFile, $reader->notetypes($this->tmpFile)[0]->id); + + $this->assertSame(['noun', 'german'], $notes[0]->tags); + } + + public function testUnknownNotetypeIsRejected(): void + { + (new ForeignDeckBuilder())->addMatureNote(['x', 'y'])->build($this->tmpFile); + + $this->expectException(RuntimeException::class); + (new ForeignApkgReader())->notes($this->tmpFile, 99999); + } + + public function testMissingFileIsRejected(): void + { + $this->expectException(RuntimeException::class); + (new ForeignApkgReader())->notetypes('/nonexistent/deck.apkg'); + } +} diff --git a/tests/backend/Modules/Vocabulary/Infrastructure/Anki/ForeignDeckBuilder.php b/tests/backend/Modules/Vocabulary/Infrastructure/Anki/ForeignDeckBuilder.php new file mode 100644 index 000000000..1dc78d944 --- /dev/null +++ b/tests/backend/Modules/Vocabulary/Infrastructure/Anki/ForeignDeckBuilder.php @@ -0,0 +1,175 @@ +, tags: string, ivl: int, queue: int, type: int}> */ + private array $notes = []; + + /** + * @param list $fieldNames + */ + public function __construct( + private readonly string $notetypeName = 'Basic', + private readonly array $fieldNames = ['Front', 'Back'], + private readonly int $notetypeId = 1500000000000, + ) { + } + + /** + * @param list $fields Values in field order + */ + public function addNote( + array $fields, + string $tags = '', + int $interval = 0, + int $queue = 0, + int $type = 0 + ): self { + $this->notes[] = [ + 'fields' => $fields, + 'tags' => $tags, + 'ivl' => $interval, + 'queue' => $queue, + 'type' => $type, + ]; + + return $this; + } + + /** + * A note Anki considers mature (interval past the 21-day threshold). + * + * @param list $fields + */ + public function addMatureNote(array $fields, int $interval = 90, string $tags = ''): self + { + return $this->addNote($fields, $tags, $interval, 2, 2); + } + + /** + * A note in active learning. + * + * @param list $fields + */ + public function addYoungNote(array $fields, int $interval, string $tags = ''): self + { + return $this->addNote($fields, $tags, $interval, 2, 2); + } + + /** + * A note the user suspended in Anki. + * + * @param list $fields + */ + public function addSuspendedNote(array $fields, int $interval = 30): self + { + return $this->addNote($fields, '', $interval, -1, 2); + } + + /** + * Write the .apkg and return its path. + */ + public function build(string $path): string + { + $dbPath = $path . '.collection'; + @unlink($dbPath); + + $pdo = new PDO('sqlite:' . $dbPath); + $pdo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); + foreach (AnkiSchema::createStatements() as $stmt) { + $pdo->exec($stmt); + } + + $flds = []; + foreach ($this->fieldNames as $ord => $name) { + $flds[] = [ + 'name' => $name, 'ord' => $ord, 'sticky' => false, 'rtl' => false, + 'font' => 'Arial', 'size' => 20, 'media' => [], + ]; + } + + $models = [ + (string) $this->notetypeId => [ + 'id' => $this->notetypeId, + 'name' => $this->notetypeName, + 'type' => 0, 'mod' => 1700000000, 'usn' => -1, 'sortf' => 0, 'did' => 1, + 'tmpls' => [[ + 'name' => 'Card 1', 'ord' => 0, + 'qfmt' => '{{' . $this->fieldNames[0] . '}}', + 'afmt' => '{{FrontSide}}', 'did' => null, 'bqfmt' => '', 'bafmt' => '', + ]], + 'flds' => $flds, + 'css' => '', 'latexPre' => '', 'latexPost' => '', 'req' => [[0, 'any', [0]]], + ], + ]; + + $pdo->exec( + 'INSERT INTO col (id, crt, mod, scm, ver, dty, usn, ls, conf, models, decks, dconf, tags) VALUES (' + . '1, 1700000000, 1700000000000, 1700000000000, 11, 0, 0, 0, ' + . $pdo->quote(json_encode(AnkiSchema::defaultConf())) . ', ' + . $pdo->quote((string) json_encode($models)) . ', ' + . $pdo->quote((string) json_encode(['1' => AnkiSchema::defaultDeck()])) . ', ' + . $pdo->quote((string) json_encode(['1' => AnkiSchema::defaultDeckConfig()])) . ", '{}')" + ); + + $nextId = 1700000000000; + foreach ($this->notes as $note) { + $nid = $nextId++; + $cid = $nextId++; + // An Anki-style guid: random, and crucially without LWT's prefix. + $guid = rtrim(strtr(base64_encode(pack('N', $nid % 4294967295)), '+/', '-_'), '='); + $joined = implode(AnkiSchema::FIELD_SEPARATOR, $note['fields']); + $sortField = $note['fields'][0] ?? ''; + + $pdo->exec( + 'INSERT INTO notes (id, guid, mid, mod, usn, tags, flds, sfld, csum, flags, data) VALUES (' + . $nid . ', ' . $pdo->quote($guid) . ', ' . $this->notetypeId . ', 1700000000, -1, ' + . $pdo->quote($note['tags']) . ', ' . $pdo->quote($joined) . ', ' + . $pdo->quote($sortField) . ', ' . AnkiSchema::fieldChecksum($sortField) . ", 0, '')" + ); + $pdo->exec( + 'INSERT INTO cards (id, nid, did, ord, mod, usn, type, queue, due, ivl, factor, ' + . 'reps, lapses, left, odue, odid, flags, data) VALUES (' + . $cid . ', ' . $nid . ', 1, 0, 1700000000, -1, ' . $note['type'] . ', ' + . $note['queue'] . ', 1, ' . $note['ivl'] . ", 2500, 3, 0, 0, 0, 0, 0, '')" + ); + } + $pdo = null; + + @unlink($path); + $zip = new ZipArchive(); + $zip->open($path, ZipArchive::CREATE); + $zip->addFile($dbPath, 'collection.anki21'); + $zip->addFromString('media', '{}'); + $zip->close(); + + // The zip holds its own copy; the loose collection file is scratch. + // It is removed by the caller's tearDown along with $path. + return $path; + } + + /** + * Remove the scratch collection file left beside an built .apkg. + */ + public static function cleanup(string $path): void + { + @unlink($path); + @unlink($path . '.collection'); + } +}