diff --git a/CHANGELOG.md b/CHANGELOG.md index a354ea259..967257521 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,22 +9,20 @@ ones are marked like "v1.0.0-fork". ### Added +* **FSRS scheduling groundwork** (#238, phase 2a): LWT now records FSRS-6 memory + state per term — stability, difficulty, due date and a review history. + Nothing user-visible changes yet: the legacy scoring still drives the review + queue and reading colours are untouched. Existing terms seed lazily from their + status, so upgrading neither floods the queue nor costs anything on a large + vocabulary. See `docs/developer/term-status-fsrs`. + * **Anki `.apkg` export and import** (refs #228): terms round-trip to Anki as a - real Anki package, written and read by LWT itself (no genanki, no Python) — - `ApkgWriter`/`ApkgReader` build the SQLite collection and zip container - directly. Export the whole language from the vocabulary list's **ALL** menu, - or just the rows you ticked via **Marked Terms → Export Selection**. Notes - carry a stable `lwt-` guid, so re-importing the file matches notes back - to the terms they came from and updates translation, romanization, notes and - tags in place; a card you suspended in Anki demotes a learning-status term to - *Ignored*. A subset export only ever touches the terms it contains — terms - outside the selection are left alone. Notes with no LWT guid are counted and - skipped rather than created, since LWT could not tell which language they - belong to. **SRS scheduling state is deliberately not exchanged**: LWT's - score model and Anki's are not comparable, and pretending otherwise would - silently corrupt review timing on both sides — see #238 for the FSRS work - that would make it meaningful. Reference page at - `docs/reference/anki-export-import`. + real `.apkg`, written and read by LWT itself — no genanki, no Python. Export a + whole language or just the rows you ticked, from the vocabulary list. Notes + carry a stable guid, so re-importing updates the terms they came from + (translation, romanization, notes, tags) and a card suspended in Anki demotes + a learning term to *Ignored*. Scheduling state is deliberately not exchanged. + See `docs/reference/anki-export-import`. ### Fixed diff --git a/db/migrations/20260805_200000_add_fsrs_scheduling.sql b/db/migrations/20260805_200000_add_fsrs_scheduling.sql new file mode 100644 index 000000000..8dab8fa96 --- /dev/null +++ b/db/migrations/20260805_200000_add_fsrs_scheduling.sql @@ -0,0 +1,69 @@ +-- Add FSRS scheduling state and review history (issue #238, phase 2a). +-- +-- Phase 2a is deliberately ADDITIVE: nothing here replaces the legacy Leitner +-- scoring. `words.WoTodayScore` / `WoTomorrowScore` / `WoRandom` and the +-- SCORE_FORMULA_* SQL keep being written exactly as before, and `WoStatus` +-- remains the manual, authoritative source for reading-view colours. These two +-- tables accumulate alongside them so the two schedulers can be compared on +-- real data before anything is retired (that is phase 2b). +-- +-- Neither table carries a user column: both are keyed by WoID, and `words` +-- already carries WoUsID with an FK to users. Ownership is therefore reached by +-- joining `words`, which is what the repository does — see +-- src/Shared/Infrastructure/Database/UserScopedQuery.php for the tables that do +-- get an automatic scope column. + +-- Per-term FSRS memory state. One row per reviewed term; rows are created +-- lazily on a term's first graded review (seeded from WoStatus/WoStatusChanged) +-- rather than backfilled, so installs with large vocabularies pay nothing here. +-- NB: the WoID-referencing columns below are `int(10) unsigned`, NOT the +-- `mediumint(8) unsigned` that db/schema/baseline.sql still declares. The +-- inter-table FK migration (20251221_120000) widens words.WoID to INT UNSIGNED, +-- and MySQL rejects a foreign key whose column type differs from its parent +-- with errno 150 ("Foreign key constraint is incorrectly formed"). Match the +-- post-migration type, not the baseline's. +CREATE TABLE IF NOT EXISTS term_schedule ( + TsWoID int(10) unsigned NOT NULL, + -- Stability: days for retrievability to decay to 0.9. FSRS clamps to >= 0.001. + TsStability double NOT NULL, + -- Difficulty: FSRS clamps to [1, 10]. + TsDifficulty double NOT NULL, + TsDue datetime NOT NULL, + TsLastReview datetime DEFAULT NULL, + TsReps int(10) unsigned NOT NULL DEFAULT 0, + TsLapses int(10) unsigned NOT NULL DEFAULT 0, + -- 0 = new, 1 = learning, 2 = review, 3 = relearning (matches Anki's card states, + -- so the .apkg exporter can populate cards.type/queue directly later). + TsState tinyint(3) unsigned NOT NULL DEFAULT 0, + PRIMARY KEY (TsWoID), + KEY TsDue (TsDue), + KEY TsState (TsState), + CONSTRAINT fk_term_schedule_word FOREIGN KEY (TsWoID) + REFERENCES words (WoID) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- Append-only review history. FSRS can schedule from current state alone, so +-- this is not read by the scheduler; it exists because per-user parameter +-- optimisation (Anki's "FSRS optimizer") needs history, and because it maps +-- onto Anki's `revlog` table for .apkg round-trip (issue #228). +CREATE TABLE IF NOT EXISTS review_log ( + RlID int(10) unsigned NOT NULL AUTO_INCREMENT, + RlWoID int(10) unsigned NOT NULL, + -- 1 = Again, 2 = Hard, 3 = Good, 4 = Easy. + RlGrade tinyint(3) unsigned NOT NULL, + -- Scheduling state BEFORE this review, so a re-optimiser can replay history. + RlState tinyint(3) unsigned NOT NULL, + -- Memory state AFTER this review. + RlStability double NOT NULL, + RlDifficulty double NOT NULL, + -- Whole days since the previous review (0 for a first or same-day review). + RlElapsedDays int(11) NOT NULL, + -- Interval in days the scheduler assigned as a result of this review. + RlScheduledDays int(11) NOT NULL, + RlReviewedAt datetime NOT NULL, + PRIMARY KEY (RlID), + KEY RlWoID (RlWoID, RlReviewedAt), + KEY RlReviewedAt (RlReviewedAt), + CONSTRAINT fk_review_log_word FOREIGN KEY (RlWoID) + REFERENCES words (WoID) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/docs-src/developer/term-status-fsrs.md b/docs-src/developer/term-status-fsrs.md index 05c13f808..a18706c31 100644 --- a/docs-src/developer/term-status-fsrs.md +++ b/docs-src/developer/term-status-fsrs.md @@ -5,14 +5,14 @@ description: Centralize the scattered word-status model into a single source of # Proposal: Term Status Model + FSRS Scheduling -**Status:** **Phase 1 implemented** (#238). Phase 2 (FSRS scheduling) remains -proposed — deferred, and gated on the product decisions in -[Trade-offs & open questions](#trade-offs-open-questions). The FSRS part is an -architectural change worth landing on its own. +**Status:** **Phase 1 and phase 2a implemented** (#238). Phase 2b (retiring the +legacy Leitner scoring and switching the review queue over) remains proposed. Tracked in [issue #238](https://github.com/HugoFara/lwt/issues/238). -Phase 1 (status as a single source of truth) is shipped; the rest is a design -proposal, not shipped work. +The three open product questions have been decided — see +[Decisions taken](#decisions-taken). Phase 2 was split in two so the +irreversible part (dropping the old score columns) is a separate, later step +from the additive part (accumulating FSRS state). ## Problem @@ -135,38 +135,163 @@ map each status to a starting `S` (reuse the current per-status intervals as the seed), set a default `D`, and `last_review = WoStatusChanged`. No review history is lost because there is none today; the `review_log` starts accumulating from rollout. +## Decisions taken + +The three questions this proposal was gated on, and how they were resolved. + +### 1. Display status stays **manual**, not derived from stability + +The original recommendation was to derive the 1–5 reading colours from FSRS +stability. That was rejected: it imports an Anki assumption that does not hold +here. **In Anki every card is reviewed; in LWT review is optional and reading is +the primary loop.** Many users never open the review page — they read, click +words, and set status by hand. + +Deriving status from stability would give those users reading colours that drift +on their own, driven by a scheduler they do not use: a word deliberately marked +5 decays to 3 because it was never reviewed. That is a regression in the core +experience. + +So `WoStatus` remains manual and authoritative for display, and FSRS state is +purely additive. This also removes the migration's biggest risk for free — +"reading-view colours are unchanged after upgrading" is true by construction +rather than something to verify. A derived-status mode can be added later as an +opt-in setting. + +### 2. **Four grades**, no 2-button mode + +Hard and Easy are where FSRS gets the information that makes it beat SM-2; a +2-button mode degrades it to roughly the Leitner behaviour we already have. +The legacy binary answer maps to Again/Good (`Rating::fromBinary()`), so old +callers keep working without a separate mode to maintain. + +### 3. **Hand-port**, not a vendored dependency + +Neither PHP option was viable: + +- [`fsrs-rs-php`](https://github.com/open-spaced-repetition/fsrs-rs-php), the + official binding, requires compiling a Rust PHP extension — manual `.so` + copy, `php.ini` edit, no Composer install. Disqualifying for a self-hosting + audience; LWT already had to fix Windows CI for a *bundled* extension + (`pdo_sqlite`, #259). +- [`scottlaurent/fsrs`](https://packagist.org/packages/scottlaurent/fsrs) is + pure PHP and MIT, but v0.1 / one commit / no validation against reference + vectors, and declares PHP 8.1–8.3 against LWT's 8.2–8.5. + +`Fsrs6Scheduler` is therefore a hand-port of +[py-fsrs](https://github.com/open-spaced-repetition/py-fsrs) **v6.3.1**, +validated against vectors generated from that exact release (see +[Verification](#verification)). LWT is public-domain under the Unlicense; that +one file carries the reference implementation's MIT notice, as the licence +requires. + +::: warning Pin the reference version +py-fsrs's unreleased `main` widens the short-term stability clamp from +`(Good, Easy)` to `(Hard, Good, Easy)`. That materially changes same-day +repeats — a same-day Hard becomes a no-op instead of a ~44% stability cut. The +port follows released **v6.3.1**. Retarget it only together with regenerated +fixtures. +::: + +## Phase 2a — additive FSRS state ✅ implemented + +Everything below accumulates FSRS data on real reviews **without changing what +users see**. The legacy scoring keeps running untouched, so the two models can +be compared on real data before anything is retired. + +- **Two new tables** (`db/migrations/20260805_200000_add_fsrs_scheduling.sql`): + `term_schedule` (stability, difficulty, due, last review, reps, lapses, state) + and the append-only `review_log`. `TsState` values match Anki's `cards.type` + so the `.apkg` exporter can write them straight through later. +- **`Fsrs6Scheduler`** behind `SchedulerInterface`, with `FsrsParameters` + (21 weights, target retention, maximum interval) — swappable by design. +- **Lazy seeding.** `LegacyStatusSeed` maps each legacy status to the stability + that reproduces its Leitner interval (1/2/9/27/71 days for statuses 1–5), with + `lastReview = WoStatusChanged`. Rows appear on a term's first graded review + rather than via a bulk backfill, so a 100k-term vocabulary costs nothing at + upgrade time and nobody's queue floods. 98/99 are never scheduled. +- **`RecordScheduledReview`** is called from `SubmitAnswer` as a *shadow write*: + it runs only after the legacy update succeeds, and swallows storage errors so + a scheduling failure can never break the review the user just submitted. +- **`SubmitAnswer::executeWithGrade()`** is the FSRS-native entry point; the + grade also drives the legacy ±1 status nudge, so reading behaviour is + identical whichever endpoint is used. + +**Not in 2a:** nothing reads `term_schedule` to choose review words yet, there +is no 4-grade UI, and interval fuzzing and the parameter optimiser are omitted +(fuzzing only exists to spread Anki's daily load; the optimiser needs +accumulated history). + +## Phase 2b — retire the legacy scoring (proposed) + +Once 2a has accumulated real data: switch the review queue to order by `TsDue`, +build the 4-grade UI, drop `SCORE_FORMULA_*` / `WoTodayScore` / +`WoTomorrowScore` / `WoRandom` and their 16 call sites, and populate +`cards.data` + `revlog` in the `.apkg` exporter so scheduling round-trips to +Anki (#228). + ## Trade-offs & open questions -- **Display status: derived vs. manual.** Deriving 1–5 from `S` is the principled - end state but changes how colours move (they now track scheduling). Alternative: - keep status fully manual and orthogonal to FSRS. *Recommended: derive, with manual - override.* — needs your call. -- **4-grade UX.** A real behaviour change for users used to "I knew it / I didn't." - Could ship a 2-button mode that maps to Again/Good. +The three gating questions are resolved in [Decisions taken](#decisions-taken). +What remains open, all deferred to phase 2b or later: + - **Per-user vs. global parameters.** FSRS ships sensible defaults; per-user - optimisation needs enough `review_log` history and an optimiser job (defer). -- **Scope.** Phase 2 touches schema, the Review module, the review UI, and stats. - Phase 1 is independent and should land first. -- **Licensing.** Confirm the chosen FSRS implementation's licence is compatible - before vendoring. + optimisation needs enough `review_log` history and an optimiser job. 2a starts + accumulating that history — revisit once real data exists. +- **Interval fuzzing.** Omitted deliberately. It exists to spread Anki's daily + review load; whether LWT wants it depends on how the 2b queue behaves. +- **Seed quality.** The status→stability mapping reproduces the *legacy* + intervals, which were hand-tuned rather than fitted. Once `review_log` has + data, check whether seeded terms behave sensibly or whether the mapping should + be re-derived. +- **Scope of 2b.** Retiring the score columns touches 16 PHP files, the review + UI, and stats. Sequence it after 2a has proven itself. ## Scope sketch (when picked up) - **Phase 1:** `TermStatus` VO (expand), `TermStatusService` + `StatusHelper` (fold in), ~11 PHP call sites (adopt VO), status-definitions API + bootstrap, ~6 TS files → `shared/stores/statuses.ts`. -- **Phase 2:** migration (schedule columns / `term_schedule` + `review_log`), - `Scheduler` interface + FSRS implementation, `Review/Application/UseCases/SubmitAnswer` - (call scheduler), review UI (4-grade), stats that read `WoTodayScore` → read `due`, - removal of `SCORE_FORMULA_*` and `WoRandom`. - -## Verification (at implementation time) - -1. Unit-test the FSRS `Scheduler` against the reference implementation's known - vectors (same `S`/`D`/grade in → same interval out). -2. PHP + frontend gates (`phpcs`, `psalm`, `composer test:no-coverage`, `typecheck`, - `lint`, `test`, `build:all`). -3. Migration round-trip on a seeded DB: every pre-existing term gets valid FSRS state; - reading-view colours are stable immediately after migration. -4. E2E: run a review session, grade across all 4 buttons, confirm due dates advance - sensibly and the reading view reflects status changes. +- **Phase 2a (done):** migration (`term_schedule` + `review_log`), + `SchedulerInterface` + `Fsrs6Scheduler` + `FsrsParameters`, `LegacyStatusSeed`, + `MySqlTermScheduleRepository`, `RecordScheduledReview`, and the shadow-write + hook in `SubmitAnswer`. +- **Phase 2b:** review UI (4-grade), review queue ordered by `TsDue`, stats that + read `WoTodayScore` → read `TsDue`, removal of `SCORE_FORMULA_*` / + `WoTodayScore` / `WoTomorrowScore` / `WoRandom` across their 16 call sites, + and `cards.data` + `revlog` in the `.apkg` exporter (#228). + +## Verification + +**Done for phase 2a:** + +1. **Reference vectors.** `Fsrs6SchedulerTest` replays 7 review sequences (24 + graded reviews) generated from py-fsrs v6.3.1, asserting stability, + difficulty, retrievability and interval at every step to 1e-9. The fixture + and its generator live in + `tests/backend/Modules/Review/Domain/Scheduling/fixtures/`. Regenerate with: + + ```bash + python3 -m venv .venv + ./.venv/bin/pip install 'fsrs==6.3.1' + ./.venv/bin/python generate_reference_vectors.py > fsrs6_reference_vectors.json + ``` + + These vectors already caught one real porting bug (the short-term clamp + described above), which is exactly what they are for. + +2. **Property tests** alongside them: difficulty saturates within [1, 10] under + 40 consecutive Again and 40 consecutive Easy; a lapse never increases + stability; Hard < Good < Easy intervals from identical state; retrievability + is exactly 0.9 after one stability period; a stricter retention target + schedules sooner. + +3. **Integration** (`TermScheduleRepositoryIntegrationTest`, real MySQL): lazy + seeding from each status, state upsert vs. append-only log, lapse counting, + due counting, and null for unowned/missing terms. + +4. **Gates:** psalm 0 errors, phpcs PSR12 clean, full PHPUnit suite green. + +**Still to do for phase 2b:** E2E through a real review session across all four +grades, and confirmation that reading-view colours are untouched on a populated +upgrade (true by construction under decision 1, but worth seeing). diff --git a/src/Modules/Review/Application/UseCases/RecordScheduledReview.php b/src/Modules/Review/Application/UseCases/RecordScheduledReview.php new file mode 100644 index 000000000..c3fda1acf --- /dev/null +++ b/src/Modules/Review/Application/UseCases/RecordScheduledReview.php @@ -0,0 +1,88 @@ +scheduler = $scheduler ?? new Fsrs6Scheduler(); + $this->repository = $repository ?? new MySqlTermScheduleRepository(); + } + + /** + * Apply a grade and persist the resulting state plus its log entry. + * + * @param int $wordId Term being reviewed + * @param Rating $rating Grade given + * @param DateTimeImmutable|null $now Review time (defaults to now) + * + * @return bool Whether scheduling state was recorded + */ + public function execute(int $wordId, Rating $rating, ?DateTimeImmutable $now = null): bool + { + $now ??= new DateTimeImmutable(); + + try { + $current = $this->repository->findOrSeed($wordId); + + $result = $this->scheduler->review($current, $rating, $now); + + $this->repository->saveReview( + $wordId, + $result, + $rating, + $current?->state->value ?? 0 + ); + + return true; + } catch (Throwable) { + // Shadow write — never fail the user's review because the + // scheduling side-table could not be updated. + return false; + } + } + + /** + * Compute the scheduling outcome without persisting it. + * + * Used by the API to preview what each grade would do (the "1d / 3d / 10d" + * hints under the review buttons). + */ + public function preview(int $wordId, Rating $rating, ?DateTimeImmutable $now = null): ?SchedulingResult + { + $now ??= new DateTimeImmutable(); + + $current = $this->repository->findOrSeed($wordId); + + return $this->scheduler->review($current, $rating, $now); + } +} diff --git a/src/Modules/Review/Application/UseCases/SubmitAnswer.php b/src/Modules/Review/Application/UseCases/SubmitAnswer.php index d5b05a422..f60685a22 100644 --- a/src/Modules/Review/Application/UseCases/SubmitAnswer.php +++ b/src/Modules/Review/Application/UseCases/SubmitAnswer.php @@ -19,6 +19,7 @@ use Lwt\Modules\Review\Domain\ReviewRepositoryInterface; use Lwt\Modules\Review\Domain\ReviewSession; +use Lwt\Modules\Review\Domain\Scheduling\Rating; use Lwt\Modules\Review\Infrastructure\SessionStateManager; use Lwt\Modules\Vocabulary\Domain\ValueObject\TermStatus; @@ -33,6 +34,7 @@ class SubmitAnswer { private ReviewRepositoryInterface $repository; private SessionStateManager $sessionManager; + private RecordScheduledReview $scheduledReview; /** * Constructor. @@ -42,10 +44,32 @@ class SubmitAnswer */ public function __construct( ReviewRepositoryInterface $repository, - ?SessionStateManager $sessionManager = null + ?SessionStateManager $sessionManager = null, + ?RecordScheduledReview $scheduledReview = null ) { $this->repository = $repository; $this->sessionManager = $sessionManager ?? new SessionStateManager(); + $this->scheduledReview = $scheduledReview ?? new RecordScheduledReview(); + } + + /** + * Submit a graded answer (issue #238, phase 2a). + * + * This is the FSRS-native entry point: the grade drives the scheduler, and + * is *also* mapped onto the legacy ±1 status nudge so the reading view + * behaves exactly as it does for a binary answer. Again lowers the status, + * every other grade raises it. + * + * @param int $wordId Word ID + * @param Rating $grade Again / Hard / Good / Easy + * + * @return array Same shape as execute(), plus a 'scheduled' flag + */ + public function executeWithGrade(int $wordId, Rating $grade): array + { + $result = $this->executeWithChange($wordId, $grade->legacyStatusChange(), $grade); + + return $result; } /** @@ -125,7 +149,7 @@ public function execute(int $wordId, int $newStatus): array * * @return array Same as execute() */ - public function executeWithChange(int $wordId, int $change): array + public function executeWithChange(int $wordId, int $change, ?Rating $grade = null): array { // Get current status $currentStatus = $this->repository->getWordStatus($wordId); @@ -145,7 +169,20 @@ public function executeWithChange(int $wordId, int $change): array // Calculate new status $newStatus = $this->calculateNewStatus($currentStatus, $change); - return $this->execute($wordId, $newStatus); + $result = $this->execute($wordId, $newStatus); + + // Shadow-write FSRS state (issue #238, phase 2a). A binary answer has + // no Hard/Easy signal, so it maps to Again/Good; the graded endpoint + // passes the real grade through. Only recorded once the legacy update + // actually succeeded, so the two models cannot diverge. + if ($result['success'] === true) { + $result['scheduled'] = $this->scheduledReview->execute( + $wordId, + $grade ?? Rating::fromBinary($change >= 0) + ); + } + + return $result; } /** diff --git a/src/Modules/Review/Domain/Scheduling/Fsrs6Scheduler.php b/src/Modules/Review/Domain/Scheduling/Fsrs6Scheduler.php new file mode 100644 index 000000000..28950c48f --- /dev/null +++ b/src/Modules/Review/Domain/Scheduling/Fsrs6Scheduler.php @@ -0,0 +1,266 @@ +params = $params ?? new FsrsParameters(); + $this->decay = $this->params->decay(); + $this->factor = $this->params->factor(); + } + + public function review(?MemoryState $current, Rating $rating, DateTimeImmutable $now): SchedulingResult + { + $elapsedDays = $current?->elapsedDays($now) ?? 0; + + if ($current === null) { + // First ever review: seed S and D from the grade alone. + $stability = $this->initialStability($rating); + $difficulty = $this->initialDifficulty($rating, true); + } elseif ($elapsedDays < 1) { + // Same-day repeat — the long-term formula assumes measurable decay + // has happened, so FSRS uses a separate short-term update. + $stability = $this->shortTermStability($current->stability, $rating); + $difficulty = $this->nextDifficulty($current->difficulty, $rating); + } else { + $retrievability = $this->retrievability($current, $now); + $stability = $this->nextStability( + $current->difficulty, + $current->stability, + $retrievability, + $rating + ); + $difficulty = $this->nextDifficulty($current->difficulty, $rating); + } + + $intervalDays = $this->nextInterval($stability); + + $state = new MemoryState( + stability: $stability, + difficulty: $difficulty, + due: $now->add(new DateInterval('P' . $intervalDays . 'D')), + lastReview: $now, + reps: ($current?->reps ?? 0) + 1, + lapses: ($current?->lapses ?? 0) + ($rating === Rating::Again ? 1 : 0), + state: $rating === Rating::Again ? SchedulingState::Relearning : SchedulingState::Review, + ); + + return new SchedulingResult($state, $intervalDays, $elapsedDays); + } + + public function retrievability(?MemoryState $state, DateTimeImmutable $now): float + { + if ($state === null || $state->lastReview === null) { + return 0.0; + } + + $elapsedDays = $state->elapsedDays($now); + + return (1 + $this->factor * $elapsedDays / $state->stability) ** $this->decay; + } + + /** + * Days until retrievability decays to the target retention. + */ + private function nextInterval(float $stability): int + { + $interval = ($stability / $this->factor) + * (($this->params->desiredRetention ** (1 / $this->decay)) - 1); + + $rounded = (int) round($interval); + + return max(1, min($rounded, $this->params->maximumInterval)); + } + + /** + * S after a first review: w0..w3, indexed by grade. + */ + private function initialStability(Rating $rating): float + { + return $this->clampStability($this->params->w($rating->value - 1)); + } + + /** + * D after a first review: w4 - e^(w5 * (G-1)) + 1. + * + * Left unclamped when used as the mean-reversion target, which is what the + * reference does — clamping it there would bias the reversion. + */ + private function initialDifficulty(Rating $rating, bool $clamp): float + { + $difficulty = $this->params->w(4) + - (M_E ** ($this->params->w(5) * ($rating->value - 1))) + + 1; + + return $clamp ? $this->clampDifficulty($difficulty) : $difficulty; + } + + /** + * S for a same-day repeat. + */ + private function shortTermStability(float $stability, Rating $rating): float + { + $increase = (M_E ** ($this->params->w(17) * ($rating->value - 3 + $this->params->w(18)))) + * ($stability ** -$this->params->w(19)); + + // Only Good and Easy are floored at "no loss of stability". Hard is + // deliberately allowed to reduce stability on a same-day repeat. + // + // Note for future updates: py-fsrs's unreleased `main` widens this + // clamp to include Hard, which changes the result materially (a same- + // day Hard becomes a no-op instead of a ~44% stability cut). We follow + // the released v6.3.1 behaviour, which is what the reference vectors in + // the test fixture were generated from — regenerate them if this is + // ever retargeted at a newer release. + if ($rating === Rating::Good || $rating === Rating::Easy) { + $increase = max($increase, 1.0); + } + + return $this->clampStability($stability * $increase); + } + + /** + * D after a review: grade-driven delta, linearly damped near the ceiling, + * then reverted toward the "Easy" baseline by w7. + */ + private function nextDifficulty(float $difficulty, Rating $rating): float + { + $target = $this->initialDifficulty(Rating::Easy, false); + + $deltaDifficulty = -($this->params->w(6) * ($rating->value - 3)); + $damped = $difficulty + ((10.0 - $difficulty) * $deltaDifficulty / 9.0); + + $next = $this->params->w(7) * $target + (1 - $this->params->w(7)) * $damped; + + return $this->clampDifficulty($next); + } + + private function nextStability( + float $difficulty, + float $stability, + float $retrievability, + Rating $rating + ): float { + $next = $rating === Rating::Again + ? $this->nextForgetStability($difficulty, $stability, $retrievability) + : $this->nextRecallStability($difficulty, $stability, $retrievability, $rating); + + return $this->clampStability($next); + } + + /** + * S after a lapse. Capped by the same-day term so a lapse can never + * increase stability. + */ + private function nextForgetStability( + float $difficulty, + float $stability, + float $retrievability + ): float { + $longTerm = $this->params->w(11) + * ($difficulty ** -$this->params->w(12)) + * ((($stability + 1) ** $this->params->w(13)) - 1) + * (M_E ** ((1 - $retrievability) * $this->params->w(14))); + + $shortTerm = $stability / (M_E ** ($this->params->w(17) * $this->params->w(18))); + + return min($longTerm, $shortTerm); + } + + /** + * S after a successful recall. The lower the retrievability at review + * time, the bigger the gain — reviewing something you almost forgot is + * worth more than reviewing something fresh. + */ + private function nextRecallStability( + float $difficulty, + float $stability, + float $retrievability, + Rating $rating + ): float { + $hardPenalty = $rating === Rating::Hard ? $this->params->w(15) : 1.0; + $easyBonus = $rating === Rating::Easy ? $this->params->w(16) : 1.0; + + return $stability * ( + 1 + + (M_E ** $this->params->w(8)) + * (11 - $difficulty) + * ($stability ** -$this->params->w(9)) + * ((M_E ** ((1 - $retrievability) * $this->params->w(10))) - 1) + * $hardPenalty + * $easyBonus + ); + } + + private function clampStability(float $stability): float + { + return max($stability, FsrsParameters::STABILITY_MIN); + } + + private function clampDifficulty(float $difficulty): float + { + return min(max($difficulty, FsrsParameters::MIN_DIFFICULTY), FsrsParameters::MAX_DIFFICULTY); + } +} diff --git a/src/Modules/Review/Domain/Scheduling/FsrsParameters.php b/src/Modules/Review/Domain/Scheduling/FsrsParameters.php new file mode 100644 index 000000000..5740eb8d5 --- /dev/null +++ b/src/Modules/Review/Domain/Scheduling/FsrsParameters.php @@ -0,0 +1,99 @@ + + */ + public const DEFAULT_WEIGHTS = [ + 0.212, 1.2931, 2.3065, 8.2956, 6.4133, 0.8334, 3.0194, 0.001, + 1.8722, 0.1666, 0.796, 1.4835, 0.0614, 0.2629, 1.6483, 0.6014, + 1.8729, 0.5425, 0.0912, 0.0658, self::DEFAULT_DECAY, + ]; + + /** @var list */ + public readonly array $weights; + + /** + * @param list|null $weights 21 FSRS weights; null = defaults. + * @param float $desiredRetention Target recall probability at the + * moment a term comes due. + * @param int $maximumInterval Hard cap on scheduled days. + */ + public function __construct( + ?array $weights = null, + public readonly float $desiredRetention = 0.9, + public readonly int $maximumInterval = 36500, + ) { + $weights ??= self::DEFAULT_WEIGHTS; + + if (count($weights) !== self::PARAMETER_COUNT) { + throw new InvalidArgumentException( + 'FSRS requires exactly ' . self::PARAMETER_COUNT . ' parameters, got ' . count($weights) + ); + } + if ($desiredRetention <= 0.0 || $desiredRetention >= 1.0) { + throw new InvalidArgumentException('Desired retention must be strictly between 0 and 1'); + } + if ($maximumInterval < 1) { + throw new InvalidArgumentException('Maximum interval must be at least 1 day'); + } + + $this->weights = $weights; + } + + /** + * Weight w{$index}. + */ + public function w(int $index): float + { + return $this->weights[$index]; + } + + /** + * Curve decay, i.e. -w20. Negative by construction. + */ + public function decay(): float + { + return -$this->weights[20]; + } + + /** + * The constant that makes the forgetting curve pass through + * R = 0.9 at t = stability. + */ + public function factor(): float + { + return 0.9 ** (1 / $this->decay()) - 1; + } +} diff --git a/src/Modules/Review/Domain/Scheduling/LegacyStatusSeed.php b/src/Modules/Review/Domain/Scheduling/LegacyStatusSeed.php new file mode 100644 index 000000000..37f12dff2 --- /dev/null +++ b/src/Modules/Review/Domain/Scheduling/LegacyStatusSeed.php @@ -0,0 +1,100 @@ + due immediately + * status 2: 6.9 - 3.50d => ~2 days + * status 3: 20 - 2.30d => ~9 days + * status 4: 46.4 - 1.75d => ~27 days + * status 5: 100 - 1.40d => ~71 days + * + * Because stability is *defined* as the time for retrievability to fall to + * 0.9, and the default target retention is also 0.9, stability in days is + * numerically the legacy interval — so these double as seed stabilities. + * + * @var array + */ + private const STABILITY_BY_STATUS = [ + 1 => 1.0, + 2 => 2.0, + 3 => 9.0, + 4 => 27.0, + 5 => 71.0, + ]; + + /** + * Build the seed state for a term, or null if the status is not schedulable. + * + * 98 (ignored) and 99 (well-known) are manual flags that were never part of + * the review queue, so they get no scheduling state. + * + * @param int $status The term's WoStatus + * @param DateTimeImmutable $statusChanged When it last changed (WoStatusChanged) + */ + public static function forStatus(int $status, DateTimeImmutable $statusChanged): ?MemoryState + { + $stability = self::STABILITY_BY_STATUS[$status] ?? null; + if ($stability === null) { + return null; + } + + // Difficulty prior: what FSRS itself assigns to a card first answered + // "Good". We have no per-term evidence, and this keeps a seeded term + // on the same footing as one genuinely reviewed once. + $difficulty = self::defaultDifficulty(); + + $due = $statusChanged->add(new DateInterval('P' . (int) round($stability) . 'D')); + + return new MemoryState( + stability: $stability, + difficulty: $difficulty, + due: $due, + lastReview: $statusChanged, + reps: 0, + lapses: 0, + state: SchedulingState::Review, + ); + } + + /** + * FSRS's initial difficulty for a "Good" first answer, clamped. + */ + public static function defaultDifficulty(): float + { + $params = new FsrsParameters(); + + $difficulty = $params->w(4) + - (M_E ** ($params->w(5) * (Rating::Good->value - 1))) + + 1; + + return min( + max($difficulty, FsrsParameters::MIN_DIFFICULTY), + FsrsParameters::MAX_DIFFICULTY + ); + } +} diff --git a/src/Modules/Review/Domain/Scheduling/MemoryState.php b/src/Modules/Review/Domain/Scheduling/MemoryState.php new file mode 100644 index 000000000..df3ae2705 --- /dev/null +++ b/src/Modules/Review/Domain/Scheduling/MemoryState.php @@ -0,0 +1,54 @@ +lastReview === null) { + return 0; + } + + $days = (int) $this->lastReview->diff($now)->days; + + return max(0, $days); + } + + /** + * Whether this term is due for review at the given moment. + */ + public function isDue(DateTimeImmutable $now): bool + { + return $this->due <= $now; + } +} diff --git a/src/Modules/Review/Domain/Scheduling/Rating.php b/src/Modules/Review/Domain/Scheduling/Rating.php new file mode 100644 index 000000000..1af336190 --- /dev/null +++ b/src/Modules/Review/Domain/Scheduling/Rating.php @@ -0,0 +1,53 @@ +appendUserScope($params); + + $row = Connection::preparedFetchOne( + 'SELECT TsStability, TsDifficulty, TsDue, TsLastReview, TsReps, TsLapses, TsState + FROM term_schedule + JOIN words ON WoID = TsWoID + WHERE TsWoID = ?' . $scope, + $params + ); + + if ($row === null) { + return null; + } + + return new MemoryState( + stability: (float) $row['TsStability'], + difficulty: (float) $row['TsDifficulty'], + due: new DateTimeImmutable((string) $row['TsDue']), + lastReview: $row['TsLastReview'] !== null + ? new DateTimeImmutable((string) $row['TsLastReview']) + : null, + reps: (int) $row['TsReps'], + lapses: (int) $row['TsLapses'], + state: SchedulingState::from((int) $row['TsState']), + ); + } + + public function findOrSeed(int $wordId): ?MemoryState + { + $existing = $this->find($wordId); + if ($existing !== null) { + return $existing; + } + + $params = [$wordId]; + $scope = $this->appendUserScope($params); + + $row = Connection::preparedFetchOne( + 'SELECT WoStatus, WoStatusChanged FROM words WHERE WoID = ?' . $scope, + $params + ); + + if ($row === null) { + return null; + } + + return LegacyStatusSeed::forStatus( + (int) $row['WoStatus'], + new DateTimeImmutable((string) $row['WoStatusChanged']) + ); + } + + public function saveReview(int $wordId, SchedulingResult $result, Rating $rating, int $stateBefore): void + { + // Ownership is checked once here rather than trusted from the caller, + // so neither write below can touch a foreign term. + if (!$this->ownsWord($wordId)) { + return; + } + + $state = $result->state; + + Connection::preparedExecute( + 'INSERT INTO term_schedule + (TsWoID, TsStability, TsDifficulty, TsDue, TsLastReview, TsReps, TsLapses, TsState) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + TsStability = VALUES(TsStability), + TsDifficulty = VALUES(TsDifficulty), + TsDue = VALUES(TsDue), + TsLastReview = VALUES(TsLastReview), + TsReps = VALUES(TsReps), + TsLapses = VALUES(TsLapses), + TsState = VALUES(TsState)', + [ + $wordId, + $state->stability, + $state->difficulty, + $state->due->format(self::DATETIME_FORMAT), + $state->lastReview?->format(self::DATETIME_FORMAT), + $state->reps, + $state->lapses, + $state->state->value, + ] + ); + + Connection::preparedExecute( + 'INSERT INTO review_log + (RlWoID, RlGrade, RlState, RlStability, RlDifficulty, + RlElapsedDays, RlScheduledDays, RlReviewedAt) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + [ + $wordId, + $rating->value, + $stateBefore, + $state->stability, + $state->difficulty, + $result->elapsedDays, + $result->intervalDays, + ($state->lastReview ?? new DateTimeImmutable())->format(self::DATETIME_FORMAT), + ] + ); + } + + public function countDue(?int $languageId = null): int + { + $params = []; + $sql = 'SELECT COUNT(*) AS value + FROM term_schedule + JOIN words ON WoID = TsWoID + WHERE TsDue <= NOW()'; + + if ($languageId !== null) { + $sql .= ' AND WoLgID = ?'; + $params[] = $languageId; + } + + $sql .= $this->appendUserScope($params); + + return (int) Connection::preparedFetchValue($sql, $params); + } + + /** + * Whether the current user owns this term. + */ + private function ownsWord(int $wordId): bool + { + $params = [$wordId]; + $scope = $this->appendUserScope($params); + + /** @var int|string|null $hit */ + $hit = Connection::preparedFetchValue( + 'SELECT 1 AS value FROM words WHERE WoID = ?' . $scope, + $params + ); + + return $hit !== null; + } + + /** + * Append the words-table user scope to $params and return the SQL fragment. + * + * Mirrors ReviewConfiguration::appendUserScope — inlined for the same + * reason, to keep Psalm's `int|string` element type on $params. + * + * @param array $params Reference to params array + */ + private function appendUserScope(array &$params): string + { + if (!Globals::isMultiUserEnabled()) { + return ''; + } + + $userId = Globals::getCurrentUserId(); + if ($userId === null) { + return ''; + } + + $params[] = $userId; + + return ' AND WoUsID = ?'; + } +} diff --git a/tests/backend/Modules/Review/Domain/Scheduling/Fsrs6SchedulerTest.php b/tests/backend/Modules/Review/Domain/Scheduling/Fsrs6SchedulerTest.php new file mode 100644 index 000000000..14253d5bc --- /dev/null +++ b/tests/backend/Modules/Review/Domain/Scheduling/Fsrs6SchedulerTest.php @@ -0,0 +1,293 @@ +}> + */ + public static function referenceSequenceProvider(): array + { + $path = __DIR__ . '/fixtures/fsrs6_reference_vectors.json'; + $raw = file_get_contents($path); + self::assertIsString($raw, 'reference vectors fixture is unreadable'); + + /** @var list>}> $sequences */ + $sequences = json_decode($raw, true, 512, JSON_THROW_ON_ERROR); + + $cases = []; + foreach ($sequences as $sequence) { + $cases[$sequence['name']] = [$sequence]; + } + + return $cases; + } + + /** + * Replay each reference sequence and compare S, D, retrievability and the + * scheduled interval at every step. + * + * @param array{name: string, reviews: list>} $sequence + */ + #[\PHPUnit\Framework\Attributes\DataProvider('referenceSequenceProvider')] + public function testMatchesReferenceImplementation(array $sequence): void + { + $scheduler = $this->scheduler(); + $now = $this->start(); + $state = null; + + foreach ($sequence['reviews'] as $index => $expected) { + $advance = (int) $expected['advance_days']; + if ($advance > 0) { + $now = $now->add(new DateInterval('P' . $advance . 'D')); + } + + $label = sprintf('%s step %d', $sequence['name'], $index); + + $this->assertEqualsWithDelta( + (float) $expected['retrievability_before'], + $scheduler->retrievability($state, $now), + self::EPSILON, + "{$label}: retrievability before review" + ); + + $result = $scheduler->review($state, Rating::from((int) $expected['grade']), $now); + $state = $result->state; + + $this->assertEqualsWithDelta( + (float) $expected['stability'], + $state->stability, + self::EPSILON, + "{$label}: stability" + ); + $this->assertEqualsWithDelta( + (float) $expected['difficulty'], + $state->difficulty, + self::EPSILON, + "{$label}: difficulty" + ); + $this->assertSame( + (int) $expected['interval_days'], + $result->intervalDays, + "{$label}: interval" + ); + } + } + + public function testFirstReviewSeedsStabilityFromGradeWeights(): void + { + $scheduler = $this->scheduler(); + + foreach (Rating::cases() as $rating) { + $result = $scheduler->review(null, $rating, $this->start()); + + $this->assertEqualsWithDelta( + FsrsParameters::DEFAULT_WEIGHTS[$rating->value - 1], + $result->state->stability, + self::EPSILON, + "initial stability for {$rating->name}" + ); + } + } + + public function testRetrievabilityIsZeroForAnUnreviewedTerm(): void + { + $this->assertSame(0.0, $this->scheduler()->retrievability(null, $this->start())); + } + + public function testRetrievabilityIsExactlyTargetAfterOneStabilityPeriod(): void + { + // Stability is *defined* as the time for retrievability to reach 0.9. + $scheduler = $this->scheduler(); + $now = $this->start(); + + $state = new MemoryState( + stability: 10.0, + difficulty: 5.0, + due: $now, + lastReview: $now, + ); + + $tenDaysLater = $now->add(new DateInterval('P10D')); + + $this->assertEqualsWithDelta(0.9, $scheduler->retrievability($state, $tenDaysLater), 1e-12); + } + + public function testAgainRecordsALapseAndMovesToRelearning(): void + { + $scheduler = $this->scheduler(); + $now = $this->start(); + + $first = $scheduler->review(null, Rating::Good, $now); + $this->assertSame(0, $first->state->lapses); + $this->assertSame(SchedulingState::Review, $first->state->state); + + $lapsed = $scheduler->review($first->state, Rating::Again, $now->add(new DateInterval('P5D'))); + + $this->assertSame(1, $lapsed->state->lapses); + $this->assertSame(2, $lapsed->state->reps); + $this->assertSame(SchedulingState::Relearning, $lapsed->state->state); + } + + public function testALapseNeverIncreasesStability(): void + { + $scheduler = $this->scheduler(); + $now = $this->start(); + + $state = new MemoryState( + stability: 50.0, + difficulty: 5.0, + due: $now, + lastReview: $now, + ); + + $result = $scheduler->review($state, Rating::Again, $now->add(new DateInterval('P50D'))); + + $this->assertLessThanOrEqual(50.0, $result->state->stability); + } + + public function testEasyGivesALongerIntervalThanHardFromTheSameState(): void + { + $scheduler = $this->scheduler(); + $now = $this->start(); + $later = $now->add(new DateInterval('P10D')); + + $state = new MemoryState( + stability: 10.0, + difficulty: 5.0, + due: $later, + lastReview: $now, + ); + + $hard = $scheduler->review($state, Rating::Hard, $later)->intervalDays; + $good = $scheduler->review($state, Rating::Good, $later)->intervalDays; + $easy = $scheduler->review($state, Rating::Easy, $later)->intervalDays; + + $this->assertLessThan($good, $hard, 'Hard must schedule sooner than Good'); + $this->assertLessThan($easy, $good, 'Good must schedule sooner than Easy'); + } + + public function testDifficultyStaysWithinBounds(): void + { + $scheduler = $this->scheduler(); + $now = $this->start(); + $state = null; + + // Hammer Again repeatedly — difficulty must saturate at 10, not exceed it. + for ($i = 0; $i < 40; $i++) { + $now = $now->add(new DateInterval('P1D')); + $state = $scheduler->review($state, Rating::Again, $now)->state; + $this->assertGreaterThanOrEqual(FsrsParameters::MIN_DIFFICULTY, $state->difficulty); + $this->assertLessThanOrEqual(FsrsParameters::MAX_DIFFICULTY, $state->difficulty); + $this->assertGreaterThanOrEqual(FsrsParameters::STABILITY_MIN, $state->stability); + } + + // ...and Easy repeatedly must saturate at 1. + $state = null; + for ($i = 0; $i < 40; $i++) { + $now = $now->add(new DateInterval('P30D')); + $state = $scheduler->review($state, Rating::Easy, $now)->state; + $this->assertGreaterThanOrEqual(FsrsParameters::MIN_DIFFICULTY, $state->difficulty); + $this->assertLessThanOrEqual(FsrsParameters::MAX_DIFFICULTY, $state->difficulty); + } + } + + public function testIntervalRespectsMaximum(): void + { + $scheduler = new Fsrs6Scheduler(new FsrsParameters(maximumInterval: 30)); + $now = $this->start(); + + $state = new MemoryState( + stability: 100000.0, + difficulty: 1.0, + due: $now, + lastReview: $now, + ); + + $result = $scheduler->review($state, Rating::Easy, $now->add(new DateInterval('P1D'))); + + $this->assertSame(30, $result->intervalDays); + } + + public function testIntervalIsAtLeastOneDay(): void + { + $scheduler = $this->scheduler(); + $result = $scheduler->review(null, Rating::Again, $this->start()); + + $this->assertGreaterThanOrEqual(1, $result->intervalDays); + } + + public function testHigherRetentionTargetSchedulesSooner(): void + { + $now = $this->start(); + $state = new MemoryState( + stability: 30.0, + difficulty: 5.0, + due: $now, + lastReview: $now, + ); + $later = $now->add(new DateInterval('P10D')); + + $relaxed = (new Fsrs6Scheduler(new FsrsParameters(desiredRetention: 0.8))) + ->review($state, Rating::Good, $later)->intervalDays; + $strict = (new Fsrs6Scheduler(new FsrsParameters(desiredRetention: 0.95))) + ->review($state, Rating::Good, $later)->intervalDays; + + $this->assertLessThan($relaxed, $strict, 'A stricter retention target must review sooner'); + } + + public function testParametersRejectWrongWeightCount(): void + { + $this->expectException(\InvalidArgumentException::class); + new FsrsParameters([1.0, 2.0, 3.0]); + } + + public function testParametersRejectOutOfRangeRetention(): void + { + $this->expectException(\InvalidArgumentException::class); + new FsrsParameters(desiredRetention: 1.0); + } +} diff --git a/tests/backend/Modules/Review/Domain/Scheduling/LegacyStatusSeedTest.php b/tests/backend/Modules/Review/Domain/Scheduling/LegacyStatusSeedTest.php new file mode 100644 index 000000000..ec930de14 --- /dev/null +++ b/tests/backend/Modules/Review/Domain/Scheduling/LegacyStatusSeedTest.php @@ -0,0 +1,115 @@ +changedAt()); + + $this->assertNotNull($state, "status {$status} must seed"); + $this->assertGreaterThanOrEqual(FsrsParameters::STABILITY_MIN, $state->stability); + $this->assertGreaterThanOrEqual(FsrsParameters::MIN_DIFFICULTY, $state->difficulty); + $this->assertLessThanOrEqual(FsrsParameters::MAX_DIFFICULTY, $state->difficulty); + $this->assertSame(SchedulingState::Review, $state->state); + $this->assertSame(0, $state->reps, 'a seeded term has no real review history'); + } + } + + public function testStabilityIncreasesWithStatus(): void + { + $previous = 0.0; + + foreach ([1, 2, 3, 4, 5] as $status) { + $state = LegacyStatusSeed::forStatus($status, $this->changedAt()); + $this->assertNotNull($state); + + $this->assertGreaterThan( + $previous, + $state->stability, + "status {$status} must seed a higher stability than status " . ($status - 1) + ); + $previous = $state->stability; + } + } + + public function testIgnoredAndWellKnownAreNotScheduled(): void + { + $this->assertNull(LegacyStatusSeed::forStatus(98, $this->changedAt())); + $this->assertNull(LegacyStatusSeed::forStatus(99, $this->changedAt())); + } + + public function testUnknownStatusIsNotScheduled(): void + { + $this->assertNull(LegacyStatusSeed::forStatus(0, $this->changedAt())); + $this->assertNull(LegacyStatusSeed::forStatus(42, $this->changedAt())); + } + + /** + * The seed's due date must reproduce the legacy Leitner schedule, so + * upgrading does not dump a user's whole vocabulary into the queue. + */ + public function testDueDateMatchesTheLegacyInterval(): void + { + $changed = $this->changedAt(); + + $expectedDays = [1 => 1, 2 => 2, 3 => 9, 4 => 27, 5 => 71]; + + foreach ($expectedDays as $status => $days) { + $state = LegacyStatusSeed::forStatus($status, $changed); + $this->assertNotNull($state); + + $this->assertSame( + $days, + (int) $changed->diff($state->due)->days, + "status {$status} should stay due {$days} days after its last status change" + ); + } + } + + /** + * A seeded term must survive a real review without the scheduler choking + * on it — this is the actual upgrade path for every existing term. + */ + public function testSeededStateFeedsTheSchedulerCleanly(): void + { + $scheduler = new Fsrs6Scheduler(); + $changed = $this->changedAt(); + $reviewedAt = $changed->modify('+30 days'); + + $seed = LegacyStatusSeed::forStatus(4, $changed); + $this->assertNotNull($seed); + + $result = $scheduler->review($seed, Rating::Good, $reviewedAt); + + $this->assertGreaterThan($seed->stability, $result->state->stability); + $this->assertSame(1, $result->state->reps); + $this->assertSame(30, $result->elapsedDays); + $this->assertGreaterThan(0, $result->intervalDays); + } +} diff --git a/tests/backend/Modules/Review/Domain/Scheduling/fixtures/fsrs6_reference_vectors.json b/tests/backend/Modules/Review/Domain/Scheduling/fixtures/fsrs6_reference_vectors.json new file mode 100644 index 000000000..fb45c0d9c --- /dev/null +++ b/tests/backend/Modules/Review/Domain/Scheduling/fixtures/fsrs6_reference_vectors.json @@ -0,0 +1,229 @@ +[ + { + "name": "all_good", + "reviews": [ + { + "grade": 3, + "advance_days": 0, + "retrievability_before": 0, + "stability": 2.3065, + "difficulty": 2.118103970459, + "interval_days": 2 + }, + { + "grade": 3, + "advance_days": 3, + "retrievability_before": 0.880947955766, + "stability": 13.826903694355, + "difficulty": 2.111214235785, + "interval_days": 14 + }, + { + "grade": 3, + "advance_days": 10, + "retrievability_before": 0.920684082573, + "stability": 47.452033728477, + "difficulty": 2.104331390846, + "interval_days": 47 + } + ] + }, + { + "name": "all_again", + "reviews": [ + { + "grade": 1, + "advance_days": 0, + "retrievability_before": 0, + "stability": 0.212, + "difficulty": 6.4133, + "interval_days": 1 + }, + { + "grade": 1, + "advance_days": 1, + "retrievability_before": 0.766195730228, + "stability": 0.100885789821, + "difficulty": 8.806304468857, + "interval_days": 1 + }, + { + "grade": 1, + "advance_days": 1, + "retrievability_before": 0.693681760194, + "stability": 0.055034530118, + "difficulty": 9.59286876534, + "interval_days": 1 + } + ] + }, + { + "name": "hard_path", + "reviews": [ + { + "grade": 2, + "advance_days": 0, + "retrievability_before": 0, + "stability": 1.2931, + "difficulty": 5.112170705601, + "interval_days": 1 + }, + { + "grade": 2, + "advance_days": 2, + "retrievability_before": 0.867367501757, + "stability": 4.46945534827, + "difficulty": 6.74045951083, + "interval_days": 4 + }, + { + "grade": 3, + "advance_days": 5, + "retrievability_before": 0.892110243127, + "stability": 13.120059931383, + "difficulty": 6.728947420616, + "interval_days": 13 + } + ] + }, + { + "name": "easy_path", + "reviews": [ + { + "grade": 4, + "advance_days": 0, + "retrievability_before": 0, + "stability": 8.2956, + "difficulty": 1.0, + "interval_days": 8 + }, + { + "grade": 4, + "advance_days": 15, + "retrievability_before": 0.854487230786, + "stability": 95.507784291677, + "difficulty": 1.0, + "interval_days": 96 + }, + { + "grade": 3, + "advance_days": 40, + "retrievability_before": 0.948337087463, + "stability": 217.491789592754, + "difficulty": 1.0, + "interval_days": 217 + } + ] + }, + { + "name": "lapse_then_recover", + "reviews": [ + { + "grade": 3, + "advance_days": 0, + "retrievability_before": 0, + "stability": 2.3065, + "difficulty": 2.118103970459, + "interval_days": 2 + }, + { + "grade": 3, + "advance_days": 7, + "retrievability_before": 0.808309940238, + "stability": 21.411391979982, + "difficulty": 2.111214235785, + "interval_days": 21 + }, + { + "grade": 1, + "advance_days": 9, + "retrievability_before": 0.94818240154, + "stability": 1.952053281067, + "difficulty": 7.392238132343, + "interval_days": 2 + }, + { + "grade": 3, + "advance_days": 1, + "retrievability_before": 0.939178195405, + "stability": 3.984164776141, + "difficulty": 7.380074263507, + "interval_days": 4 + } + ] + }, + { + "name": "same_day_repeat", + "reviews": [ + { + "grade": 3, + "advance_days": 0, + "retrievability_before": 0, + "stability": 2.3065, + "difficulty": 2.118103970459, + "interval_days": 2 + }, + { + "grade": 3, + "advance_days": 0, + "retrievability_before": 1.0, + "stability": 2.3065, + "difficulty": 2.111214235785, + "interval_days": 2 + }, + { + "grade": 2, + "advance_days": 0, + "retrievability_before": 1.0, + "stability": 1.333378716804, + "difficulty": 4.748284761595, + "interval_days": 1 + } + ] + }, + { + "name": "mixed", + "reviews": [ + { + "grade": 3, + "advance_days": 0, + "retrievability_before": 0, + "stability": 2.3065, + "difficulty": 2.118103970459, + "interval_days": 2 + }, + { + "grade": 2, + "advance_days": 4, + "retrievability_before": 0.857985789233, + "stability": 10.648376990077, + "difficulty": 4.752858488533, + "interval_days": 11 + }, + { + "grade": 4, + "advance_days": 12, + "retrievability_before": 0.8915822096, + "stability": 59.887280766199, + "difficulty": 2.984736681491, + "interval_days": 60 + }, + { + "grade": 1, + "advance_days": 30, + "retrievability_before": 0.940254547582, + "stability": 2.97795144376, + "difficulty": 7.679359020294, + "interval_days": 3 + }, + { + "grade": 3, + "advance_days": 2, + "retrievability_before": 0.924961761919, + "stability": 6.277857254672, + "difficulty": 7.66690803057, + "interval_days": 6 + } + ] + } +] diff --git a/tests/backend/Modules/Review/Domain/Scheduling/fixtures/generate_reference_vectors.py b/tests/backend/Modules/Review/Domain/Scheduling/fixtures/generate_reference_vectors.py new file mode 100644 index 000000000..0d1334200 --- /dev/null +++ b/tests/backend/Modules/Review/Domain/Scheduling/fixtures/generate_reference_vectors.py @@ -0,0 +1,58 @@ +"""Generate FSRS-6 ground-truth vectors from py-fsrs for the PHP port's tests. + +Configured with learning_steps=() / relearning_steps=() so the reference goes +straight to the Review state — the same simplification the PHP port makes. + +Pinned to fsrs==6.3.1. The version matters: py-fsrs's unreleased `main` widens +the short-term stability clamp from (Good, Easy) to (Hard, Good, Easy), which +changes same-day-repeat results materially. Regenerate against a different +release only together with the matching change in Fsrs6Scheduler. + + python3 -m venv .venv + ./.venv/bin/pip install 'fsrs==6.3.1' + ./.venv/bin/python generate_reference_vectors.py > fsrs6_reference_vectors.json +""" + +import json +from datetime import datetime, timedelta, timezone + +from fsrs import Card, Rating, Scheduler + +scheduler = Scheduler(learning_steps=(), relearning_steps=(), enable_fuzzing=False) + +START = datetime(2026, 1, 1, 12, 0, 0, tzinfo=timezone.utc) + +# (name, [(rating, days_to_advance_before_this_review), ...]) +SEQUENCES = [ + ("all_good", [(Rating.Good, 0), (Rating.Good, 3), (Rating.Good, 10)]), + ("all_again", [(Rating.Again, 0), (Rating.Again, 1), (Rating.Again, 1)]), + ("hard_path", [(Rating.Hard, 0), (Rating.Hard, 2), (Rating.Good, 5)]), + ("easy_path", [(Rating.Easy, 0), (Rating.Easy, 15), (Rating.Good, 40)]), + ("lapse_then_recover", [(Rating.Good, 0), (Rating.Good, 7), (Rating.Again, 9), (Rating.Good, 1)]), + ("same_day_repeat", [(Rating.Good, 0), (Rating.Good, 0), (Rating.Hard, 0)]), + ("mixed", [(Rating.Good, 0), (Rating.Hard, 4), (Rating.Easy, 12), (Rating.Again, 30), (Rating.Good, 2)]), +] + +out = [] +for name, steps in SEQUENCES: + card = Card() + now = START + reviews = [] + for rating, advance in steps: + now = now + timedelta(days=advance) + retr_before = scheduler.get_card_retrievability(card, current_datetime=now) + card, _log = scheduler.review_card(card, rating, review_datetime=now) + interval = (card.due - now).days + reviews.append( + { + "grade": int(rating), + "advance_days": advance, + "retrievability_before": round(retr_before, 12), + "stability": round(card.stability, 12), + "difficulty": round(card.difficulty, 12), + "interval_days": interval, + } + ) + out.append({"name": name, "reviews": reviews}) + +print(json.dumps(out, indent=2)) diff --git a/tests/backend/Modules/Review/Infrastructure/TermScheduleRepositoryIntegrationTest.php b/tests/backend/Modules/Review/Infrastructure/TermScheduleRepositoryIntegrationTest.php new file mode 100644 index 000000000..ebbf0f48c --- /dev/null +++ b/tests/backend/Modules/Review/Infrastructure/TermScheduleRepositoryIntegrationTest.php @@ -0,0 +1,248 @@ + */ + private array $createdTermIds = []; + + public static function setUpBeforeClass(): void + { + $config = EnvLoader::getDatabaseConfig(); + $testDbName = 'test_' . $config['dbname']; + + if (!Globals::getDbConnection()) { + try { + $connection = Configuration::connect( + $config['server'], + $config['userid'], + $config['passwd'], + $testDbName, + $config['socket'] ?? '' + ); + Globals::setDbConnection($connection); + self::$dbConnected = true; + } catch (\Throwable) { + self::$dbConnected = false; + } + } else { + self::$dbConnected = true; + } + + if (!self::$dbConnected) { + return; + } + + Connection::query( + "INSERT INTO languages ( + LgName, LgDict1URI, LgDict2URI, LgGoogleTranslateURI, + LgTextSize, LgRegexpSplitSentences, LgRegexpWordCharacters, + LgRemoveSpaces, LgSplitEachChar, LgRightToLeft, LgShowRomanization + ) VALUES ( + 'FsrsScheduleTest_Lang', 'https://dict.test/fsrs', '', '', + 100, '.!?', 'a-zA-Z', + 0, 0, 0, 1 + )" + ); + self::$languageId = (int) mysqli_insert_id(Globals::getDbConnection()); + } + + public static function tearDownAfterClass(): void + { + if (!self::$dbConnected || self::$languageId === 0) { + return; + } + + // review_log / term_schedule cascade from words, but delete explicitly + // so the test is self-cleaning even if the FKs are absent. + Connection::query( + 'DELETE FROM review_log WHERE RlWoID IN (' + . 'SELECT WoID FROM words WHERE WoLgID = ' . self::$languageId . ')' + ); + Connection::query( + 'DELETE FROM term_schedule WHERE TsWoID IN (' + . 'SELECT WoID FROM words WHERE WoLgID = ' . self::$languageId . ')' + ); + Connection::query('DELETE FROM words WHERE WoLgID = ' . self::$languageId); + Connection::query('DELETE FROM languages WHERE LgID = ' . self::$languageId); + } + + protected function setUp(): void + { + if (!defined('LWT_TEST_DB_AVAILABLE') || !LWT_TEST_DB_AVAILABLE) { + $this->markTestSkipped('Database connection required'); + } + if (!self::$dbConnected) { + $this->markTestSkipped('Test database setup failed'); + } + + $this->createdTermIds = []; + } + + protected function tearDown(): void + { + foreach ($this->createdTermIds as $id) { + Connection::query('DELETE FROM review_log WHERE RlWoID = ' . $id); + Connection::query('DELETE FROM term_schedule WHERE TsWoID = ' . $id); + Connection::query('DELETE FROM words WHERE WoID = ' . $id); + } + } + + private function createTerm(string $text, int $status, string $statusChanged): int + { + Connection::preparedExecute( + 'INSERT INTO words (WoLgID, WoText, WoTextLC, WoStatus, WoTranslation, + WoStatusChanged, WoTodayScore, WoTomorrowScore, WoRandom) + VALUES (?, ?, ?, ?, ?, ?, 0, 0, RAND())', + [self::$languageId, $text, strtolower($text), $status, 'translation', $statusChanged] + ); + + $id = (int) mysqli_insert_id(Globals::getDbConnection()); + $this->createdTermIds[] = $id; + + return $id; + } + + public function testUnreviewedTermHasNoStoredStateButSeedsFromStatus(): void + { + $repo = new MySqlTermScheduleRepository(); + $wordId = $this->createTerm('seedme', 4, '2026-01-01 12:00:00'); + + $this->assertNull($repo->find($wordId), 'nothing should be stored yet'); + + $seeded = $repo->findOrSeed($wordId); + $this->assertNotNull($seeded); + $this->assertEqualsWithDelta(27.0, $seeded->stability, 1e-9, 'status 4 seeds the legacy 27-day interval'); + $this->assertSame('2026-01-01', $seeded->lastReview?->format('Y-m-d')); + } + + public function testIgnoredTermSeedsNothing(): void + { + $repo = new MySqlTermScheduleRepository(); + $wordId = $this->createTerm('ignoreme', 98, '2026-01-01 12:00:00'); + + $this->assertNull($repo->findOrSeed($wordId)); + } + + public function testSaveReviewPersistsStateAndAppendsLog(): void + { + $repo = new MySqlTermScheduleRepository(); + $scheduler = new Fsrs6Scheduler(); + $wordId = $this->createTerm('persistme', 3, '2026-01-01 12:00:00'); + + $reviewedAt = new DateTimeImmutable('2026-02-01 09:00:00'); + $result = $scheduler->review($repo->findOrSeed($wordId), Rating::Good, $reviewedAt); + + $repo->saveReview($wordId, $result, Rating::Good, 2); + + $stored = $repo->find($wordId); + $this->assertNotNull($stored, 'state must round-trip'); + $this->assertEqualsWithDelta($result->state->stability, $stored->stability, 1e-6); + $this->assertEqualsWithDelta($result->state->difficulty, $stored->difficulty, 1e-6); + $this->assertSame(1, $stored->reps); + $this->assertSame(0, $stored->lapses); + + $logCount = (int) Connection::preparedFetchValue( + 'SELECT COUNT(*) AS value FROM review_log WHERE RlWoID = ?', + [$wordId] + ); + $this->assertSame(1, $logCount, 'exactly one log row per review'); + + $log = Connection::preparedFetchOne( + 'SELECT RlGrade, RlScheduledDays, RlElapsedDays FROM review_log WHERE RlWoID = ?', + [$wordId] + ); + $this->assertNotNull($log); + $this->assertSame(Rating::Good->value, (int) $log['RlGrade']); + $this->assertSame($result->intervalDays, (int) $log['RlScheduledDays']); + // 2026-01-01 12:00 -> 2026-02-01 09:00 is 30 days and 21 hours, and + // FSRS works in whole elapsed days, so this floors to 30. + $this->assertSame(30, (int) $log['RlElapsedDays']); + } + + public function testRepeatedReviewsUpsertStateAndAccumulateLog(): void + { + $repo = new MySqlTermScheduleRepository(); + $useCase = new RecordScheduledReview(new Fsrs6Scheduler(), $repo); + $wordId = $this->createTerm('repeatme', 2, '2026-01-01 12:00:00'); + + $this->assertTrue($useCase->execute($wordId, Rating::Good, new DateTimeImmutable('2026-02-01 09:00:00'))); + $this->assertTrue($useCase->execute($wordId, Rating::Again, new DateTimeImmutable('2026-02-10 09:00:00'))); + $this->assertTrue($useCase->execute($wordId, Rating::Good, new DateTimeImmutable('2026-02-12 09:00:00'))); + + $stateRows = (int) Connection::preparedFetchValue( + 'SELECT COUNT(*) AS value FROM term_schedule WHERE TsWoID = ?', + [$wordId] + ); + $this->assertSame(1, $stateRows, 'state is upserted, never duplicated'); + + $logRows = (int) Connection::preparedFetchValue( + 'SELECT COUNT(*) AS value FROM review_log WHERE RlWoID = ?', + [$wordId] + ); + $this->assertSame(3, $logRows, 'log is append-only'); + + $stored = $repo->find($wordId); + $this->assertNotNull($stored); + $this->assertSame(3, $stored->reps); + $this->assertSame(1, $stored->lapses, 'the single Again counted as one lapse'); + } + + public function testCountDueOnlyCountsTermsPastTheirDueDate(): void + { + $repo = new MySqlTermScheduleRepository(); + $useCase = new RecordScheduledReview(new Fsrs6Scheduler(), $repo); + + $overdue = $this->createTerm('overdueterm', 3, '2026-01-01 12:00:00'); + $fresh = $this->createTerm('freshterm', 3, '2026-01-01 12:00:00'); + + // Reviewed long ago with a short interval -> due by now. + $useCase->execute($overdue, Rating::Again, new DateTimeImmutable('2026-01-02 09:00:00')); + // Reviewed right now with a long interval -> not due. + $useCase->execute($fresh, Rating::Easy, new DateTimeImmutable()); + + $due = $repo->countDue(self::$languageId); + + $this->assertGreaterThanOrEqual(1, $due, 'the overdue term must be counted'); + + $freshState = $repo->find($fresh); + $this->assertNotNull($freshState); + $this->assertFalse( + $freshState->isDue(new DateTimeImmutable()), + 'a term just rated Easy must not be due' + ); + } + + public function testStateForAMissingTermIsNull(): void + { + $repo = new MySqlTermScheduleRepository(); + + $this->assertNull($repo->find(999999999)); + $this->assertNull($repo->findOrSeed(999999999)); + } +}