fix(infinifi): re-arm liquid reserves alert when cache is stale

monitoring.yaml

@@ -26,23 +26,23 @@ protocols:
       - name: "Price Per Share"
         description: "USD3 PPS decrease (any drop vs cached prior, CRITICAL) and sUSD3 PPS decrease (HIGH)"
       - name: "TVL"
-        description: "Absolute TVL change >=15% for USD3 or sUSD3"
+        description: "Absolute TVL change >=15% for USD3 or sUSD3 using a baseline no older than 3 hours"
       - name: "Junior Buffer Ratio"
-        description: "sUSD3 backing <15% of deployed credit"
+        description: "sUSD3 backing <15% of deployed credit; re-armed after a monitoring gap >3 hours"
       - name: "USD3 Overcollateralization"
-        description: "OC <111% (HIGH) or <106% (CRITICAL)"
+        description: "OC <111% (HIGH) or <106% (CRITICAL); re-armed after a monitoring gap >3 hours"
       - name: "Insurance Fund Outflow"
         description: "waUSDC outflow >=$50k since prior run"
       - name: "Withdraw Liquidity"
-        description: "USD3 availableWithdrawLimit < $4M"
+        description: "USD3 availableWithdrawLimit < $4M; re-armed after a monitoring gap >3 hours"
       - name: "Vault Shutdown"
-        description: "Alert-once when isShutdown() transitions to true"
+        description: "Alert-once when isShutdown() is true; re-armed after a monitoring gap >3 hours"
       - name: "Debt Cap"
         description: "Any change to ProtocolConfig.getDebtCap()"
       - name: "Nominal sUSD3 Backing Floor"
-        description: "Any floor change; alert-once when floor > sUSD3 backing"
+        description: "Any floor change; floor-breach dedupe re-armed after a monitoring gap >3 hours"
       - name: "Protocol Pause"
-        description: "Alert-once when ProtocolConfig IS_PAUSED flips true"
+        description: "Alert-once when IS_PAUSED is true; re-armed after a monitoring gap >3 hours"
       - name: "Borrower Default Watch"
         description: "Envio-backed MorphoCredit borrower watch; MEDIUM alerts when unpaid obligations become delinquent after grace or reach default"
       - name: "Timelock"
@@ -85,7 +85,7 @@ protocols:
       - name: "Withdrawable Liquidity"
         description: "Total withdrawable liquidity across cUSD assets < $15M"
       - name: "Large cUSD Mints"
-        description: "totalSupply delta >5% vs cached prior"
+        description: "totalSupply delta >5% vs a cached prior no older than 36 hours"
       - name: "Timelock"
         description: "CAP TimelockController events via Envio"
       - name: "Safe Multisig"
@@ -171,19 +171,19 @@ protocols:
       - protocols/infinifi/main.py
     monitors:
       - name: "Liquid Reserves"
-        description: "Liquid reserves < $8M"
+        description: "Liquid reserves < $8M; crossing detection re-armed after a monitoring gap >3 hours"
       - name: "Backing Per iUSD"
-        description: "totalTVL / iUSD supply < 0.999"
+        description: "totalTVL / iUSD supply < 0.999; re-armed after a monitoring gap >3 hours"
       - name: "Redemption Pressure"
-        description: "Pending redemptions / liquid reserves >80%"
+        description: "Pending redemptions / liquid reserves >80%; re-armed after a monitoring gap >3 hours"
       - name: "Large iUSD Mints"
-        description: "totalSupply delta >5% vs cached prior"
+        description: "totalSupply delta >5% vs a cached prior no older than 3 hours"
       - name: "Farm Allocation Shift"
         description: "Farm allocation ratio change >30% vs prior run (farms <1% of TVL excluded)"
       - name: "Farm Activation"
         description: "Farm previously at 0 ratio moves above 3% of TVL"
       - name: "Junior TVL Coverage"
-        description: "Junior TVL <50% of risky farm exposure"
+        description: "Junior TVL <50% of risky farm exposure; re-armed after a monitoring gap >3 hours"
       - name: "Timelock"
         description: "Longtimelock and Shorttimelock events via Envio"
 
@@ -245,7 +245,7 @@ protocols:
       - name: "Price Per Share"
         description: "Any PPS decrease"
       - name: "TVL"
-        description: "Absolute TVL change >=15%"
+        description: "Absolute TVL change >=15% using a baseline no older than 3 hours"
       - name: "Unrealized Losses"
         description: "Any non-zero on-chain unrealized losses; >=0.5% of pool assets via subgraph"
       - name: "Withdrawal Queue"
@@ -259,7 +259,7 @@ protocols:
       - name: "Unknown Collateral"
         description: "Collateral assets not in risk mapping"
       - name: "Pool Delegate Cover"
-        description: "Cover balance drops to $0 or decreases vs cached prior"
+        description: "Cover balance drops to $0 or decreases vs cached prior; zero state re-armed after >3 hours"
       - name: "Timelock"
         description: "Maple GovernorTimelock events via Envio"
       - name: "Safe Multisig"
@@ -388,13 +388,13 @@ protocols:
       - name: "srUSDe Exchange Rate"
         description: "srUSDe convertToAssets decrease vs cached prior"
       - name: "Senior Coverage Ratio"
-        description: "Senior coverage <105%"
+        description: "Senior coverage <105%; re-armed after a monitoring gap >36 hours"
       - name: "Junior Tranche Drain"
-        description: "jrUSDe totalAssets drop >=15%"
+        description: "jrUSDe totalAssets drop >=15% using a baseline no older than 36 hours"
       - name: "Strategy Balance"
-        description: "sUSDe strategy balance / total deposits drops >=20%"
+        description: "sUSDe strategy balance / total deposits drops >=20% using a baseline no older than 36 hours"
       - name: "TVL"
-        description: "Total deposits change >=15%"
+        description: "Total deposits change >=15% using a baseline no older than 36 hours"
       - name: "sUSDe Dependency"
         description: "sUSDe vault rate monotonicity and cooldown duration changes"
       - name: "Timelock"
@@ -424,7 +424,7 @@ protocols:
       - name: "Oracle Divergence"
         description: "Continuous Oracle vs Chainlink differ by >0.5%"
       - name: "Supply Changes"
-        description: "Total supply change >10% vs prior run"
+        description: "Total supply change >10% vs a prior observation no older than 3 hours"
       - name: "Oracle Staleness"
         description: "Latest checkpoint effectiveAt >4 days old"
       - name: "Stablecoin Price"
@@ -439,13 +439,13 @@ protocols:
       - protocols/usdai/large_mints.py
     monitors:
       - name: "Backing Invariant"
-        description: "totalSupply + bridgedSupply - PYUSD balance >= 100 USDai"
+        description: "totalSupply + bridgedSupply - PYUSD balance >= 100 USDai; re-armed after a gap >3 hours"
       - name: "Loan Activity"
         description: "Total verified principal change >1% vs prior run"
       - name: "Legacy Loan Expiry"
         description: "Hardcoded NVIDIA H200s legacy loan past 2028-07-27"
       - name: "Large Mints"
-        description: "totalSupply delta >5% via large_mints.py"
+        description: "totalSupply delta >5% via large_mints.py using a baseline no older than 3 hours"
       - name: "Safe Multisigs"
         description: "USDai and sUSDai Admin Safe queues"
 

protocols/3jane/README.md

@@ -45,6 +45,13 @@
 | Borrower delinquent/default watch | New milestone: delinquent, ≤14d, ≤7d, ≤3d, ≤1d, default | MEDIUM |
 | Monitoring run failure | Uncaught exception in `main()` | LOW |
 
+## Cache Freshness
+
+TVL delta baselines expire after 3 hours and initialize from the next valid observation. Current-state and threshold
+dedupe for junior buffer, USD3 OC, withdraw liquidity, vault shutdown, nominal-floor breach, and protocol pause is
+re-armed after the same monitoring gap. PPS, insurance-fund, governance-value, and borrower-event history does not
+expire.
+
 ## Borrower default watch
 
 Set `ENVIO_GRAPHQL_URL` to the 3Jane Envio GraphQL endpoint to enable proactive borrower monitoring. Without this env var, the borrower default watch is skipped and all other 3Jane checks continue normally.

protocols/3jane/main.py

@@ -34,7 +34,14 @@
 
 from utils.abi import load_abi
 from utils.alert import Alert, AlertSeverity, send_alert
-from utils.cache import cache_path, get_last_value_for_key_from_file, write_last_value_to_file
+from utils.cache import (
+    HOURLY_CACHE_STALE_AFTER_SECONDS,
+    cache_path,
+    get_fresh_last_value_for_key_from_file,
+    get_last_value_for_key_from_file,
+    write_last_value_to_file,
+    write_last_value_with_timestamp_to_file,
+)
 from utils.chains import Chain
 from utils.formatting import format_duration, format_usd
 from utils.logger import get_logger
@@ -168,9 +175,23 @@ def set_cache_value(key: str, value: int | float) -> None:
     write_last_value_to_file(CACHE_FILENAME, key, value)
 
 
+def get_fresh_cache_value(key: str) -> float:
+    """Read an hourly cache value, returning zero when its observation is stale."""
+    val = get_fresh_last_value_for_key_from_file(CACHE_FILENAME, key, HOURLY_CACHE_STALE_AFTER_SECONDS)
+    try:
+        return float(val)
+    except (ValueError, TypeError):
+        return 0.0
+
+
+def set_fresh_cache_value(key: str, value: int | float) -> None:
+    """Write an hourly cache value and its observation timestamp."""
+    write_last_value_with_timestamp_to_file(CACHE_FILENAME, key, value)
+
+
 def _get_alerted_value(cache_key: str) -> float:
     """Read the last alerted value for a threshold alert (-1 = none outstanding)."""
-    raw_cached = get_last_value_for_key_from_file(CACHE_FILENAME, cache_key)
+    raw_cached = get_fresh_last_value_for_key_from_file(CACHE_FILENAME, cache_key, HOURLY_CACHE_STALE_AFTER_SECONDS)
     try:
         return float(raw_cached) if isinstance(raw_cached, str) else -1.0
     except ValueError:
@@ -199,18 +220,20 @@ def should_alert_value_drop(cache_key: str, value: float, threshold: float) -> b
         return False
 
     cached = _get_alerted_value(cache_key)
-    return not 0 <= cached <= value
+    should_alert = not 0 <= cached <= value
+    if not should_alert:
+        set_fresh_cache_value(cache_key, cached)
+    return should_alert
 
 
 def mark_alerted_value(cache_key: str, value: float) -> None:
     """Record the value a threshold alert fired for; call after send_alert() returns."""
-    set_cache_value(cache_key, value)
+    set_fresh_cache_value(cache_key, value)
 
 
 def clear_alerted_value(cache_key: str) -> None:
     """Clear an outstanding threshold alert so the next breach alerts again."""
-    if _get_alerted_value(cache_key) >= 0:
-        set_cache_value(cache_key, -1)
+    set_fresh_cache_value(cache_key, -1)
 
 
 def _as_bool(value: Any) -> bool:
@@ -544,7 +567,7 @@ def check_tvl(usd3_tvl: float, susd3_tvl: float) -> None:
         susd3_tvl: Current sUSD3 totalAssets in USD3 terms.
     """
     # --- USD3 TVL ---
-    previous_usd3_tvl = get_cache_value(CACHE_KEY_USD3_TVL)
+    previous_usd3_tvl = get_fresh_cache_value(CACHE_KEY_USD3_TVL)
     logger.info("USD3 TVL: %s (previous: %s)", format_usd(usd3_tvl), format_usd(previous_usd3_tvl))
 
     if previous_usd3_tvl > 0:
@@ -559,11 +582,10 @@ def check_tvl(usd3_tvl: float, susd3_tvl: float) -> None:
             )
             send_alert(Alert(AlertSeverity.LOW, message, PROTOCOL))
 
-    if usd3_tvl != previous_usd3_tvl:
-        set_cache_value(CACHE_KEY_USD3_TVL, usd3_tvl)
+    set_fresh_cache_value(CACHE_KEY_USD3_TVL, usd3_tvl)
 
     # --- sUSD3 TVL ---
-    previous_susd3_tvl = get_cache_value(CACHE_KEY_SUSD3_TVL)
+    previous_susd3_tvl = get_fresh_cache_value(CACHE_KEY_SUSD3_TVL)
     logger.info("sUSD3 TVL: %s (previous: %s)", format_usd(susd3_tvl), format_usd(previous_susd3_tvl))
 
     if previous_susd3_tvl > 0:
@@ -579,8 +601,7 @@ def check_tvl(usd3_tvl: float, susd3_tvl: float) -> None:
             )
             send_alert(Alert(AlertSeverity.LOW, message, PROTOCOL))
 
-    if susd3_tvl != previous_susd3_tvl:
-        set_cache_value(CACHE_KEY_SUSD3_TVL, susd3_tvl)
+    set_fresh_cache_value(CACHE_KEY_SUSD3_TVL, susd3_tvl)
 
 
 def check_junior_buffer(susd3_backing: float, deployed_credit: float) -> None:
@@ -765,28 +786,26 @@ def check_vault_shutdown(client, usd3_vault, susd3_vault) -> None:  # type: igno
     logger.info("Vault shutdown — USD3: %s, sUSD3: %s", usd3_shutdown, susd3_shutdown)
 
     # Alert once on USD3 shutdown
-    previous_usd3_shutdown = get_cache_value(CACHE_KEY_SHUTDOWN_USD3)
+    previous_usd3_shutdown = get_fresh_cache_value(CACHE_KEY_SHUTDOWN_USD3)
     if usd3_shutdown and previous_usd3_shutdown == 0:
         message = (
             f"🚨 *3Jane USD3 Vault SHUTDOWN*\n"
             f"⚠️ USD3 vault has entered emergency shutdown\n"
             f"🔗 [USD3](https://etherscan.io/address/{USD3_ADDRESS})"
         )
         send_alert(Alert(AlertSeverity.CRITICAL, message, PROTOCOL))
-    if float(usd3_shutdown) != previous_usd3_shutdown:
-        set_cache_value(CACHE_KEY_SHUTDOWN_USD3, float(usd3_shutdown))
+    set_fresh_cache_value(CACHE_KEY_SHUTDOWN_USD3, float(usd3_shutdown))
 
     # Alert once on sUSD3 shutdown
-    previous_susd3_shutdown = get_cache_value(CACHE_KEY_SHUTDOWN_SUSD3)
+    previous_susd3_shutdown = get_fresh_cache_value(CACHE_KEY_SHUTDOWN_SUSD3)
     if susd3_shutdown and previous_susd3_shutdown == 0:
         message = (
             f"🚨 *3Jane sUSD3 Vault SHUTDOWN*\n"
             f"⚠️ sUSD3 vault has entered emergency shutdown\n"
             f"🔗 [sUSD3](https://etherscan.io/address/{SUSD3_ADDRESS})"
         )
         send_alert(Alert(AlertSeverity.CRITICAL, message, PROTOCOL))
-    if float(susd3_shutdown) != previous_susd3_shutdown:
-        set_cache_value(CACHE_KEY_SHUTDOWN_SUSD3, float(susd3_shutdown))
+    set_fresh_cache_value(CACHE_KEY_SHUTDOWN_SUSD3, float(susd3_shutdown))
 
 
 def check_debt_cap(client) -> None:  # type: ignore[no-untyped-def]
@@ -862,7 +881,7 @@ def check_nominal_backing_floor(nominal_floor: float, susd3_backing: float) -> N
 
     # --- Alert-once on breach transition (floor > backing) ---
     breach = nominal_floor > susd3_backing and nominal_floor > 0
-    previous_breach = get_cache_value(CACHE_KEY_FLOOR_BREACH)
+    previous_breach = get_fresh_cache_value(CACHE_KEY_FLOOR_BREACH)
     if breach and previous_breach == 0:
         shortfall = nominal_floor - susd3_backing
         message = (
@@ -873,8 +892,7 @@ def check_nominal_backing_floor(nominal_floor: float, susd3_backing: float) -> N
             f"🔗 [sUSD3](https://etherscan.io/address/{SUSD3_ADDRESS})"
         )
         send_alert(Alert(AlertSeverity.MEDIUM, message, PROTOCOL))
-    if float(breach) != previous_breach:
-        set_cache_value(CACHE_KEY_FLOOR_BREACH, float(breach))
+    set_fresh_cache_value(CACHE_KEY_FLOOR_BREACH, float(breach))
 
 
 def check_protocol_paused(is_paused: bool) -> None:
@@ -888,16 +906,15 @@ def check_protocol_paused(is_paused: bool) -> None:
     """
     logger.info("Protocol IS_PAUSED: %s", is_paused)
 
-    previous_paused = get_cache_value(CACHE_KEY_IS_PAUSED)
+    previous_paused = get_fresh_cache_value(CACHE_KEY_IS_PAUSED)
     if is_paused and previous_paused == 0:
         message = (
             f"🚨 *3Jane Protocol PAUSED*\n"
             f"⚠️ ProtocolConfig IS_PAUSED flipped to true\n"
             f"🔗 [ProtocolConfig](https://etherscan.io/address/{PROTOCOL_CONFIG_ADDRESS})"
         )
         send_alert(Alert(AlertSeverity.CRITICAL, message, PROTOCOL))
-    if float(is_paused) != previous_paused:
-        set_cache_value(CACHE_KEY_IS_PAUSED, float(is_paused))
+    set_fresh_cache_value(CACHE_KEY_IS_PAUSED, float(is_paused))
 
 
 def main() -> None:

protocols/cap/README.md

@@ -29,3 +29,6 @@ Large mint monitoring is integrated into [liquidity.py](./liquidity.py).
 It intentionally does **not** scan events. Instead, it compares cached `totalSupply` values and alerts when the increase is above:
 
 - `CUSD_LARGE_MINT_THRESHOLD_PERCENT` (default: `0.05`, i.e. `5%` of previous `totalSupply`)
+
+The daily supply baseline expires after 36 hours and initializes from the next valid observation, preventing multiple
+missed daily runs from being treated as one large mint interval.

protocols/cap/liquidity.py

@@ -2,7 +2,12 @@
 
 from utils.abi import load_abi
 from utils.alert import Alert, AlertSeverity, send_alert
-from utils.cache import cache_filename, get_last_value_for_key_from_file, write_last_value_to_file
+from utils.cache import (
+    DAILY_CACHE_STALE_AFTER_SECONDS,
+    cache_filename,
+    get_fresh_last_value_for_key_from_file,
+    write_last_value_with_timestamp_to_file,
+)
 from utils.chains import Chain
 from utils.config import Config
 from utils.logger import get_logger
@@ -77,7 +82,9 @@ def main():
 
     # --- cUSD Large Mint Monitoring (No Event Scanning) ---
     current_supply_raw = int(ctoken.functions.totalSupply().call())
-    last_supply_cached = _to_int(get_last_value_for_key_from_file(cache_filename, CACHE_KEY_LAST_SUPPLY))
+    last_supply_cached = _to_int(
+        get_fresh_last_value_for_key_from_file(cache_filename, CACHE_KEY_LAST_SUPPLY, DAILY_CACHE_STALE_AFTER_SECONDS)
+    )
     if last_supply_cached > 0:
         delta_raw = current_supply_raw - last_supply_cached
         threshold_raw = int(last_supply_cached * MINT_THRESHOLD_PERCENT)
@@ -99,7 +106,7 @@ def main():
             )
             send_alert(Alert(AlertSeverity.LOW, msg, PROTOCOL))
 
-    write_last_value_to_file(cache_filename, CACHE_KEY_LAST_SUPPLY, current_supply_raw)
+    write_last_value_with_timestamp_to_file(cache_filename, CACHE_KEY_LAST_SUPPLY, current_supply_raw)
 
 
 if __name__ == "__main__":

protocols/infinifi/README.md

@@ -28,6 +28,11 @@ It compares cached `totalSupply` deltas and alerts when the increase is above:
 
 - `IUSD_LARGE_MINT_THRESHOLD_PERCENT` (default: `0.05`, i.e. `5%` of previous `totalSupply`)
 
+## Cache Freshness
+
+Hourly delta baselines expire after 3 hours and initialize from the next valid observation. Breach-dedupe state and
+liquid-reserve crossing detection are re-armed after the same monitoring gap. PPS-style loss baselines are not affected.
+
 ### Emergency dispatch
 
 HIGH and CRITICAL alerts automatically trigger a signed webhook to