fix: prevent canonical-csv merge from collapsing distinct conferences

.github/workflows/canonical-source-update.yml

@@ -117,6 +117,7 @@ jobs:
         run: |
           cp _data/conferences.yml /tmp/conferences_before.yml
           cp _data/archive.yml /tmp/archive_before.yml 2>/dev/null || true
+          cp _data/legacy.yml /tmp/legacy_before.yml 2>/dev/null || true
 
       - name: Setup Pixi
         uses: prefix-dev/setup-pixi@v0.10.0
@@ -150,6 +151,15 @@ jobs:
           # Capture which source was processed for commit message
           echo "source_label=$SOURCE" >> $GITHUB_OUTPUT
 
+      - name: Verify no conference data loss
+        run: |
+          # An automated merge must never delete or rename existing conferences.
+          # This blocks e.g. two distinct conferences being fuzzy-matched into one
+          # (PyCon Africa vs PyCon South Africa) before anything is committed.
+          pixi run python ./utils/check_data_loss.py \
+            --before /tmp/conferences_before.yml /tmp/archive_before.yml /tmp/legacy_before.yml \
+            --after _data/conferences.yml _data/archive.yml _data/legacy.yml
+
       - name: Check for changes
         id: check_changes
         run: |

tests/test_fuzzy_match.py

@@ -495,6 +495,115 @@ def test_below_90_percent_no_prompt(self, mock_title_mappings):
         assert len(remote) >= 1
 
 
+class TestSubsetNameCollision:
+    """Regression tests for distinct conferences whose names score 100.
+
+    token_set_ratio returns 100 when one name's tokens are a subset of the
+    other's (e.g. "PyCon Africa" vs "PyCon South Africa"). Such pairs must
+    never be auto-merged as "exact" matches - that previously renamed
+    PyCon South Africa to PyCon Africa and deleted the real PyCon Africa
+    entry (commit d76e737).
+    """
+
+    @staticmethod
+    def _africa_yaml(include_africa: bool) -> pd.DataFrame:
+        rows = {
+            "conference": ["PyCon South Africa"],
+            "year": [2026],
+            "cfp": ["2026-06-01 23:59:00"],
+            "link": ["https://za.pycon.org/"],
+            "place": ["Rondebosch, South Africa"],
+            "start": ["2026-10-14"],
+            "end": ["2026-10-18"],
+        }
+        df = pd.DataFrame(rows)
+        if include_africa:
+            africa = pd.DataFrame(
+                {
+                    "conference": ["PyCon Africa"],
+                    "year": [2026],
+                    "cfp": ["2026-04-16 23:59:00"],
+                    "link": ["https://africa.pycon.org/"],
+                    "place": ["Kampala, Uganda"],
+                    "start": ["2026-10-07"],
+                    "end": ["2026-10-11"],
+                },
+            )
+            df = pd.concat([df, africa], ignore_index=True)
+        return df
+
+    @staticmethod
+    def _africa_remote() -> pd.DataFrame:
+        return pd.DataFrame(
+            {
+                "conference": ["PyCon Africa"],
+                "year": [2026],
+                "cfp": [""],
+                "link": ["https://africa.pycon.org/"],
+                "place": ["Kampala, Uganda"],
+                "start": ["2026-10-07"],
+                "end": ["2026-10-11"],
+                "sponsor": ["https://africa.pycon.org/2026/sponsor-us/"],
+            },
+        )
+
+    def test_subset_name_is_not_an_exact_match(self, mock_title_mappings):
+        """A token-subset name pair must not auto-merge without confirmation.
+
+        Contract: "PyCon South Africa" vs "PyCon Africa" scores 100 via
+        token_set_ratio, but the names are not identical, so it must go
+        through the fuzzy confirmation path (which defaults to "no" in CI).
+        """
+        df_yml = self._africa_yaml(include_africa=False)
+        df_remote = self._africa_remote()
+
+        # Non-interactive / user rejects: conferences stay separate
+        with patch("builtins.input", return_value="n"):
+            result, _remote, _report = fuzzy_match(df_yml, df_remote)
+
+        conf_list = result["conference"].tolist()
+        assert "PyCon South Africa" in conf_list, f"PyCon South Africa was renamed/merged away: {conf_list}"
+
+        za_row = result[result["conference"] == "PyCon South Africa"].iloc[0]
+        assert za_row["link"] == "https://za.pycon.org/"
+        assert za_row["start"] == "2026-10-14"
+
+    def test_both_africa_conferences_survive_merge(self, mock_title_mappings):
+        """Reproduce the d76e737 incident: both conferences must survive.
+
+        YAML has PyCon Africa (Kampala) and PyCon South Africa (Rondebosch);
+        the remote CSV has only PyCon Africa. The remote row belongs to its
+        identically-named YAML entry - PyCon South Africa must be left
+        completely alone, without prompting.
+        """
+        df_yml = self._africa_yaml(include_africa=True)
+        df_remote = self._africa_remote()
+
+        with patch(
+            "builtins.input",
+            side_effect=AssertionError("Should not prompt: remote row belongs to identical YAML entry"),
+        ):
+            result, _remote, _report = fuzzy_match(df_yml, df_remote)
+
+        conf_list = result["conference"].tolist()
+        assert "PyCon Africa" in conf_list, f"PyCon Africa was lost: {conf_list}"
+        assert "PyCon South Africa" in conf_list, f"PyCon South Africa was lost: {conf_list}"
+
+        za_row = result[result["conference"] == "PyCon South Africa"].iloc[0]
+        assert za_row["link"] == "https://za.pycon.org/"
+        assert za_row["place"] == "Rondebosch, South Africa"
+        assert za_row["start"] == "2026-10-14"
+        assert pd.isna(za_row.get("sponsor")) or za_row.get("sponsor") in (
+            "",
+            None,
+        ), "PyCon South Africa must not inherit PyCon Africa's sponsor link"
+
+        africa_row = result[result["conference"] == "PyCon Africa"].iloc[0]
+        assert africa_row["place"] == "Kampala, Uganda"
+        assert africa_row["start"] == "2026-10-07"
+        assert africa_row["sponsor"] == "https://africa.pycon.org/2026/sponsor-us/"
+
+
 class TestDataPreservation:
     """Test that original data is preserved through fuzzy matching."""
 

tests/test_redundant_links.py

@@ -0,0 +1,122 @@
+"""Tests for dropping sub-page link fields that just repeat the homepage.
+
+Upstream sources sometimes fill every URL column with the conference
+homepage (e.g. python-organizers' Proposal URL). A cfp_link/sponsor/finaid
+identical to the main link carries no information and must be dropped
+during sanitation - but a different path, subdomain, query, or #anchor is a
+different pointer and must survive.
+"""
+
+import sys
+from pathlib import Path
+
+sys.path.append(str(Path(__file__).parent.parent / "utils"))
+
+from tidy_conf.links import drop_redundant_link_fields
+from tidy_conf.links import normalize_url_pointer
+
+
+class TestNormalizeUrlPointer:
+    """Test URL normalization for same-pointer comparison."""
+
+    def test_scheme_ignored(self):
+        assert normalize_url_pointer("http://pycon.de/") == normalize_url_pointer("https://pycon.de/")
+
+    def test_www_prefix_ignored(self):
+        assert normalize_url_pointer("https://www.pycon.de/") == normalize_url_pointer("https://pycon.de/")
+
+    def test_trailing_slash_ignored(self):
+        assert normalize_url_pointer("https://2027.pycon.de/") == normalize_url_pointer("https://2027.pycon.de")
+
+    def test_fragment_is_different_pointer(self):
+        assert normalize_url_pointer("http://pycon.sg/#sponsors") != normalize_url_pointer("http://pycon.sg/")
+
+    def test_path_is_different_pointer(self):
+        assert normalize_url_pointer("https://pycon.de/sponsoring/") != normalize_url_pointer("https://pycon.de/")
+
+    def test_subdomain_is_different_pointer(self):
+        assert normalize_url_pointer("https://cfp.pycon.de/") != normalize_url_pointer("https://pycon.de/")
+
+    def test_query_is_different_pointer(self):
+        assert normalize_url_pointer("https://pycon.de/?page=cfp") != normalize_url_pointer("https://pycon.de/")
+
+
+class TestDropRedundantLinkFields:
+    """Test removal of sub-page links identical to the main link."""
+
+    def test_cfp_link_same_as_link_dropped(self):
+        """Reproduces the PyCon DE 2027 case: cfp_link is just the homepage."""
+        data = [
+            {
+                "conference": "PyCon DE & PyData",
+                "year": 2027,
+                "link": "https://2027.pycon.de/",
+                "cfp_link": "https://2027.pycon.de/",
+                "cfp": "TBA",
+            },
+        ]
+        result = drop_redundant_link_fields(data)
+        assert "cfp_link" not in result[0]
+        assert result[0]["link"] == "https://2027.pycon.de/"
+
+    def test_all_redundant_sub_fields_dropped(self):
+        data = [
+            {
+                "conference": "Cheeky Conf",
+                "year": 2026,
+                "link": "https://cheeky.conf/",
+                "cfp_link": "https://cheeky.conf",
+                "sponsor": "http://www.cheeky.conf/",
+                "finaid": "https://cheeky.conf/",
+            },
+        ]
+        result = drop_redundant_link_fields(data)
+        assert "cfp_link" not in result[0]
+        assert "sponsor" not in result[0]
+        assert "finaid" not in result[0]
+
+    def test_anchor_on_homepage_kept(self):
+        """A #anchor is a different pointer (e.g. PyCon SG's sponsor link)."""
+        data = [
+            {
+                "conference": "PyCon Singapore",
+                "year": 2026,
+                "link": "http://pycon.sg/",
+                "sponsor": "http://pycon.sg/index.html#sponsors",
+            },
+        ]
+        result = drop_redundant_link_fields(data)
+        assert result[0]["sponsor"] == "http://pycon.sg/index.html#sponsors"
+
+    def test_sub_page_and_subdomain_kept(self):
+        data = [
+            {
+                "conference": "PyCon Africa",
+                "year": 2026,
+                "link": "https://africa.pycon.org/",
+                "cfp_link": "https://africa.pycon.org/2026/talks/proposals/",
+                "sponsor": "https://africa.pycon.org/2026/sponsor-us/",
+                "finaid": "https://africa.pycon.org/2026/opportunity-grants/",
+            },
+            {
+                "conference": "Sub Conf",
+                "year": 2026,
+                "link": "https://sub.conf/",
+                "cfp_link": "https://cfp.sub.conf/",
+            },
+        ]
+        result = drop_redundant_link_fields(data)
+        assert result[0]["cfp_link"] == "https://africa.pycon.org/2026/talks/proposals/"
+        assert result[0]["sponsor"] == "https://africa.pycon.org/2026/sponsor-us/"
+        assert result[0]["finaid"] == "https://africa.pycon.org/2026/opportunity-grants/"
+        assert result[1]["cfp_link"] == "https://cfp.sub.conf/"
+
+    def test_entry_without_link_untouched(self):
+        data = [{"conference": "No Link Conf", "year": 2026, "cfp_link": "https://example.com/"}]
+        result = drop_redundant_link_fields(data)
+        assert result[0]["cfp_link"] == "https://example.com/"
+
+    def test_entry_without_sub_fields_untouched(self):
+        data = [{"conference": "Plain Conf", "year": 2026, "link": "https://plain.conf/"}]
+        result = drop_redundant_link_fields(data)
+        assert result[0] == {"conference": "Plain Conf", "year": 2026, "link": "https://plain.conf/"}

tests/test_youtube_extraction.py

@@ -75,6 +75,29 @@ def test_generic_mastodon_still_works(self, mock_links):
         assert "mastodon" in result
         assert "youtube" not in result
 
+    @patch("enrich_tba.get_all_links")
+    def test_google_maps_viewport_not_mastodon(self, mock_links):
+        """Google Maps /@lat,lng,zoom viewport URLs must not be detected as mastodon.
+
+        Regression test: a venue map link like
+        https://www.google.com/maps/place/Transformatorhuis/@52.386807,4.8698442,17z
+        contains "/@" but is not a Mastodon profile.
+        """
+        mock_links.return_value = [
+            "https://www.google.com/maps/place/Transformatorhuis/@52.386807,4.8698442,17z",
+        ]
+        result = extract_links_from_url("https://fastapiconf.com")
+        assert "mastodon" not in result
+
+    @patch("enrich_tba.get_all_links")
+    def test_user_at_instance_profile_still_works(self, mock_links):
+        """Full /@user@instance profile paths are still detected as mastodon."""
+        mock_links.return_value = [
+            "https://social.example.org/@pyconf@mastodon.social",
+        ]
+        result = extract_links_from_url("https://pyconf.org")
+        assert result.get("mastodon") == "https://social.example.org/@pyconf@mastodon.social"
+
     @patch("enrich_tba.get_all_links")
     def test_youtube_first_seen_wins(self, mock_links):
         """Only the first YouTube link is kept."""

utils/check_data_loss.py

@@ -0,0 +1,90 @@
+"""Guard against silent conference loss during automated data merges.
+
+Compares snapshots of the conference data files taken before a merge with the
+files after the merge. Every (conference, year) pair that existed before must
+still exist afterwards - in any of the given files, since conferences may
+legitimately move between conferences.yml, archive.yml and legacy.yml.
+
+Exits non-zero if any conference disappeared, so CI can block the commit.
+A legitimate rename (via titles.yml mappings) will also trip this check;
+that is intentional - renames of existing entries should be reviewed by a
+human, not auto-committed.
+"""
+
+import argparse
+import sys
+from pathlib import Path
+
+import yaml
+
+
+def load_conference_keys(paths: list[str]) -> set[tuple[str, int | str]]:
+    """Collect (conference, year) pairs from a list of YAML data files.
+
+    Missing files are skipped, so the same invocation works whether or not
+    optional files like legacy.yml exist.
+    """
+    keys = set()
+    for path in paths:
+        file = Path(path)
+        if not file.exists():
+            continue
+        with file.open(encoding="utf-8") as f:
+            data = yaml.safe_load(f)
+        if not data:
+            continue
+        for entry in data:
+            if not isinstance(entry, dict):
+                continue
+            conference = entry.get("conference")
+            year = entry.get("year")
+            if conference:
+                keys.add((str(conference).strip(), year))
+    return keys
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Fail if conferences disappeared from the data files.",
+    )
+    parser.add_argument(
+        "--before",
+        nargs="+",
+        required=True,
+        help="Data files snapshotted before the merge",
+    )
+    parser.add_argument(
+        "--after",
+        nargs="+",
+        required=True,
+        help="Data files after the merge",
+    )
+    args = parser.parse_args()
+
+    before = load_conference_keys(args.before)
+    after = load_conference_keys(args.after)
+    missing = before - after
+
+    if missing:
+        print(
+            f"ERROR: {len(missing)} conference(s) disappeared during the merge:",
+            file=sys.stderr,
+        )
+        for conference, year in sorted(missing, key=str):
+            print(f"  - {conference} ({year})", file=sys.stderr)
+        print(
+            "\nAn automated merge must never delete or rename existing conferences.\n"
+            "This usually means two distinct conferences were fuzzy-matched into one\n"
+            "(e.g. 'PyCon Africa' vs 'PyCon South Africa'). Add the pair to\n"
+            "utils/tidy_conf/data/rejections.yml, or if the rename is intentional,\n"
+            "apply it manually.",
+            file=sys.stderr,
+        )
+        return 1
+
+    print(f"OK: all {len(before)} conference entries survived the merge.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())