feat (consistency): check for duplicated lines

chaen · Jul 26, 2024 · 12376a8 · 12376a8
1 parent 5d60afe
commit 12376a8
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/consistency_check/consistency.py b/consistency_check/consistency.py
@@ -27,6 +27,8 @@ def load_se_dump(se_dump_path):
     se_dump = pd.read_csv(se_dump_path, names=["pfn", "se_cks"], delimiter="|", index_col="pfn")
     se_dump["se_cks"] = se_dump["se_cks"].str.lower().str.pad(8, fillchar="0")
     se_dump["version"] = "se_dump"
+    assert not se_dump.index.duplicated().any(), f"Duplicated entries in SE dump {se_dump[se_dump.index.duplicated()]}"
+
     return se_dump