Merge pull request #187 from opensanctions/pudo/remove-target

Remove `target` from statement model

nomenklatura/entity.py

@@ -122,14 +122,6 @@ def last_seen(self) -> Optional[str]:
         seen = (s.last_seen for s in self._iter_stmt() if s.last_seen is not None)
         return max(seen, default=None)
 
-    @property
-    def target(self) -> Optional[bool]:
-        target: Optional[bool] = None
-        for stmt in self._iter_stmt():
-            if stmt.target is not None:
-                target = target or stmt.target
-        return target
-
     @property
     def datasets(self) -> Set[str]:
         datasets: Set[str] = set()
@@ -163,7 +155,6 @@ def caption(self) -> str:
         return self._caption
 
     def add_statement(self, stmt: Statement) -> None:
-        # TODO: change target, schema etc. based on data
         if not self.schema.is_a(stmt.schema):
             try:
                 self.schema = model.common_schema(self.schema, stmt.schema)

nomenklatura/statement/db.py

@@ -31,7 +31,6 @@ def make_statement_table(
         Column("original_value", Unicode(VALUE_LEN), nullable=True),
         Column("dataset", Unicode(KEY_LEN), index=True),
         Column("lang", Unicode(KEY_LEN), nullable=True),
-        Column("target", Boolean, default=False, nullable=False),
         Column("external", Boolean, default=False, nullable=False),
         Column("first_seen", DateTime, nullable=True),
         Column("last_seen", DateTime, nullable=True),

nomenklatura/statement/serialize.py

@@ -27,7 +27,6 @@
     "dataset",
     "lang",
     "original_value",
-    "target",
     "external",
     "first_seen",
     "last_seen",
@@ -41,7 +40,7 @@
     "dataset",
     "lang",
     "original_value",
-    "target",
+    "_",
     "external",
     "first_seen",
     "last_seen",
@@ -191,9 +190,6 @@ def __init__(self, fh: TextIO) -> None:
     def write(self, stmt: S) -> None:
         # HACK: This is very similar to the CSV writer, but at the very inner
         # loop of the application, so we're duplicating code here.
-        target_value: Optional[str] = "t" if stmt.target else "f"
-        if stmt.target is None:
-            target_value = None
         external_value: Optional[str] = "t" if stmt.external else "f"
         if stmt.external is None:
             external_value = None
@@ -204,7 +200,7 @@ def write(self, stmt: S) -> None:
             stmt.dataset,
             stmt.lang,
             stmt.original_value,
-            target_value,
+            None,
             external_value,
             stmt.first_seen,
             stmt.last_seen,

nomenklatura/statement/statement.py

@@ -26,7 +26,6 @@ class StatementDict(TypedDict):
     dataset: str
     lang: Optional[str]
     original_value: Optional[str]
-    target: Optional[bool]
     external: Optional[bool]
     first_seen: Optional[str]
     last_seen: Optional[str]
@@ -56,7 +55,6 @@ class Statement(object):
         "dataset",
         "lang",
         "original_value",
-        "target",
         "external",
         "first_seen",
         "last_seen",
@@ -89,7 +87,6 @@ def __init__(
         self.original_value = original_value
         self.first_seen = first_seen
         self.last_seen = last_seen or first_seen
-        self.target = target
         self.external = external
         if id is None:
             id = self.generate_key()
@@ -108,14 +105,12 @@ def to_dict(self) -> StatementDict:
             "original_value": self.original_value,
             "first_seen": self.first_seen,
             "last_seen": self.last_seen,
-            "target": self.target,
             "external": self.external,
             "id": self.id,
         }
 
     def to_csv_row(self) -> Dict[str, Optional[str]]:
         data = cast(Dict[str, Optional[str]], self.to_dict())
-        data["target"] = bool_text(self.target)
         data["external"] = bool_text(self.external)
         return data
 
@@ -188,7 +183,6 @@ def from_dict(cls: Type[S], data: StatementDict) -> S:
             lang=data.get("lang", None),
             original_value=data.get("original_value", None),
             first_seen=data.get("first_seen", None),
-            target=data.get("target"),
             external=data.get("external"),
             id=data.get("id", None),
             canonical_id=data.get("canonical_id", None),
@@ -198,7 +192,6 @@ def from_dict(cls: Type[S], data: StatementDict) -> S:
     @classmethod
     def from_row(cls: Type[S], data: Dict[str, str]) -> S:
         typed_data = cast(StatementDict, data)
-        typed_data["target"] = text_bool(data.get("target"))
         typed_data["external"] = text_bool(data.get("external"))
         if data.get("lang") == "":
             typed_data["lang"] = None
@@ -219,7 +212,6 @@ def from_db_row(cls: Type[S], row: Row) -> S:
             lang=row.lang,
             original_value=row.original_value,
             first_seen=datetime_iso(row.first_seen),
-            target=row.target,
             external=row.external,
             last_seen=datetime_iso(row.last_seen),
         )
@@ -231,7 +223,6 @@ def from_entity(
         dataset: str,
         first_seen: Optional[str] = None,
         last_seen: Optional[str] = None,
-        target: Optional[bool] = None,
         external: Optional[bool] = None,
     ) -> Generator[S, None, None]:
         if entity.id is None:
@@ -242,7 +233,6 @@ def from_entity(
             schema=entity.schema.name,
             value=entity.id,
             dataset=dataset,
-            target=target,
             external=external,
             first_seen=first_seen,
             last_seen=last_seen,
@@ -254,7 +244,6 @@ def from_entity(
                 schema=entity.schema.name,
                 value=value,
                 dataset=dataset,
-                target=target,
                 external=external,
                 first_seen=first_seen,
                 last_seen=last_seen,

nomenklatura/store/level.py

@@ -30,7 +30,6 @@ def pack_statement(stmt: Statement) -> bytes:
         stmt.original_value or 0,
         stmt.first_seen,
         # stmt.last_seen,
-        1 if stmt.target else 0,
     )
     return orjson.dumps(values)
 
@@ -46,7 +45,6 @@ def unpack_statement(data: bytes, canonical_id: str, external: bool) -> Statemen
         original_value,
         first_seen,
         # last_seen,
-        target,
     ) = orjson.loads(data)
     schema, _, prop = unpack_prop(prop_id)
     return Statement(
@@ -60,7 +58,6 @@ def unpack_statement(data: bytes, canonical_id: str, external: bool) -> Statemen
         original_value=None if original_value == 0 else original_value,
         first_seen=first_seen,
         # last_seen=last_seen,
-        target=target == 1,
         canonical_id=canonical_id,
         external=external,
     )

nomenklatura/store/resolved.py

@@ -100,7 +100,6 @@ def flush(self) -> None:
                     stmt.dataset,
                     stmt.lang,
                     stmt.original_value,
-                    stmt.target,
                     # stmt.external,
                     stmt.first_seen,
                     stmt.last_seen,
@@ -151,7 +150,6 @@ def _unpack(self, data: bytes) -> CE:
                 dataset,
                 lang,
                 original_value,
-                target,
                 # external,
                 first_seen,
                 last_seen,
@@ -167,7 +165,6 @@ def _unpack(self, data: bytes) -> CE:
                     dataset=dataset,
                     lang=lang,
                     original_value=original_value,
-                    target=target,
                     external=False,
                     first_seen=first_seen,
                     last_seen=last_seen,

nomenklatura/store/sql.py

@@ -95,7 +95,6 @@ def _upsert_batch(self) -> None:
                 canonical_id=istmt.excluded.canonical_id,
                 schema=istmt.excluded.schema,
                 prop_type=istmt.excluded.prop_type,
-                target=istmt.excluded.target,
                 lang=istmt.excluded.lang,
                 original_value=istmt.excluded.original_value,
                 last_seen=istmt.excluded.last_seen,

nomenklatura/store/util.py

@@ -15,7 +15,6 @@ def pack_statement(stmt: Statement) -> bytes:
         stmt.original_value,
         stmt.first_seen,
         stmt.last_seen,
-        stmt.target,
     )
     return orjson.dumps(values)
 
@@ -31,7 +30,6 @@ def unpack_statement(data: bytes, canonical_id: str, external: bool) -> Statemen
         original_value,
         first_seen,
         last_seen,
-        target,
     ) = orjson.loads(data)
     schema, _, prop = unpack_prop(prop_id)
     return Statement(
@@ -45,7 +43,6 @@ def unpack_statement(data: bytes, canonical_id: str, external: bool) -> Statemen
         original_value=original_value,
         first_seen=first_seen,
         last_seen=last_seen,
-        target=target,
         canonical_id=canonical_id,
         external=external,
     )

nomenklatura/store/versioned.py

@@ -28,7 +28,6 @@ def _pack_statement(stmt: Statement) -> bytes:
         stmt.original_value or 0,
         stmt.first_seen,
         stmt.last_seen,
-        1 if stmt.target else 0,
         1 if stmt.external else 0,
     )
     return orjson.dumps(values)
@@ -45,7 +44,6 @@ def _unpack_statement(data: bytes, canonical_id: Optional[str] = None) -> Statem
         original_value,
         first_seen,
         last_seen,
-        target,
         external,
     ) = orjson.loads(data)
     schema, _, prop = unpack_prop(prop_id)
@@ -60,7 +58,6 @@ def _unpack_statement(data: bytes, canonical_id: Optional[str] = None) -> Statem
         original_value=None if original_value == 0 else original_value,
         first_seen=first_seen,
         last_seen=last_seen,
-        target=target == 1,
         canonical_id=canonical_id or entity_id,
         external=external == 1,
     )