feat: Start using kaitai for first level decoding

acd/comps.py

@@ -1,7 +1,10 @@
 import struct
 from dataclasses import dataclass
+from io import BytesIO
 from sqlite3 import Cursor
 
+from kaitaistruct import KaitaiStream
+
 from acd.dbextract import DatRecord
 from acd.generated.comps.fafa_comps import FafaComps
 from acd.generated.comps.fdfd_comps import FdfdComps
@@ -24,34 +27,14 @@ class CompsRecord:
     def __post_init__(self):
 
         if self.dat_record.identifier == 64250:
-            fafa_comps = FafaComps.from_bytes(self.dat_record.record.record_buffer)
-            self.seq_number = fafa_comps.header.seq_number
-            self.record_type = fafa_comps.header.record_type
-            self.object_id = fafa_comps.header.object_id
-            self.parent_id = fafa_comps.header.parent_id
-            self.text = fafa_comps.header.record_name
-
-            query: str = "INSERT INTO comps VALUES (?, ?, ?, ?, ?, ?)"
-            enty: tuple = (self.object_id, self.parent_id, self.text, self.seq_number, self.record_type, fafa_comps.record_buffer)
-            self._cur.execute(query, enty)
-
-
+            r = FafaComps.from_bytes(self.dat_record.record.record_buffer)
         elif self.dat_record.identifier == 65021:
-            # Pointer to Data record
-            fdfd_comps = FdfdComps.from_bytes(self.dat_record.record.record_buffer)
-            self.seq_number = fafa_comps.header.seq_number
-            self.record_type = fafa_comps.header.record_type
-            self.object_id = fafa_comps.header.object_id
-            self.parent_id = fafa_comps.header.parent_id
-            self.text = fafa_comps.header.record_name
-
-            query: str = "INSERT INTO comps VALUES (?, ?, ?, ?, ?, ?)"
-            enty: tuple = (
-            self.object_id, self.parent_id, self.text, self.seq_number, self.record_type, fafa_comps.record_buffer)
-            self._cur.execute(query, enty)
-
+            r = FdfdComps(self.dat_record.record_length, KaitaiStream(BytesIO(self.dat_record.record.record_buffer)))
         else:
-            self.text = ""
-            self.object_id = -1
-
+            return
 
+        query: str = "INSERT INTO comps VALUES (?, ?, ?, ?, ?, ?)"
+        entry: tuple = (
+            r.header.object_id, r.header.parent_id, r.header.record_name, r.header.seq_number, r.header.record_type,
+            r.record_buffer)
+        self._cur.execute(query, entry)

acd/export_l5x.py

@@ -62,13 +62,13 @@ def __post_init__(self):
         self.populate_region_map()
 
         log.info("Getting records from ACD SbRegion file and storing in sqllite database")
-        sb_region_db = DbExtract(os.path.join(self._temp_dir, "SbRegion.Dat"))
-        for record in sb_region_db.records:
+        sb_region_db = DbExtract(os.path.join(self._temp_dir, "SbRegion.Dat")).read()
+        for record in sb_region_db.records.record:
             SbRegionRecord(self._cur, record)
         self._db.commit()
 
         log.info("Getting records from ACD Comments file and storing in sqllite database")
-        comments_db = DbExtract(os.path.join(self._temp_dir, "Comments.Dat"))
+        comments_db = DbExtract(os.path.join(self._temp_dir, "Comments.Dat")).read()
         for record in comments_db.records:
             CommentsRecord(self._cur, record)
         self._db.commit()

acd/generated/comps/fafa_comps.py

@@ -27,14 +27,62 @@ def __init__(self, _io, _parent=None, _root=None):
             self._read()
 
         def _read(self):
-            self.unknown_1 = self._io.read_u4le()
-            self.seq_number = self._io.read_u2le()
-            self.record_type = self._io.read_u2le()
-            self.unknown_4 = self._io.read_u2le()
-            self.unknown_5 = self._io.read_u2le()
-            self.object_id = self._io.read_u4le()
-            self.parent_id = self._io.read_u4le()
-            self.record_name = (self._io.read_bytes(124)).decode(u"UTF-16")
+            pass
+
+        @property
+        def record_type(self):
+            if hasattr(self, '_m_record_type'):
+                return self._m_record_type
+
+            _pos = self._io.pos()
+            self._io.seek(6)
+            self._m_record_type = self._io.read_u2le()
+            self._io.seek(_pos)
+            return getattr(self, '_m_record_type', None)
+
+        @property
+        def object_id(self):
+            if hasattr(self, '_m_object_id'):
+                return self._m_object_id
+
+            _pos = self._io.pos()
+            self._io.seek(16)
+            self._m_object_id = self._io.read_u4le()
+            self._io.seek(_pos)
+            return getattr(self, '_m_object_id', None)
+
+        @property
+        def record_name(self):
+            if hasattr(self, '_m_record_name'):
+                return self._m_record_name
+
+            _pos = self._io.pos()
+            self._io.seek(24)
+            self._m_record_name = (self._io.read_bytes(124)).decode(u"UTF-16")
+            self._io.seek(_pos)
+            return getattr(self, '_m_record_name', None)
+
+        @property
+        def seq_number(self):
+            if hasattr(self, '_m_seq_number'):
+                return self._m_seq_number
+
+            _pos = self._io.pos()
+            self._io.seek(4)
+            self._m_seq_number = self._io.read_u2le()
+            self._io.seek(_pos)
+            return getattr(self, '_m_seq_number', None)
+
+        @property
+        def parent_id(self):
+            if hasattr(self, '_m_parent_id'):
+                return self._m_parent_id
+
+            _pos = self._io.pos()
+            self._io.seek(20)
+            self._m_parent_id = self._io.read_u4le()
+            self._io.seek(_pos)
+            return getattr(self, '_m_parent_id', None)
 
 
 

acd/generated/comps/fdfd_comps.py

@@ -17,7 +17,7 @@ def __init__(self, record_length, _io, _parent=None, _root=None):
 
     def _read(self):
         self.header = FdfdComps.Header(self._io, self, self._root)
-        self.record_buffer = self._io.read_bytes(((self.record_length - 144) - 4))
+        self.record_buffer = self._io.read_bytes(((self.record_length - 155) - 4))
 
     class Header(KaitaiStruct):
         def __init__(self, _io, _parent=None, _root=None):
@@ -27,14 +27,62 @@ def __init__(self, _io, _parent=None, _root=None):
             self._read()
 
         def _read(self):
-            self.unknown_1 = self._io.read_u4le()
-            self.seq_number = self._io.read_u2le()
-            self.record_type = self._io.read_u2le()
-            self.unknown_4 = self._io.read_u2le()
-            self.unknown_5 = self._io.read_u2le()
-            self.object_id = self._io.read_u4le()
-            self.parent_id = self._io.read_u4le()
-            self.record_name = (self._io.read_bytes(124)).decode(u"UTF-16")
+            pass
+
+        @property
+        def record_type(self):
+            if hasattr(self, '_m_record_type'):
+                return self._m_record_type
+
+            _pos = self._io.pos()
+            self._io.seek(10)
+            self._m_record_type = self._io.read_u2le()
+            self._io.seek(_pos)
+            return getattr(self, '_m_record_type', None)
+
+        @property
+        def object_id(self):
+            if hasattr(self, '_m_object_id'):
+                return self._m_object_id
+
+            _pos = self._io.pos()
+            self._io.seek(16)
+            self._m_object_id = self._io.read_u4le()
+            self._io.seek(_pos)
+            return getattr(self, '_m_object_id', None)
+
+        @property
+        def record_name(self):
+            if hasattr(self, '_m_record_name'):
+                return self._m_record_name
+
+            _pos = self._io.pos()
+            self._io.seek(24)
+            self._m_record_name = (self._io.read_bytes(124)).decode(u"UTF-16")
+            self._io.seek(_pos)
+            return getattr(self, '_m_record_name', None)
+
+        @property
+        def seq_number(self):
+            if hasattr(self, '_m_seq_number'):
+                return self._m_seq_number
+
+            _pos = self._io.pos()
+            self._io.seek(4)
+            self._m_seq_number = self._io.read_u2le()
+            self._io.seek(_pos)
+            return getattr(self, '_m_seq_number', None)
+
+        @property
+        def parent_id(self):
+            if hasattr(self, '_m_parent_id'):
+                return self._m_parent_id
+
+            _pos = self._io.pos()
+            self._io.seek(20)
+            self._m_parent_id = self._io.read_u4le()
+            self._io.seek(_pos)
+            return getattr(self, '_m_parent_id', None)
 
 
 

database.py

@@ -0,0 +1,161 @@
+# This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild
+
+import kaitaistruct
+from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO
+
+
+if getattr(kaitaistruct, 'API_VERSION', (0, 9)) < (0, 9):
+    raise Exception("Incompatible Kaitai Struct Python API: 0.9 or later is required, but you have %s" % (kaitaistruct.__version__))
+
+class Database(KaitaiStruct):
+    def __init__(self, _io, _parent=None, _root=None):
+        self._io = _io
+        self._parent = _parent
+        self._root = _root if _root else self
+        self._read()
+
+    def _read(self):
+        self.header = Database.Header(self._io, self, self._root)
+        self._raw_records = self._io.read_bytes(((self.header.file_length - self.header.first_record_position) + 1))
+        _io__raw_records = KaitaiStream(BytesIO(self._raw_records))
+        self.records = Database.Records(_io__raw_records, self, self._root)
+
+    class FefeRecord(KaitaiStruct):
+        def __init__(self, _io, _parent=None, _root=None):
+            self._io = _io
+            self._parent = _parent
+            self._root = _root if _root else self
+            self._read()
+
+        def _read(self):
+            self.record_length = self._io.read_u4le()
+            self.blank_1 = self._io.read_u4le()
+            self.unknown_1 = self._io.read_u4le()
+            self.unknown_2 = self._io.read_u4le()
+            self.record_buffer = []
+            for i in range(self.record_length):
+                self.record_buffer.append(self._io.read_u1())
+
+
+
+    class Header(KaitaiStruct):
+        def __init__(self, _io, _parent=None, _root=None):
+            self._io = _io
+            self._parent = _parent
+            self._root = _root if _root else self
+            self._read()
+
+        def _read(self):
+            self.format_type = self._io.read_u4le()
+            self.blank_2 = self._io.read_u4le()
+            self.file_length = self._io.read_u4le()
+            self.first_record_position = self._io.read_u4le()
+            self.blank_3 = self._io.read_u4le()
+            self.number_records_fafa = self._io.read_u4le()
+            self.header_buffer = []
+            for i in range((self.first_record_position - 24)):
+                self.header_buffer.append(self._io.read_u1())
+
+
+
+    class Records(KaitaiStruct):
+        def __init__(self, _io, _parent=None, _root=None):
+            self._io = _io
+            self._parent = _parent
+            self._root = _root if _root else self
+            self._read()
+
+        def _read(self):
+            self.record = []
+            i = 0
+            while not self._io.is_eof():
+                self.record.append(Database.Record(self._io, self, self._root))
+                i += 1
+
+
+
+    class Record(KaitaiStruct):
+        def __init__(self, _io, _parent=None, _root=None):
+            self._io = _io
+            self._parent = _parent
+            self._root = _root if _root else self
+            self._read()
+
+        def _read(self):
+            self.record_header = self._io.read_u2le()
+            if not  ((self.record_header == 65278) or (self.record_header == 65021) or (self.record_header == 64250) or (self.record_header == 64447)) :
+                raise kaitaistruct.ValidationNotAnyOfError(self.record_header, self._io, u"/types/record/seq/0")
+            self.record_length = self._io.read_u4le()
+            _on = self.record_header
+            if _on == 64250:
+                self._raw_record_buffer = self._io.read_bytes((self.record_length - 6))
+                _io__raw_record_buffer = KaitaiStream(BytesIO(self._raw_record_buffer))
+                self.record_buffer = Database.FafaRecord(_io__raw_record_buffer, self, self._root)
+            elif _on == 65278:
+                self._raw_record_buffer = self._io.read_bytes((self.record_length - 6))
+                _io__raw_record_buffer = KaitaiStream(BytesIO(self._raw_record_buffer))
+                self.record_buffer = Database.FefeRecord(_io__raw_record_buffer, self, self._root)
+            else:
+                self.record_buffer = self._io.read_bytes((self.record_length - 6))
+
+
+    class FafaRecord(KaitaiStruct):
+        def __init__(self, _io, _parent=None, _root=None):
+            self._io = _io
+            self._parent = _parent
+            self._root = _root if _root else self
+            self._read()
+
+        def _read(self):
+            self.record_length = self._io.read_u4le()
+            self.record_buffer = []
+            for i in range(self.record_length):
+                self.record_buffer.append(self._io.read_u1())
+
+
+
+    @property
+    def third_array_dimension(self):
+        if hasattr(self, '_m_third_array_dimension'):
+            return self._m_third_array_dimension
+
+        _pos = self._io.pos()
+        self._io.seek(182)
+        self._m_third_array_dimension = self._io.read_u4le()
+        self._io.seek(_pos)
+        return getattr(self, '_m_third_array_dimension', None)
+
+    @property
+    def data_type_id(self):
+        if hasattr(self, '_m_data_type_id'):
+            return self._m_data_type_id
+
+        _pos = self._io.pos()
+        self._io.seek(190)
+        self._m_data_type_id = self._io.read_u4le()
+        self._io.seek(_pos)
+        return getattr(self, '_m_data_type_id', None)
+
+    @property
+    def tag_name_length(self):
+        if hasattr(self, '_m_tag_name_length'):
+            return self._m_tag_name_length
+
+        _pos = self._io.pos()
+        self._io.seek(238)
+        self._m_tag_name_length = self._io.read_u2le()
+        self._io.seek(_pos)
+        return getattr(self, '_m_tag_name_length', None)
+
+    @property
+    def tag_name(self):
+        if hasattr(self, '_m_tag_name'):
+            return self._m_tag_name
+
+        _pos = self._io.pos()
+        self._io.seek(240)
+        self._m_tag_name = (self._io.read_bytes(self.tag_name_length)).decode(u"UTF-8")
+        self._io.seek(_pos)
+        return getattr(self, '_m_tag_name', None)
+
+