Update main_test.py

src/main_test.py

@@ -1,9 +1,42 @@
-import defusedxml.ElementTree as ET
+"""
+Perform total record count test between input file and database
+"""
 
-import main
 
+import xml.etree.ElementTree as ElemTree
+import sqlite3
 
-def test_to_sql_over_example_data():
-    example_data = '''<?xml version="1.0" encoding="utf-8"?>
-<row Id="1" PostTypeId="1" AcceptedAnswerId="3" CreationDate="2016-08-02T15:39:14.947" Score="10" ViewCount="607" Body="&lt;p&gt;What does &quot;backprop&quot; mean? Is the &quot;backprop&quot; term basically the same as &quot;backpropagation&quot; or does it have a different meaning?&lt;/p&gt;&#xA;" OwnerUserId="8" LastEditorUserId="2444" LastEditDate="2019-11-16T17:56:22.093" LastActivityDate="2021-07-08T10:45:23.250" Title="What is &quot;backprop&quot;?" Tags="&lt;neural-networks&gt;&lt;backpropagation&gt;&lt;terminology&gt;&lt;definitions&gt;" AnswerCount="5" CommentCount="0" FavoriteCount="1" ContentLicense="CC BY-SA 4.0" />'''
-    assert main.to_sql(ET.fromstring(example_data)) == ('1', '1', '2016-08-02T15:39:14.947')
+FINAL_SCHEMA = []
+OUT_DATA_PATH = 'xml_in_out/data_warehouse/'
+IN_FILE_PATH = 'xml_in_out/stackexchange_dataset/'
+ABS_DB_PATH = OUT_DATA_PATH + 'large_dataset.db'
+TABLE_LIST = ['staging_posts', 'staging_tags']
+FILE_LIST = ['Posts.xml', 'Tags.xml']
+
+
+
+def test_results(file_name, table_name):
+    """
+    test method for result validation
+    :return:
+    """
+
+    with sqlite3.connect(ABS_DB_PATH) as connection:
+        db_cursor = connection.cursor()
+    db_cursor.execute(f"select count(*) from {table_name}")
+    rows = db_cursor.fetchone()
+
+    total_file_name = IN_FILE_PATH + file_name
+    xml_data = ElemTree.iterparse(total_file_name)
+    count = 0
+    for event, data in xml_data:
+        if list(data.attrib.keys()):
+            count = count + 1
+
+    debug_text = f"Rows inserted: {rows[0]} and actual: {count}"
+    print(debug_text)
+    assert count == rows[0], 'Number of rows inserted do not match'
+
+
+for iter_idx, table in enumerate(TABLE_LIST):
+    test_results(FILE_LIST[iter_idx], TABLE_LIST[iter_idx])