Convert lists and dicts to str in parquet

plaidcloud/utilities/query.py

@@ -5,6 +5,7 @@
 import uuid
 import unicodecsv as csv
 
+import pyarrow as pa
 import pandas as pd
 import numpy as np
 import requests
@@ -21,7 +22,7 @@
 from plaidcloud.utilities.remote.dimension import Dimensions
 
 __author__ = 'Paul Morel'
-__copyright__ = 'Copyright 2010-2021, Tartan Solutions, Inc'
+__copyright__ = 'Copyright 2010-2024, Tartan Solutions, Inc'
 __credits__ = ['Paul Morel']
 __license__ = 'Apache 2.0'
 __maintainer__ = 'Paul Morel'
@@ -496,8 +497,12 @@ def bulk_insert_dataframe(self, table_object, df, append=False, chunk_size=50000
             load_type='parquet',
         )
         if data_load:
+            schema = pa.Schema.from_pandas(df)
+            for index, col in enumerate(schema):
+                if isinstance(col.type, pa.ListType, pa.StructType):
+                    schema = schema.set(index, col.with_type(pa.string()))
             with tempfile.NamedTemporaryFile(mode='wb+') as pq_file:
-                df.to_parquet(pq_file)
+                df.to_parquet(pq_file, schema=schema)
                 # upload the file
                 pq_file.seek(0)
                 self._upload(data_load['load_type'], data_load['upload_path'], pq_file)