Implementation - QuoteChar

impl/c-qube/ingest/config.json

@@ -1,6 +1,7 @@
 {
   "globals": {
-    "onlyCreateWhitelisted": true
+    "onlyCreateWhitelisted": true,
+    "quoteChar": "`"
   },
   "dimensions": {
     "namespace": "dimensions",

impl/c-qube/src/services/csv-adapter/parser/utils/csvcleaner.spec.ts

@@ -28,8 +28,9 @@ describe('remove empty lines', () => {
       'test/fixtures/test-csvs/csvcleaner/withEmpty.csv',
       'test/fixtures/test-csvs/csvcleaner/withoutEmpty.csv',
       'test/fixtures/test-csvs/csvreader/invalid.reader.csv',
+      'test/fixtures/test-csvs/csvreader/quote.csv',
       'test/fixtures/test-csvs/csvreader/valid.reader.csv',
       'test/fixtures/test-csvs/event-grammars/test-dimension.grammar.csv',
-    ]);
+    ]);    
   });
 });

impl/c-qube/src/services/csv-adapter/parser/utils/csvreader.spec.ts

@@ -1,4 +1,7 @@
-import { readCSV, readCSVFile } from './csvreader';
+import { readCSV, readCSVFile, getquoteChar } from './csvreader';
+const fs1 = require('fs'); 
+
+
 
 describe('CSVReader', () => {
   test('parse the file fine', async () => {
@@ -37,4 +40,34 @@ describe('CSVReader', () => {
   test('benchmarking', () => {
     // add a benchmark test here similar to date parser
   });
-});
+
+  test('should return the quote character from the config file', () => {
+    jest.spyOn(fs1, 'readFileSync').mockReturnValue('{"globals": {"quoteChar": "`"}}');
+    const configPath ='ingest/config.json';
+    const quoteChar = getquoteChar(configPath);
+    expect(quoteChar).toBe('`');
+  });
+
+  it('should take quote value from config , incase both are defined ', async () => {
+    let rows = await readCSV('./test/fixtures/test-csvs/csvreader/quote.csv', 'ingest/config.json', '`');
+    console.log(rows)
+    expect(rows).toEqual([
+      [ 'Name', 'Age', 'Country' ],
+      [ 'John Doe', '30', 'USA' ],
+      [ 'Jane Smith', '25', 'Canada' ],
+      [ 'Backtick User', '40', 'Australia' ]
+    ]);
+  });
+
+  it('should take quote value from config , incase one is defined ', async () => {
+    let rows = await readCSV('./test/fixtures/test-csvs/csvreader/quote.csv', 'ingest/config.json', undefined);
+    console.log(rows)
+    expect(rows).toEqual([
+      [ 'Name', 'Age', 'Country' ],
+      [ 'John Doe', '30', 'USA' ],
+      [ 'Jane Smith', '25', 'Canada' ],
+      [ 'Backtick User', '40', 'Australia' ]
+    ]);
+  });
+
+});

impl/c-qube/src/services/csv-adapter/parser/utils/csvreader.ts

@@ -3,13 +3,20 @@ const fs = require('fs').promises;
 
 import * as csv from 'csv-parser';
 
-export async function readCSV(filePath: string): Promise<string[][]> {
+export async function readCSV(filePath: string, configPath?: string, quoteChar?: string): Promise<string[][]> {
   return new Promise((resolve, reject) => {
-    const rows: string[][] = [];
+    const rows: string[][] = configPath ? [] : [[]]; 
+    if (configPath) {
+      const configquote = getquoteChar(configPath);
+      quoteChar = configquote || quoteChar || "'";
+    } else {
+      quoteChar = quoteChar || "'";
+    }
+
     // TODO: Add checking here
     fs1
       .createReadStream(filePath)
-      .pipe(csv({ separator: ',', headers: false, quote: "'" }))
+      .pipe(csv({ separator: ',', headers: false, quote: quoteChar }))
       .on('data', (data) => {
         rows.push(Object.values(data));
       })
@@ -30,3 +37,13 @@ export async function readCSVFile(filePath: string): Promise<string[]> {
     .map((row: string) => row.trim())
     .filter((row: string) => row !== '');
 }
+
+export function getquoteChar (configPath: string): string | undefined {
+  try {
+    const configContent = fs1.readFileSync(configPath, 'utf-8');
+    const config = JSON.parse(configContent);
+    return config.globals.quoteChar;
+  } catch (error) {
+    return undefined;
+  }
+}

impl/c-qube/test/fixtures/test-csvs/csvreader/quote.csv

@@ -0,0 +1,4 @@
+Name,Age,Country
+John Doe,30,USA
+Jane Smith,25,Canada
+`Backtick User`,40,Australia