diff --git a/impl/c-qube/ingest/config.json b/impl/c-qube/ingest/config.json index 68752ca0..303f38ec 100644 --- a/impl/c-qube/ingest/config.json +++ b/impl/c-qube/ingest/config.json @@ -1,6 +1,7 @@ { "globals": { - "onlyCreateWhitelisted": true + "onlyCreateWhitelisted": true, + "quoteChar": "`" }, "dimensions": { "namespace": "dimensions", diff --git a/impl/c-qube/src/services/csv-adapter/parser/utils/csvcleaner.spec.ts b/impl/c-qube/src/services/csv-adapter/parser/utils/csvcleaner.spec.ts index 9031c026..801bfdbc 100644 --- a/impl/c-qube/src/services/csv-adapter/parser/utils/csvcleaner.spec.ts +++ b/impl/c-qube/src/services/csv-adapter/parser/utils/csvcleaner.spec.ts @@ -28,8 +28,9 @@ describe('remove empty lines', () => { 'test/fixtures/test-csvs/csvcleaner/withEmpty.csv', 'test/fixtures/test-csvs/csvcleaner/withoutEmpty.csv', 'test/fixtures/test-csvs/csvreader/invalid.reader.csv', + 'test/fixtures/test-csvs/csvreader/quote.csv', 'test/fixtures/test-csvs/csvreader/valid.reader.csv', 'test/fixtures/test-csvs/event-grammars/test-dimension.grammar.csv', - ]); + ]); }); }); diff --git a/impl/c-qube/src/services/csv-adapter/parser/utils/csvreader.spec.ts b/impl/c-qube/src/services/csv-adapter/parser/utils/csvreader.spec.ts index cf702b4f..654653df 100644 --- a/impl/c-qube/src/services/csv-adapter/parser/utils/csvreader.spec.ts +++ b/impl/c-qube/src/services/csv-adapter/parser/utils/csvreader.spec.ts @@ -1,4 +1,7 @@ -import { readCSV, readCSVFile } from './csvreader'; +import { readCSV, readCSVFile, getquoteChar } from './csvreader'; +const fs1 = require('fs'); + + describe('CSVReader', () => { test('parse the file fine', async () => { @@ -37,4 +40,34 @@ describe('CSVReader', () => { test('benchmarking', () => { // add a benchmark test here similar to date parser }); -}); + + test('should return the quote character from the config file', () => { + jest.spyOn(fs1, 'readFileSync').mockReturnValue('{"globals": {"quoteChar": "`"}}'); + const configPath ='ingest/config.json'; + const quoteChar = getquoteChar(configPath); + expect(quoteChar).toBe('`'); + }); + + it('should take quote value from config , incase both are defined ', async () => { + let rows = await readCSV('./test/fixtures/test-csvs/csvreader/quote.csv', 'ingest/config.json', '`'); + console.log(rows) + expect(rows).toEqual([ + [ 'Name', 'Age', 'Country' ], + [ 'John Doe', '30', 'USA' ], + [ 'Jane Smith', '25', 'Canada' ], + [ 'Backtick User', '40', 'Australia' ] + ]); + }); + + it('should take quote value from config , incase one is defined ', async () => { + let rows = await readCSV('./test/fixtures/test-csvs/csvreader/quote.csv', 'ingest/config.json', undefined); + console.log(rows) + expect(rows).toEqual([ + [ 'Name', 'Age', 'Country' ], + [ 'John Doe', '30', 'USA' ], + [ 'Jane Smith', '25', 'Canada' ], + [ 'Backtick User', '40', 'Australia' ] + ]); + }); + +}); \ No newline at end of file diff --git a/impl/c-qube/src/services/csv-adapter/parser/utils/csvreader.ts b/impl/c-qube/src/services/csv-adapter/parser/utils/csvreader.ts index d3ae2962..ebbf65e4 100644 --- a/impl/c-qube/src/services/csv-adapter/parser/utils/csvreader.ts +++ b/impl/c-qube/src/services/csv-adapter/parser/utils/csvreader.ts @@ -3,13 +3,20 @@ const fs = require('fs').promises; import * as csv from 'csv-parser'; -export async function readCSV(filePath: string): Promise { +export async function readCSV(filePath: string, configPath?: string, quoteChar?: string): Promise { return new Promise((resolve, reject) => { - const rows: string[][] = []; + const rows: string[][] = configPath ? [] : [[]]; + if (configPath) { + const configquote = getquoteChar(configPath); + quoteChar = configquote || quoteChar || "'"; + } else { + quoteChar = quoteChar || "'"; + } + // TODO: Add checking here fs1 .createReadStream(filePath) - .pipe(csv({ separator: ',', headers: false, quote: "'" })) + .pipe(csv({ separator: ',', headers: false, quote: quoteChar })) .on('data', (data) => { rows.push(Object.values(data)); }) @@ -30,3 +37,13 @@ export async function readCSVFile(filePath: string): Promise { .map((row: string) => row.trim()) .filter((row: string) => row !== ''); } + +export function getquoteChar (configPath: string): string | undefined { + try { + const configContent = fs1.readFileSync(configPath, 'utf-8'); + const config = JSON.parse(configContent); + return config.globals.quoteChar; + } catch (error) { + return undefined; + } +} \ No newline at end of file diff --git a/impl/c-qube/test/fixtures/test-csvs/csvreader/quote.csv b/impl/c-qube/test/fixtures/test-csvs/csvreader/quote.csv new file mode 100644 index 00000000..2f356fc3 --- /dev/null +++ b/impl/c-qube/test/fixtures/test-csvs/csvreader/quote.csv @@ -0,0 +1,4 @@ +Name,Age,Country +John Doe,30,USA +Jane Smith,25,Canada +`Backtick User`,40,Australia \ No newline at end of file