Feature/cde 40

demo/integration.js

@@ -1,4 +1,6 @@
-import {init} from './cde-mapper.js';
+
+import {init, mapElasticSearchHitsToOptions} from '../lib/main.tsx';
+import {getQueryObject} from "./query.js";
 
 export function mapAndInit(datasetMappingFile, additionalDatasetMappingsFiles, datasetFile) {
     let datasetMappings = [];
@@ -64,15 +66,15 @@ export function mapAndInit(datasetMappingFile, additionalDatasetMappingsFiles, d
                         datasetMapping: datasetMappings,
                         additionalDatasetMappings: additionalDatasetMappings,
                         datasetSample: datasetSample,
-                        collections: [],
+                        collections: getCollections(),
                         config: {width: '60%', height: '80%'},
                         name: 'TestLabName',
                         callback: (cdeFileMapping) => console.log(cdeFileMapping),
-                        headerMapping: {
-                            variableNameIndex: 0,
-                            preciseAbbreviationIndex: 1,
-                            titleIndex: 2,
-                            interlexIdIndex: 11
+                        headerIndexes: {
+                            variableName: 0,
+                            preciseAbbreviation: 1,
+                            title: 2,
+                            interlexId: 11
                         }
                     });
                 },
@@ -95,4 +97,30 @@ export function mapAndInit(datasetMappingFile, additionalDatasetMappingsFiles, d
     };
 
     startProcessing();
-}
+}
+
+function getCollections() {
+    return [
+        {
+            id: 'global',
+            name: "Global",
+            fetch: fetchElasticSearchData,
+        }
+    ]
+}
+
+async function fetchElasticSearchData(queryString) {
+    const query = getQueryObject(queryString)
+    const response = await fetch('/api', {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+            ...query
+        }),
+    });
+
+    const data = await response.json();
+    return mapElasticSearchHitsToOptions(data.hits.hits || [])
+}

demo/query.js

@@ -0,0 +1,78 @@
+export const getQueryObject = (queryString) => {
+    if (queryString) {
+        return {
+            "size": 20,
+            "from": 0,
+            "query": {
+                "bool": {
+                    "must": [
+                        {
+                            "query_string": {
+                                "fields": [
+                                    "label"
+                                ],
+                                "query": `${queryString}~`,
+                                "type": "cross_fields",
+                                "default_operator": "and",
+                                "lenient": "true",
+                                "fuzziness": 2,
+                                "fuzzy_max_expansions": 50,
+                                "fuzzy_prefix_length": 0,
+                                "fuzzy_transpositions": "true"
+                            }
+                        }
+                    ],
+                    "should": [
+                        {
+                            "match": {
+                                "label": {
+                                    "query": `"${queryString}"`,
+                                    "boost": 20
+                                }
+                            }
+                        },
+                        {
+                            "term": {
+                                "label.aggregate": {
+                                    "term": `${queryString}`,
+                                    "boost": 2000
+                                }
+                            }
+                        }
+                    ],
+                    "filter": [
+                        {
+                            "terms": {
+                                "type.aggregate": [
+                                    "cde"
+                                ]
+                            }
+                        }
+                    ]
+                }
+            },
+            "aggregations": {}
+        }
+    }
+
+    return {
+        "size": 20,
+        "from": 0,
+        "query": {
+            "bool": {
+                "filter": [
+                    {
+                        "terms": {
+                            "type.aggregate": [
+                                "cde"
+                            ]
+                        }
+                    }
+                ]
+            }
+        }
+    }
+
+};
+
+

lib/CdeContext.ts

@@ -2,8 +2,8 @@ import {createContext, useContext} from "react";
 import {
     Collection,
     Config,
-    DatasetMapping, Entity,
-    HeaderMapping,
+    DatasetMapping,
+    HeaderIndexes, OptionDetail,
     Suggestions
 } from "./models.ts";
 import {ABBREVIATION_INDEX, INTERLEX_ID_INDEX, TITLE_INDEX, VARIABLE_NAME_INDEX} from "./settings.ts";
@@ -14,10 +14,10 @@ export const CdeContext = createContext<{
     datasetSample: string[][];
     datasetMapping: DatasetMapping;
     datasetMappingHeader: string[];
-    handleUpdateDatasetMappingRow: (key: string, newData: Entity) => void;
+    handleUpdateDatasetMappingRow: (key: string, newData: OptionDetail[]) => void;
     getSuggestions: () => Suggestions;
-    headerMapping: HeaderMapping;
-    collections: Collection[];
+    headerIndexes: HeaderIndexes;
+    collections: { [key: string]: Collection };
     config: Config;
 
 
@@ -40,13 +40,13 @@ export const CdeContext = createContext<{
     getSuggestions: () => {
         return {}
     },
-    headerMapping: {
-        variableNameIndex: VARIABLE_NAME_INDEX,
-        preciseAbbreviationIndex: ABBREVIATION_INDEX,
-        titleIndex: TITLE_INDEX,
-        interlexIdIndex: INTERLEX_ID_INDEX,
+    headerIndexes: {
+        variableName: VARIABLE_NAME_INDEX,
+        preciseAbbreviation: ABBREVIATION_INDEX,
+        title: TITLE_INDEX,
+        interlexId: INTERLEX_ID_INDEX,
     },
-    collections: [],
+    collections: {},
     config: {
         width: "100%",
         height: "100%",

lib/CdeContextProvider.tsx

@@ -1,30 +1,30 @@
 import {PropsWithChildren, useMemo, useState} from 'react';
-import {DatasetMapping, Entity, InitParams, STEPS} from "./models.ts";
+import {Collection, DatasetMapping, InitParams, OptionDetail, STEPS} from "./models.ts";
 import theme from "./theme/index.tsx";
 import {ThemeProvider} from "@mui/material";
 import CssBaseline from '@mui/material/CssBaseline';
-import {validateDatasetMapping,} from "./services/validatorsService.ts";
-import {mapStringTableToDatasetMapping} from "./services/initialMappingService.ts";
+import {validateDataset, validateDatasetMapping,} from "./services/validatorsService.ts";
+import {getDatasetMapping} from "./services/initialMappingService.ts";
 import {updateDatasetMappingRow} from "./services/updateMappingService.ts";
 import ErrorPage from "./components/ErrorPage.tsx";
 import {ABBREVIATION_INDEX, INTERLEX_ID_INDEX, TITLE_INDEX, VARIABLE_NAME_INDEX} from "./settings.ts";
 import {CdeContext} from './CdeContext.ts';
 import {computeSuggestions} from "./services/suggestionsService.ts";
 
 
-const defaultHeaderMapping = {
-    variableNameIndex: VARIABLE_NAME_INDEX,
-    preciseAbbreviationIndex: ABBREVIATION_INDEX,
-    titleIndex: TITLE_INDEX,
-    interlexIdIndex: INTERLEX_ID_INDEX,
+const defaultHeaderIndexes = {
+    variableName: VARIABLE_NAME_INDEX,
+    preciseAbbreviation: ABBREVIATION_INDEX,
+    title: TITLE_INDEX,
+    interlexId: INTERLEX_ID_INDEX,
 };
 
 export const CdeContextProvider = ({
                                        datasetSample,
                                        datasetMapping: rawDatasetMapping,
                                        additionalDatasetMappings: rawAdditionalDatasetMappings = [],
-                                       headerMapping: providedHeaderMapping = defaultHeaderMapping,
-                                       collections,
+                                       headerIndexes: providedHeaderIndexes = defaultHeaderIndexes,
+                                       collections: rawCollections,
                                        config,
                                        name,
                                        callback,
@@ -35,34 +35,56 @@ export const CdeContextProvider = ({
     const [loadingMessage, setLoadingMessage] = useState<string | null>(null);
     const [errorMessage, setErrorMessage] = useState<string | null>(null);
 
-    const headerMapping = useMemo(() => ({
-        ...defaultHeaderMapping,
-        ...providedHeaderMapping
-    }), [providedHeaderMapping]);
-
-
-    // Process and validate datasetMapping
-    const [initialDatasetMapping, initialDatasetMappingHeader, areFilesValid] = useMemo(() => {
-        let localDatasetMapping = {};
-        let localDatasetMappingHeader: string[] = [];
-        let localAreFilesValid = true;
-
-        if (rawDatasetMapping && rawDatasetMapping.length > 0) {
-            try {
-                validateDatasetMapping(rawDatasetMapping, headerMapping.variableNameIndex);
-            } catch (error) {
-                const message = error instanceof Error ? error.message : 'An unknown error occurred';
-                const errorMessage = `Invalid dataset mapping: ${message}`;
-                console.error(errorMessage);
-                localAreFilesValid = false;
-            }
-            const datasetMappingData = mapStringTableToDatasetMapping(rawDatasetMapping, headerMapping);
-            localDatasetMapping = datasetMappingData[0];
-            localDatasetMappingHeader = datasetMappingData[1]
+
+    // Defines the mapping of the mandatory columns in the dataset mapping file
+
+    const headerIndexes = useMemo(() => {
+        // If the dataset mapping is not provided or has no data we use the default header indexes
+        if (!rawDatasetMapping || rawDatasetMapping.length === 0) {
+            return defaultHeaderIndexes;
+        }
+        // Merge providedHeaderIndexes with defaultHeaderIndexes if rawDatasetMapping is valid
+        return {
+            ...defaultHeaderIndexes,
+            ...providedHeaderIndexes
+        };
+    }, [providedHeaderIndexes, rawDatasetMapping]);
+
+
+    // validate dataset sample
+
+    const isDatasetInvalid = useMemo(() => {
+        let tmpIsDatasetInvalid = false;
+
+        try {
+            validateDataset(datasetSample);
+        } catch (error) {
+            const message = error instanceof Error ? error.message : 'An unknown error occurred';
+            const errorMessage = `Invalid dataset: ${message}`;
+            console.error(errorMessage);
+            tmpIsDatasetInvalid = true;
         }
 
-        return [localDatasetMapping, localDatasetMappingHeader, localAreFilesValid];
-    }, [rawDatasetMapping, headerMapping]);
+        return tmpIsDatasetInvalid;
+    }, [datasetSample]);
+
+    // Validate and process datasetMapping
+
+    const [initialDatasetMapping, initialDatasetMappingHeader, isDatasetMappingInvalid] = useMemo(() => {
+
+        try {
+            validateDatasetMapping(rawDatasetMapping, headerIndexes.variableName);
+        } catch (error) {
+            const message = error instanceof Error ? error.message : 'An unknown error occurred';
+            const errorMessage = `Invalid dataset mapping: ${message}`;
+            console.error(errorMessage);
+            return [{}, [], true]
+        }
+        const datasetHeader = datasetSample[0]
+        const [tmpDatasetMapping, tmpDatasetMappingHeader] = getDatasetMapping(rawDatasetMapping, headerIndexes, datasetHeader);
+
+        return [tmpDatasetMapping, tmpDatasetMappingHeader, false];
+    }, [rawDatasetMapping, headerIndexes, datasetSample]);
 
 
     const [datasetMapping, setDatasetMapping] = useState<DatasetMapping>(initialDatasetMapping);
@@ -71,7 +93,7 @@ export const CdeContextProvider = ({
 
     const additionalDatasetMappings: DatasetMapping[] = rawAdditionalDatasetMappings.map((additionalMapping, index) => {
         try {
-            validateDatasetMapping(additionalMapping, headerMapping.variableNameIndex);
+            validateDatasetMapping(additionalMapping, headerIndexes.variableName);
         } catch (error) {
             if (error instanceof Error) {
                 console.warn(`Skipping invalid additionalDatasetMapping at index ${index}: ${error.message}`);
@@ -80,33 +102,43 @@ export const CdeContextProvider = ({
             }
             return null;
         }
-        const mappedAdditionalMappingData = mapStringTableToDatasetMapping(additionalMapping, headerMapping);
-        return mappedAdditionalMappingData[0];
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        const [mappedAdditionalMapping, _] = getDatasetMapping(additionalMapping, headerIndexes);
+        return mappedAdditionalMapping;
     }).filter(mapping => mapping !== null) as DatasetMapping[];
 
     const suggestions = useMemo(() => {
-        return computeSuggestions(initialDatasetMapping, additionalDatasetMappings, datasetMappingHeader, headerMapping);
-    }, [initialDatasetMapping, additionalDatasetMappings, datasetMappingHeader, headerMapping]);
+        return computeSuggestions(initialDatasetMapping, additionalDatasetMappings, headerIndexes);
+    }, [initialDatasetMapping, additionalDatasetMappings, headerIndexes]);
 
     const getSuggestions = () => {
         return suggestions
     };
 
+    const collectionsDictionary = useMemo(() => {
+        return rawCollections.reduce((acc, collection, index) => {
+            acc[collection.id] = {
+                ...collection,
+                suggested: index === 0 // Set 'suggested' for the first collection, false for others
+            };
+            return acc;
+        }, {} as { [key: string]: Collection });
+    }, [rawCollections]);
 
     const handleClose = () => {
         setErrorMessage(null);
         setLoadingMessage(null);
         callback(datasetMapping)
     };
 
-    const handleUpdateDatasetMappingRow = (key: string, newData: Entity) => {
+    const handleUpdateDatasetMappingRow = (key: string, newData: OptionDetail[]) => {
         updateDatasetMappingRow(
             key,
             newData,
             datasetMapping,
             datasetMappingHeader,
             setDatasetMapping,
-            setDatasetMappingHeader
+            setDatasetMappingHeader,
         );
     };
 
@@ -117,8 +149,8 @@ export const CdeContextProvider = ({
         datasetMappingHeader,
         handleUpdateDatasetMappingRow,
         getSuggestions,
-        headerMapping,
-        collections,
+        headerIndexes,
+        collections: collectionsDictionary,
         config,
         step,
         setStep,
@@ -128,14 +160,16 @@ export const CdeContextProvider = ({
         setErrorMessage,
         handleClose
     };
+
+    const hasErrors = isDatasetInvalid || isDatasetMappingInvalid || rawCollections.length == 0
     return (
         <ThemeProvider theme={theme}>
             <CssBaseline/>
-            {areFilesValid ? (
+            {hasErrors ? <ErrorPage/> : (
                 <CdeContext.Provider value={contextValue}>
                     {children}
                 </CdeContext.Provider>
-            ) : <ErrorPage/>}
+            )}
 
         </ThemeProvider>