Merge pull request #6657 from opencrvs/release-v1.4.1

Release v1.4.1
opencrvs · Mar 22, 2024 · 1577398 · 1577398
2 parents a7f72ee + 0d79fd0
commit 1577398
Show file tree

Hide file tree

Showing 12 changed files with 644 additions and 662 deletions.
diff --git a/packages/client/extract-translations.sh b/packages/client/extract-translations.sh
@@ -12,9 +12,14 @@ get_abs_filename() {
 }
 
 write=false
+outdated=false
 
 for i in "$@"; do
   case $i in
+    --outdated)
+      outdated=true
+      shift
+      ;;
     --write)
       write=true
       shift
@@ -40,6 +45,11 @@ elif [[ ! -d "${COUNTRY_CONFIG_PATH}" ]]; then
   exit 1
 fi
 
+if $outdated; then
+  $(yarn bin)/ts-node --compiler-options='{"module": "commonjs"}' -r tsconfig-paths/register src/extract-translations.ts -- $COUNTRY_CONFIG_PATH --outdated
+  exit 0
+fi
+
 if $write; then
   $(yarn bin)/ts-node --compiler-options='{"module": "commonjs"}' -r tsconfig-paths/register src/extract-translations.ts -- $COUNTRY_CONFIG_PATH --write
   exit 0

diff --git a/packages/client/package.json b/packages/client/package.json
@@ -116,6 +116,7 @@
     "@graphql-codegen/introspection": "^3.0.0",
     "@graphql-codegen/typescript": "^3.0.0",
     "@graphql-codegen/typescript-operations": "^3.0.0",
+    "@types/csv2json": "^1.4.5",
     "@types/enzyme": "^3.1.13",
     "@types/fetch-mock": "^7.3.0",
     "@types/google-libphonenumber": "^7.4.23",
@@ -137,6 +138,8 @@
     "@vitest/coverage-c8": "^0.25.5",
     "@wojtekmaj/enzyme-adapter-react-17": "^0.8.0",
     "chalk": "^2.4.1",
+    "csv-stringify": "^6.4.6",
+    "csv2json": "^2.0.2",
     "enzyme": "^3.4.4",
     "eslint": "^7.11.0",
     "eslint-config-prettier": "^8.3.0",
@@ -163,7 +166,6 @@
     "traverse": "^0.6.6",
     "ts-node": "^7.0.1",
     "typescript": "^4.7.4",
-    "typescript-react-intl": "^0.3.0",
     "vite-plugin-pwa": "^0.16.4",
     "vitest": "0.25.5",
     "vitest-fetch-mock": "^0.2.1",

diff --git a/packages/client/src/extract-translations.ts b/packages/client/src/extract-translations.ts
@@ -11,47 +11,124 @@
 /* eslint-disable */
 import * as fs from 'fs'
 import glob from 'glob'
-import main, { Message } from 'typescript-react-intl'
 import chalk from 'chalk'
-import { ILanguage } from '@client/i18n/reducer'
+import csv2json from 'csv2json'
+import { stringify, Options } from 'csv-stringify'
+import { promisify } from 'util'
+import { sortBy } from 'lodash'
+import ts from 'typescript'
+import { MessageDescriptor } from 'react-intl'
+const csvStringify = promisify<Array<Record<string, any>>, Options>(stringify)
+
+export async function writeJSONToCSV(
+  filename: string,
+  data: Array<Record<string, any>>
+) {
+  const csv = await csvStringify(data, {
+    header: true
+  })
+  return fs.promises.writeFile(filename, csv, 'utf8')
+}
 
-interface IReactIntlDescriptions {
-  [key: string]: string
+export async function readCSVToJSON<T>(filename: string) {
+  return new Promise<T>((resolve, reject) => {
+    const chunks: string[] = []
+    fs.createReadStream(filename)
+      .on('error', reject)
+      .pipe(
+        csv2json({
+          separator: ','
+        })
+      )
+      .on('data', (chunk: string) => chunks.push(chunk))
+      .on('error', reject)
+      .on('end', () => {
+        resolve(JSON.parse(chunks.join('')))
+      })
+  })
 }
 
+type CSVRow = { id: string; description: string } & Record<string, string>
+
 const write = process.argv.includes('--write')
+const outdated = process.argv.includes('--outdated')
+
 const COUNTRY_CONFIG_PATH = process.argv[2]
-type LocalisationFile = {
-  data: Array<{
-    lang: string
-    displayName: string
-    messages: Record<string, string>
-  }>
-}
+
+type LocalisationFile = CSVRow[]
 
 function writeTranslations(data: LocalisationFile) {
-  fs.writeFileSync(
-    `${COUNTRY_CONFIG_PATH}/src/api/content/client/client.json`,
-    JSON.stringify(data, null, 2)
+  return writeJSONToCSV(
+    `${COUNTRY_CONFIG_PATH}/src/translations/client.csv`,
+    data
   )
 }
 
 function readTranslations() {
-  return JSON.parse(
-    fs
-      .readFileSync(`${COUNTRY_CONFIG_PATH}/src/api/content/client/client.json`)
-      .toString()
+  return readCSVToJSON<CSVRow[]>(
+    `${COUNTRY_CONFIG_PATH}/src/translations/client.csv`
   )
 }
 
-function isEnglish(obj: ILanguage) {
-  return obj.lang === 'en-US' || obj.lang === 'en'
+function findObjectLiteralsWithIdAndDefaultMessage(
+  filePath: string,
+  sourceCode: string
+): MessageDescriptor[] {
+  const sourceFile = ts.createSourceFile(
+    'temp.ts',
+    sourceCode,
+    ts.ScriptTarget.Latest,
+    true
+  )
+  const matches: MessageDescriptor[] = []
+
+  function visit(node: ts.Node) {
+    if (!ts.isObjectLiteralExpression(node)) {
+      ts.forEachChild(node, visit)
+      return
+    }
+    const idProperty = node.properties.find(
+      (p) => ts.isPropertyAssignment(p) && p.name.getText() === 'id'
+    )
+    const defaultMessageProperty = node.properties.find(
+      (p) => ts.isPropertyAssignment(p) && p.name.getText() === 'defaultMessage'
+    )
+
+    if (!(idProperty && defaultMessageProperty)) {
+      ts.forEachChild(node, visit)
+      return
+    }
+
+    const objectText = node.getText(sourceFile) // The source code representation of the object
+
+    try {
+      const func = new Function(`return (${objectText});`)
+      const objectValue = func()
+      matches.push(objectValue)
+    } catch (error) {
+      console.log(chalk.yellow.bold('Warning'))
+      console.error(
+        `Found a dynamic message identifier in file ${filePath}.`,
+        'Message identifiers should never be dynamic and should always be hardcoded instead.',
+        'This enables us to confidently verify that a country configuration has all required keys.',
+        '\n',
+        objectText,
+        '\n'
+      )
+    }
+
+    ts.forEachChild(node, visit)
+  }
+
+  visit(sourceFile)
+
+  return matches
 }
 
 async function extractMessages() {
   let translations: LocalisationFile
   try {
-    translations = readTranslations()
+    translations = await readTranslations()
   } catch (error: unknown) {
     const err = error as Error & { code: string }
     if (err.code === 'ENOENT') {
@@ -60,88 +137,118 @@ async function extractMessages() {
         `Your environment variables may not be set.
         Please add valid COUNTRY_CONFIG_PATH, as an environment variable.
         If they are set correctly, then something is wrong with
-        this file: ${COUNTRY_CONFIG_PATH}/src/api/content/client/client.json`
+        this file: ${COUNTRY_CONFIG_PATH}/src/translations/client.csv`
       )
     } else {
       console.error(err)
     }
     process.exit(1)
   }
-  let results: Message[] = []
-  const pattern = 'src/**/*.@(tsx|ts)'
-  try {
-    // eslint-disable-line no-console
-    console.log('Checking translations in application...')
-    console.log()
 
-    glob(pattern, (err: any, files) => {
-      if (err) {
-        throw new Error(err)
-      }
+  const knownLanguages =
+    translations.length > 0
+      ? Object.keys(translations[0]).filter(
+          (key) => !['id', 'description'].includes(key)
+        )
+      : ['en']
 
-      files.forEach((f) => {
-        const contents = fs.readFileSync(f).toString()
-        results = results.concat(main(contents))
-      })
+  console.log('Checking translations in application...')
+  console.log()
 
-      const reactIntlDescriptions: IReactIntlDescriptions = {}
-      results.forEach((r) => {
-        reactIntlDescriptions[r.id] = r.description!
-      })
-      const englishTranslations = translations.data.find(isEnglish)?.messages
-      const missingKeys = Object.keys(reactIntlDescriptions).filter(
-        (key) => !englishTranslations?.hasOwnProperty(key)
-      )
+  const files = await promisify(glob)('src/**/*.@(tsx|ts)', {
+    ignore: ['**/*.test.@(tsx|ts)', 'src/tests/**/*.*']
+  })
+
+  const messagesParsedFromApp: MessageDescriptor[] = files
+    .map((f) => {
+      const contents = fs.readFileSync(f).toString()
+      return findObjectLiteralsWithIdAndDefaultMessage(f, contents)
+    })
+    .flat()
+
+  const reactIntlDescriptions: Record<string, string> = Object.fromEntries(
+    messagesParsedFromApp.map(({ id, description }) => [id, description || ''])
+  )
+
+  const missingKeys = Object.keys(reactIntlDescriptions).filter(
+    (key) => !translations.find(({ id }) => id === key)
+  )
+
+  if (outdated) {
+    const extraKeys = translations
+      .map(({ id }) => id)
+      .filter((key) => !reactIntlDescriptions[key])
+
+    console.log(chalk.yellow.bold('Potentially outdated translations'))
+    console.log(
+      'The following keys were not found in the code, but are part of the copy file:',
+      '\n'
+    )
+    console.log(extraKeys.join('\n'))
+  }
 
-      if (missingKeys.length > 0) {
-        // eslint-disable-line no-console
-        console.log(chalk.red.bold('Missing translations'))
-        console.log(`You are missing the following content keys from your country configuration package:\n
+  if (missingKeys.length > 0) {
+    // eslint-disable-line no-console
+    console.log(chalk.red.bold('Missing translations'))
+    console.log(`You are missing the following content keys from your country configuration package:\n
 ${chalk.white(missingKeys.join('\n'))}\n
 Translate the keys and add them to this file:
-${chalk.white(`${COUNTRY_CONFIG_PATH}/src/api/content/client/client.json`)}`)
-
-        if (write) {
-          console.log(
-            `${chalk.yellow('Warning ⚠️:')} ${chalk.white(
-              'The --write command is experimental and only adds new translations for English.'
-            )}`
-          )
-
-          const defaultsToBeAdded = missingKeys.map((key) => [
-            key,
-            results.find(({ id }) => id === key)?.defaultMessage
-          ])
-          const newEnglishTranslations: Record<string, string> = {
-            ...englishTranslations,
-            ...Object.fromEntries(defaultsToBeAdded)
-          }
-
-          const english = translations.data.find(isEnglish)!
-          english.messages = newEnglishTranslations
-          writeTranslations(translations)
-        } else {
-          console.log(`
-${chalk.green('Tip 🪄')}: ${chalk.white(
-            `If you want this command do add the missing English keys for you, run it with the ${chalk.bold(
-              '--write'
-            )} flag. Note that you still need to add non-English translations to the file.`
-          )}`)
-        }
-
-        process.exit(1)
-      }
-
-      fs.writeFileSync(
-        `${COUNTRY_CONFIG_PATH}/src/api/content/client/descriptions.json`,
-        JSON.stringify({ data: reactIntlDescriptions }, null, 2)
+${chalk.white(`${COUNTRY_CONFIG_PATH}/src/translations/client.csv`)}`)
+
+    if (write) {
+      console.log(
+        `${chalk.yellow('Warning ⚠️:')} ${chalk.white(
+          'The --write command is experimental and only adds new translations for English.'
+        )}`
       )
-    })
-  } catch (err) {
-    // eslint-disable-line no-console
-    console.log(err)
+
+      // This is just to ensure that all languages stay in the CVS file
+      const emptyLanguages = Object.fromEntries(
+        knownLanguages.map((lang) => [lang, ''])
+      )
+
+      const defaultsToBeAdded = missingKeys.map(
+        (key): CSVRow => ({
+          id: key,
+          description: reactIntlDescriptions[key],
+          ...emptyLanguages,
+          en:
+            messagesParsedFromApp
+              .find(({ id }) => id === key)
+              ?.defaultMessage?.toString() || ''
+        })
+      )
+
+      const allIds = Array.from(
+        new Set(
+          defaultsToBeAdded
+            .map(({ id }) => id)
+            .concat(translations.map(({ id }) => id))
+        )
+      )
+
+      const allTranslations = allIds.map((id) => {
+        const existingTranslation = translations.find(
+          (translation) => translation.id === id
+        )
+
+        return (
+          existingTranslation ||
+          defaultsToBeAdded.find((translation) => translation.id === id)!
+        )
+      })
+
+      await writeTranslations(sortBy(allTranslations, (row) => row.id))
+    } else {
+      console.log(`
+${chalk.green('Tip 🪄')}: ${chalk.white(
+        `If you want this command to add the missing English keys for you, run it with the ${chalk.bold(
+          '--write'
+        )} flag. Note that you still need to add non-English translations to the file.`
+      )}`)
+    }
+
     process.exit(1)
-    return
   }
 }