diff --git a/tdrive/backend/node/src/cli/cmds/dev.ts b/tdrive/backend/node/src/cli/cmds/dev.ts new file mode 100644 index 000000000..ad43858b5 --- /dev/null +++ b/tdrive/backend/node/src/cli/cmds/dev.ts @@ -0,0 +1,14 @@ +import { CommandModule } from "yargs"; + +const command: CommandModule = { + describe: false, // "Tools for Tdrive developers", + command: "dev ", + builder: yargs => + yargs.commandDir("dev_cmds", { + visit: commandModule => commandModule.default, + }), + // eslint-disable-next-line @typescript-eslint/no-empty-function + handler: () => {}, +}; + +export default command; diff --git a/tdrive/backend/node/src/cli/cmds/dev_cmds/debug.ts b/tdrive/backend/node/src/cli/cmds/dev_cmds/debug.ts new file mode 100644 index 000000000..7915e83c5 --- /dev/null +++ b/tdrive/backend/node/src/cli/cmds/dev_cmds/debug.ts @@ -0,0 +1,322 @@ +import { CommandModule } from "yargs"; +import runWithPlatform from "../../lib/run-with-platform"; +import Repository from "src/core/platform/services/database/services/orm/repository/repository"; +import { DatabaseServiceAPI } from "src/core/platform/services/database/api"; +import { + EntityTarget, + FindFilter, + FindOptions, + SearchServiceAPI, +} from "src/core/platform/services/search/api"; +import { TdrivePlatform } from "src/core/platform/platform"; +import SearchRepository from "src/core/platform/services/search/repository"; +import { IndentedPrinter } from "../../utils/text"; + +const waitForFlush = (stream: NodeJS.WriteStream) => + new Promise((resolve, reject) => + stream.write("\r\n", err => (err ? reject(err) : resolve())), + ); + +interface EntityWithID { + id: string; +} + +const repoKinds = {}; +class EntityRepositoryKind { + private readonly fields?: string[]; + constructor( + private readonly tableName: string, + private readonly entity: EntityTarget, + private readonly entityType: string, + private readonly fieldNames?: string, + ) { + this.fields = fieldNames ? fieldNames.split(/\s+/g) : undefined; + repoKinds[tableName] = this; + } + get name() { + return this.tableName; + } + labelFor(item: Entity) { + return `${this.entityType} ${(item as EntityWithID).id} - ${this.fieldNames!.split(" ") + .map(f => `${f}: ${JSON.stringify(item[f])}`) + .join(" ")}`; + } + async databaseRepo(platform: TdrivePlatform): Promise> { + return platform + .getProvider("database") + .getRepository(this.tableName, this.entity); + } + async searchRepo(platform: TdrivePlatform): Promise> { + return platform + .getProvider("search") + .getRepository(this.tableName, this.entity); + } + print(entities: Entity[]) { + if (entities.length) console.log("Total fields:", Object.keys(entities[0]).sort().join(" ")); + console.table(entities, this.fields); + console.log(`--- ${entities.length} ${this.tableName}`); + } + + async getAll( + platform: TdrivePlatform, + repoType: "database" | "search", + findFilter: FindFilter = {}, + findOptions: FindOptions = {}, + ): Promise { + switch (repoType) { + case "database": + const dbRepo = await this.databaseRepo(platform); + return (await dbRepo.find(findFilter, findOptions, undefined)).getEntities(); + case "search": + const searchRepo = await this.searchRepo(platform); + const mergedFindOptions: FindOptions = { + ...findOptions, + $text: { $search: "", ...findOptions.$text }, + // $sort: { id: 'asc' }, // id is created as text, not keyword, so needs inverted index: set fielddata=true on [id] + pagination: { ...findOptions.pagination }, + }; + + return (await searchRepo.search(findFilter, mergedFindOptions, undefined)).getEntities(); + default: + throw new Error("Unknown repoType: " + repoType); + } + } +} + +// Fields: _status_icon cache creation_date deleted devices email_canonical first_name id identity_provider identity_provider_id language last_activity last_name mail_verified notification_preference password phone picture preferences salt thumbnail_id timezone token_login type username_canonical +import User, { TYPE as UserTYPE } from "../../../services/user/entities/user"; +const UserRepoKind = new EntityRepositoryKind( + UserTYPE, + User, + "👨", + "email_canonical first_name last_name type", +); + +// access_info added company_id content_keywords creator description extension id is_directory is_in_trash last_modified last_version_cache name parent_id scope size tags +import { DriveFile, TYPE as DriveFileTYPE } from "../../../services/documents/entities/drive-file"; +const DriveFileRepoKind = new EntityRepositoryKind( + DriveFileTYPE, + DriveFile, + "📄", + "company_id id creator name size content_keywords", +); +// 'access_info', 'added', +// 'company_id', 'content_keywords', +// 'creator', 'description', +// 'extension', 'id', +// 'is_directory', 'is_in_trash', +// 'last_modified', 'last_version_cache', +// 'name', 'parent_id', +// 'scope', 'size', +// 'tags' + +// group_usercompany +// applications dateAdded group_id id lastUpdateDay nbWorkspaces role user_id +import CompanyUser, { TYPE as CompanyUserTYPE } from "../../../services/user/entities/company_user"; +const CompanyUserRepoKind = new EntityRepositoryKind( + CompanyUserTYPE, + CompanyUser, + "👨🏻‍💼", +); + +// group_entity +// dateAdded displayName id identity_provider identity_provider_id logo logofile memberCount name onCreationData plan stats +import Company, { TYPE as CompanyTYPE } from "../../../services/user/entities/company"; +const CompanyRepoKind = new EntityRepositoryKind(CompanyTYPE, Company, "🏛️", "name plan"); + +// // Seems empty/unused: +// import Device, { TYPE as DeviceTYPE } from "../../../services/user/entities/device"; +// const DeviceRepoKind = new EntityRepositoryKind(DeviceTYPE, Device, "💻"); + +interface JoinedEntity extends EntityWithID { + id: string; + db?: Entity; + search?: Entity; +} + +async function getAllJoined( + platform: TdrivePlatform, + repoKind: EntityRepositoryKind, +) { + async function getAllOrEmpty(repoType: "database" | "search") { + try { + return await repoKind.getAll(platform, repoType); + } catch (e) { + return []; + } + } + const db = await getAllOrEmpty("database"); + const search = await getAllOrEmpty("search"); + const result = new Map>(); + db.forEach((entity: EntityWithID) => { + const entry = result.get(entity.id); + if (!entry) result.set(entity.id, { id: entity.id, db: entity as Entity }); + else entry.db = entity as Entity; + }); + search.forEach((entity: EntityWithID) => { + const entry = result.get(entity.id); + if (!entry) result.set(entity.id, { id: entity.id, search: entity as Entity }); + else entry.search = entity as Entity; + }); + return result; +} + +async function _printDBAndSearch( + platform: TdrivePlatform, + kind: EntityRepositoryKind, +) { + console.log(`\n\t${kind.name} from database`); + kind.print(await kind.getAll(platform, "database")); + console.log(`\n\t${kind.name} from search`); + try { + kind.print(await kind.getAll(platform, "search")); + } catch (e) { + console.error(e); + } + await waitForFlush(process.stdout); + await waitForFlush(process.stderr); +} + +async function report(platform: TdrivePlatform) { + let companyUsers = await CompanyUserRepoKind.getAll(platform, "database"); + const companies = await CompanyRepoKind.getAll(platform, "database"); + const users = await getAllJoined(platform, UserRepoKind); + const usersAll = new Map(users); + const files = await getAllJoined(platform, DriveFileRepoKind); + const printer = new IndentedPrinter(); + + const shortCompanyLabel = (id: string) => { + const company = companies.find(({ id: companyID }) => companyID == id); + return `🏛️ ${company ? `${company.name} (${id})` : id}`; + }; + const shortUserLabel = (id: string) => { + const user = usersAll.get(id)?.db; + if (!user) return `👨 ${id}`; + return `👨 ${user.first_name} ${user.last_name} (${user.email_canonical} - ${id})`; + }; + function describeAccessInfoEntities(file: DriveFile, creatorUserID: string, companyID: string) { + function pluck(list: T[], filter: (entity: T) => boolean): T { + let foundIndex = undefined; + list.forEach((item, index) => { + if (filter(item)) { + if (foundIndex !== undefined) throw new Error(`Pluck failed in ${JSON.stringify(list)}`); + foundIndex = index; + } + }); + return foundIndex === undefined ? undefined : list.splice(foundIndex, 1)[0]; + } + const entities = file.access_info.entities; + const creatorPerm = pluck(entities, ({ type, id }) => type === "user" && id === creatorUserID); + const parentFolderPerm = pluck( + entities, + ({ type, id }) => type === "folder" && id === "parent", + ); + const companyPerm = pluck(entities, ({ type, id }) => type === "company" && id === companyID); + const result = []; + if (creatorPerm) result.push(`👨 ${creatorPerm.level}`); + if (parentFolderPerm) result.push(`📁 ${parentFolderPerm.level}`); + if (companyPerm && companyPerm.level != "none") result.push(`🏛️ ${companyPerm.level}`); + if (file.access_info.public && file.access_info.public.level != "none") { + let description = `🔗 ${file.access_info.public.level}`; + if (file.access_info.public.password) + description += ` password: ${JSON.stringify(file.access_info.public.password)}`; + if (file.access_info.public.expiration) + description += ` expires: ${new Date(file.access_info.public.expiration).toISOString()}`; + result.push(description); + } + return ` (${result.join(" - ")})`; + } + const fileLabel = (file: DriveFile) => + `${file.is_directory ? "📁" : "📄"}${file.is_in_trash ? "🗑" : ""} ${file.name} - ${( + file.size / 1024 + ).toFixed(0)}kb`; + function filePrintDetails(file: JoinedEntity, userID: string, companyID: string) { + const db = file.db!; + if (!db.is_directory && !file.search) printer.appendToPrevious(" (⚠️ not indexed)"); + if (!db.is_directory && !db.content_keywords) printer.appendToPrevious(" (⚠️ 🔎 no keywords)"); + if (db.access_info) { + printer.appendToPrevious(describeAccessInfoEntities(db, userID, companyID)); + for (const permitted of db.access_info.entities) { + const prefix = `• 🔑 ${permitted.level} `; + if (permitted.type == "user") printer.push(`${prefix}for ${shortUserLabel(permitted.id)}`); + else if (permitted.type == "company") + printer.push(`${prefix}for ${shortCompanyLabel(permitted.id)}`); + else if (permitted.type == "folder" && permitted.id == "parent") + printer.push(`${prefix}from parent 📂`); + else printer.push(`${prefix}for ${permitted.type}: ${permitted.id}`); + } + } + if (db.is_directory) return; + if (db.content_keywords) printer.push("• 🔎", db.content_keywords); + } + for (const company of companies) { + await printer.inside(" ", [CompanyRepoKind.labelFor(company)], async () => { + const userIDs = companyUsers + .filter(({ group_id }) => group_id === company.id) + .map(e => e.user_id); + companyUsers = companyUsers.filter(entity => entity.group_id !== company.id); + printer.appendToPrevious(` (${userIDs.length} user(s))`); + for (const userID of userIDs) { + const user = users.get(userID); + users.delete(userID); + await printer.inside(" ", [shortUserLabel(user.id)], async () => { + const filesOfUser = [...files.values()].filter(file => file.db!.creator === userID); + if (!user.search) printer.appendToPrevious(" (⚠️ not indexed)"); + printer.appendToPrevious(` (${filesOfUser.length} file(s))`); + filesOfUser.forEach(file => files.delete(file.id)); + async function printFilesUnder(parentId: string) { + const indicesToDelete = []; + const filesWithParent = filesOfUser.filter((f, i) => { + if (f.db!.parent_id === parentId) { + indicesToDelete.push(i); + return true; + } + return false; + }); + indicesToDelete.reverse().forEach(i => filesOfUser.splice(i, 1)); + for (const file of filesWithParent) { + await printer.inside(" ", [fileLabel(file.db!)], async () => { + filePrintDetails(file, userID, company.id); + if (file.db!.is_directory) await printFilesUnder(file.db!.id); + }); + } + } + await printFilesUnder("user_" + userID); + if (filesOfUser.length) + await printer.inside(" ", ["⚠️ Files with unknown parent:"], async () => { + for (const file of filesOfUser) + await printer.inside(" ", [fileLabel(file.db!)], async () => { + filePrintDetails(file, userID, company.id); + }); + }); + }); + } + }); + } + + if (companyUsers.length) + printer.push("Warning: company_users with unknown company: ", companyUsers); + if (users.size) printer.push("Warning: users with unknown company: ", users); + if (files.size) printer.push("Warning: files with unknown creator: ", files); + + return printer.toString(); +} + +const command: CommandModule = { + describe: + "Debug command used by developers to run against platform by editing " + + __filename + + "." + + "In current state loads everything to memory then prints it. Probably usefull only in debug.", + command: "debug", + builder: {}, + handler: async _argv => { + let result; + await runWithPlatform("Debug command", async ({ spinner: _spinner, platform }) => { + result = await report(platform); + }); + console.log(result); + }, +}; + +export default command; diff --git a/tdrive/backend/node/src/cli/cmds/dev_cmds/labels.ts b/tdrive/backend/node/src/cli/cmds/dev_cmds/labels.ts new file mode 100644 index 000000000..bde5b2728 --- /dev/null +++ b/tdrive/backend/node/src/cli/cmds/dev_cmds/labels.ts @@ -0,0 +1,362 @@ +import yargs from "yargs"; + +import parseYargsCommaSeparatedStringArray from "../../utils/yargs-comma-array"; +import { NonPlatformCommandYargsBuilder } from "../../utils/non-plaform-command-yargs-builder"; + +import { startProgress, box, printTable } from "../../utils/text"; + +import FS from "fs/promises"; +import Path from "path"; + +import { getAgVersion, runRegexp, runSearchLiteral } from "../../utils/exec-ag"; + +import { checkBuiltFileYoungerThanSource } from "../../utils/check-built-younger-than-source"; + +function upsertMapOfContainers( + map: Map, + key: TK, + ctor: (TK) => TV, + add: (TV) => void, +) { + let container = map.get(key); + if (container === undefined) map.set(key, (container = ctor(key))); + add(container); + return container; +} +const upsertMapOfSets = (map: Map>, key: TK, value: TV) => + upsertMapOfContainers( + map, + key, + () => new Set(), + set => set.add(value), + ); +const upsertMapOfArrays = (map: Map, key: TK, value: TV) => + upsertMapOfContainers( + map, + key, + () => [], + array => array.push(value), + ); +const upsertMapValue = (map: Map, key: TK, update: (value: TV | undefined) => TV) => + map.set(key, update(map.get(key))); +const upsertMapOfCounts = (map: Map, key: TK) => + upsertMapValue(map, key, count => (count || 0) + 1); +const mapOfCountsEntriesOrderedByCount = (map: Map) => + [...map.entries()].sort(([_a, a], [_b, b]) => b - a); +const mapOfCountsKeysOrderedByCount = (map: Map) => + mapOfCountsEntriesOrderedByCount(map).map(([key]) => key); + +const REPO_ROOT = Path.join(__dirname, ...new Array(6 + 1).fill("..")); +const FRONT_ROOT = Path.join(REPO_ROOT, "tdrive/frontend"); +const FRONT_LOCALES = Path.join(FRONT_ROOT, "public/locales"); +const BACK_ROOT = Path.join(REPO_ROOT, "tdrive/backend/node"); +const BACK_LOCALES = Path.join(BACK_ROOT, "locales"); + +/** Represents a single locale json file of labels -> translations */ +class LabelFile { + private constructor( + private readonly lang: string, + private readonly filename: string, + private readonly data: { [key: string]: string }, + ) {} + static async load(lang: string, filename: string) { + const data = JSON.parse(await FS.readFile(filename, "utf-8")); + return new LabelFile(lang, filename, data); + } + private get keys() { + return Object.keys(this.data); + } + get count() { + return this.keys.length; + } + get labels() { + return this.keys.sort(); + } + translate(label: string): string | undefined { + return this.data[label]; + } + toString() { + return `<${this.count} ${this.lang} labels from ${this.filename}>`; + } +} + +/** Represents a set of json files, one per language, and a project root */ +class LabelRepository { + private constructor( + private readonly projectPath: string, + private readonly localesPath: string, + private readonly langs: { [key: string]: LabelFile }, + ) {} + static async load(project: "front" | "back", filterLanguages?: string[]) { + const [projectPath, localesPath] = + project == "front" ? [FRONT_ROOT, FRONT_LOCALES] : [BACK_ROOT, BACK_LOCALES]; + const files = (await FS.readdir(localesPath, { withFileTypes: true })).filter( + ent => ent.isFile() && ent.name.match(/\.json$/i), + ); + const result = {}; + for (const ent of files) { + const lang = Path.basename(ent.name, ".json"); + if (filterLanguages && filterLanguages.indexOf(lang.toLowerCase()) < 0) continue; + const filename = Path.join((ent as unknown as { path: string }).path, ent.name); + result[lang] = await LabelFile.load(lang, filename); + } + if (Object.keys(result).length === 0) + throw new Error( + `No locales found in ${localesPath}.${ + filterLanguages ? " Check language filter argument." : "" + }`, + ); + + return new LabelRepository(projectPath, localesPath, result); + } + + get languages() { + return Object.keys(this.langs).sort(); + } + get languagesByCount() { + return this.languages + .map(lang => [lang, this.langs[lang].count]) + .sort(([_a, a]: [string, number], [_b, b]: [string, number]) => b - a); + } + translate(label, lang) { + return this.langs[lang].translate(label); + } + private __indexedLabels?: Map>; + /** Returns translation keys to sets of the languages knowing them */ + get indexedLabels() { + if (this.__indexedLabels) return this.__indexedLabels; + const keys = new Map>(); + for (const lang of this.languages) + for (const label of this.langs[lang].labels) upsertMapOfSets(keys, label, lang); + return (this.__indexedLabels = keys); + } + + /** Return labels that are in at least one locale file and are not in all locale files */ + indexedLabelsThatAreInOneButNotEveryLanguage(includeAnyway?: (label: string) => boolean) { + const labels = this.indexedLabels; + const count = this.languages.length; + const result = new Map>(); + for (const [label, set] of labels.entries()) + if (set.size != count || (includeAnyway && includeAnyway(label))) result.set(label, set); + return result; + } + + private __findLabelsInSource?: Map; + /** This is a very simple heuristic, it won't catch most labels actually. Returns key to list of files. */ + async findUnkownLabelsInSource(includeKnown: boolean = false): Promise> { + if (this.__findLabelsInSource) return this.__findLabelsInSource; + const matches = await runRegexp( + /'(?[a-z_-]+\.[a-z_-]+\.[a-z_.-]+)'/i, + FRONT_ROOT, + FRONT_LOCALES, + ); + const labelsToFiles = new Map(); + matches.forEach(({ file, match }) => { + if (!includeKnown && this.indexedLabels.has(match)) return; + const files = labelsToFiles.get(match); + if (files) files.push(file); + else labelsToFiles.set(match, [file]); + }); + return (this.__findLabelsInSource = labelsToFiles); + } + + private __getUnusedLabels?: Map>; + /** Search each label in the source code and return those with no matches and the languages they exist in */ + async getUnusedLabels() { + if (this.__getUnusedLabels) return this.__getUnusedLabels; + const logger = startProgress("Searching source for label "); + const unusedKeys = new Map>(); + const indexed = this.indexedLabels; + let keyIndex = 0; + for (const [label, languages] of indexed.entries()) { + logger(`${(keyIndex++ + "").padStart(3)} / ${indexed.size}: ${label}`); + const found = !!(await runSearchLiteral(label, this.projectPath, this.localesPath)).length; + if (!found) unusedKeys.set(label, languages); + } + logger(`- Found ${unusedKeys.size} unused keys`, true); + return (this.__getUnusedLabels = unusedKeys); + } + /** Return all the translation of the given label. Keys are languages the label existed in. */ + getTranslationsOf(label): { [language: string]: string } { + const result = {}; + for (const lang of this.languages) { + const translation = this.translate(label, lang); + if (translation) result[lang] = translation; + } + return result; + } + /** Return every label that has translations with identical values. + * The values in the map are arrays of the languages that are identical, themselves in an array + */ + getRepeatedTranslation(): Map { + const result = new Map(); + for (const [label, languages] of this.indexedLabels.entries()) { + const valuesToLang = new Map>(); + languages.forEach(lang => upsertMapOfSets(valuesToLang, this.translate(label, lang), lang)); + for (const [_text, languages] of valuesToLang.entries()) + if (languages.size > 1) upsertMapOfArrays(result, label, [...languages]); + } + return result; + } + makeTableOfIncompleteLabels( + unusedLabels: Map> | undefined, + unknownFromSource: Map | undefined, + previewIdenticalMaxLength: number, + forceColumns?: string[], + showMissingInsteadOfPresent?: boolean, + ): string { + const table = []; + let columns = ["label"]; + + if (unusedLabels?.size) columns.push("unused"); + if (unknownFromSource?.size) columns.push("src"); + const columnsToLeaveIntact = columns.length; + + const repeatedLabels = this.getRepeatedTranslation(); + const markRowRepetitions = (row, label) => { + const repeatedLanguagesSet = repeatedLabels.get(label); + if (repeatedLanguagesSet?.length) { + let repetitionIndex = 0; + for (const languages of repeatedLanguagesSet) { + const text = this.translate(label, languages[0]); + const indicator = + text.length < previewIdenticalMaxLength + ? JSON.stringify(text) + : `=${repetitionIndex ? repetitionIndex : "="}`; + repetitionIndex++; + for (const language of languages) row[language] = indicator; + } + } + }; + const seenLanguages = new Map(); + const mark = "✔"; + const labels = this.indexedLabelsThatAreInOneButNotEveryLanguage( + label => repeatedLabels.has(label) || unusedLabels?.has(label), + ); + for (const [label, set] of labels) { + const row = { label }; + for (const lang of set) { + row[lang] = mark; + upsertMapOfCounts(seenLanguages, lang); + } + if (unusedLabels?.has(label)) row["unused"] = mark; + markRowRepetitions(row, label); + table.push(row); + } + if (unknownFromSource) + for (const label of unknownFromSource.keys()) table.push({ label, src: mark }); + + columns = columns.concat(mapOfCountsKeysOrderedByCount(seenLanguages)); + if (forceColumns) + for (const col of forceColumns) if (columns.indexOf(col) < 0) columns.push(col); + return printTable( + table, + columns, + false, + true, + showMissingInsteadOfPresent + ? (cell, x, _y) => + x < columnsToLeaveIntact ? cell : cell === mark ? "" : cell == null ? "?" : cell + : undefined, + ); + } +} + +async function printReport(component: "back" | "front", options: LabelsArguments) { + let title = `Report for ${component} labels`; + if (options.languages?.length) title += ` in ${options.languages.sort().join("+")}`; + if (options.skipSourceSearch) title += " (without source code searches)"; + console.log(box(title)); + const repo = await LabelRepository.load(component, options.languages); + for (const lang of options.languages || []) + if (repo.languages.indexOf(lang) < 0) + throw new Error(`Language '${lang}' was not found in locale files`); + if (!options.languages) console.error(`Loaded languages: ${repo.languages.join(" ")}`); + const labelsFromSource = options.skipSourceSearch + ? undefined + : await repo.findUnkownLabelsInSource(); + const unusedLabels = options.skipSourceSearch ? undefined : await repo.getUnusedLabels(); + console.log( + repo.makeTableOfIncompleteLabels( + unusedLabels, + labelsFromSource, + options.previewIdenticalMaxLength, + options.languages, + options.showMissingInsteadOfPresent, + ), + ); +} + +interface LabelsArguments { + skipSourceSearch: boolean; + languages?: string[]; + showMissingInsteadOfPresent: boolean; + previewIdenticalMaxLength: number; +} +const ScanGroup = "Scan options"; +const command: yargs.CommandModule = { + command: "labels", + describe: ` + Load the locale json files, and optionally scan the source code, and + print a report listing labels that are incomplete (not in all the + translations loaded, or not in the source code). + `.trim(), + builder: { + ...NonPlatformCommandYargsBuilder, + skipSourceSearch: { + type: "boolean", + alias: "s", + describe: "Skip using ag to scan source code for missing labels", + default: false, + group: ScanGroup, + }, + languages: { + type: "string", + array: true, + alias: "l", + description: "Comma separated languages to consider, all others will be ignored", + group: ScanGroup, + }, + showMissingInsteadOfPresent: { + type: "boolean", + alias: "i", + default: false, + description: "If set show '?' on absent translations rather than ticks on present ones.", + }, + previewIdenticalMaxLength: { + type: "number", + alias: "p", + default: 10, + description: "When displaying identical translations, put them inline if shorter than this", + }, + }, + handler: async argv => { + try { + await checkBuiltFileYoungerThanSource(__filename); + const args = argv as unknown as LabelsArguments; + args.languages = parseYargsCommaSeparatedStringArray(args.languages).map(x => + x.toLowerCase(), + ); + if (!args.languages?.length) args.languages = null; + if (!args.skipSourceSearch) { + const agVersion = !args.skipSourceSearch && (await getAgVersion()); + if (agVersion === false && !args.skipSourceSearch) + console.warn( + "Warning: command 'ag' not found, please install https://github.com/ggreer/the_silver_searcher . Source code searches disabled.", + ); + args.skipSourceSearch = !agVersion; + } + + if (args.languages?.length === 1 && args.skipSourceSearch) + throw new Error("Only one language selected and source scan disabled - nothing to do."); + + await printReport("front", args); + // Back doesn't really do translations + // await printReport("back", args); + } catch (e) { + console.error(e.stack || e); + process.exit(1); + } + }, +}; +export default command; diff --git a/tdrive/backend/node/src/cli/cmds/dev_cmds/translate.ts b/tdrive/backend/node/src/cli/cmds/dev_cmds/translate.ts new file mode 100644 index 000000000..147ae03d4 --- /dev/null +++ b/tdrive/backend/node/src/cli/cmds/dev_cmds/translate.ts @@ -0,0 +1,75 @@ +import yargs from "yargs"; + +import parseYargsCommaSeparatedStringArray from "../../utils/yargs-comma-array"; +import { NonPlatformCommandYargsBuilder } from "../../utils/non-plaform-command-yargs-builder"; +import { openWithSystemViewer } from "../../../utils/exec"; + +const CORE_LANGUAGES = "en fr ru vn".split(" "); + +// const iso639ToTwakeDriveISO = set1 => (set1 === "vi" ? "vn" : set1); +const twakeDriveISOToISO639 = twakeLang => (twakeLang === "vn" ? "vi" : twakeLang); + +const TEMPLATE_VAR_NAME = "TDCLI_TRANSLATOR_URL"; +const urlToTranslate = (text: string, to: string, from?: string) => { + const template = process.env[TEMPLATE_VAR_NAME]; + if (!template) throw new Error(`${TEMPLATE_VAR_NAME} environment variable must be set.`); + const variables = { + to: twakeDriveISOToISO639(to), + from: from ? twakeDriveISOToISO639(from) : from, + text, + }; + return template.replace(/%\{([^}:]+)(?::([^}]+))?\}/g, (_, varName, absentValue) => { + if (!(varName in variables)) + throw new Error(`Invalid variable ${JSON.stringify(varName)} in ${TEMPLATE_VAR_NAME}`); + const replacement = variables[varName] ? variables[varName] : absentValue; + if (!replacement?.length && !absentValue?.length) + throw new Error(`No value for variable ${JSON.stringify(varName)} in ${TEMPLATE_VAR_NAME}`); + return encodeURIComponent(replacement); + }); +}; + +interface TranslateArguments { + text: string; + to: string[]; + from?: string; +} + +const LanguageGroup = "Languages"; + +const command: yargs.CommandModule = { + command: "translate ", + describe: ` + Open translations for the provided english string. + Note: Do not call with unknown values as this will be injected in a command line.\n + Set ${TEMPLATE_VAR_NAME} to a translation service URL with url encoded: + - %{from:auto} and %{to} to be replaced by languages from arguments. If from is not set, use the value after ':'." + + - %{text} to be replaced by the text to translate + `.trim(), + builder: { + ...NonPlatformCommandYargsBuilder, + to: { + type: "string", + array: true, + description: "Comma separated destination languages", + choices: CORE_LANGUAGES, + default: CORE_LANGUAGES.filter(x => x != "en"), + group: LanguageGroup, + }, + from: { + type: "string", + description: "Language of the text to translate", + group: LanguageGroup, + }, + }, + handler: async argv => { + try { + const args = argv as unknown as TranslateArguments; + const to = parseYargsCommaSeparatedStringArray(args.to); + for (const lang of to) await openWithSystemViewer(urlToTranslate(args.text, lang, args.from)); + } catch (e) { + console.error(e.stack || e); + process.exit(1); + } + }, +}; +export default command; diff --git a/tdrive/backend/node/src/cli/utils/check-built-younger-than-source.ts b/tdrive/backend/node/src/cli/utils/check-built-younger-than-source.ts new file mode 100644 index 000000000..1bcd95a35 --- /dev/null +++ b/tdrive/backend/node/src/cli/utils/check-built-younger-than-source.ts @@ -0,0 +1,15 @@ +import FS from "fs/promises"; + +/** Throws if this file is more recent than it's built counter-part. */ +export const checkBuiltFileYoungerThanSource = async built => { + //TODO: Doesnt work when build fails, rebuild touches it even if failed. Does work for + // stuff watching the source file and running something before it compiled + // intented use: await checkBuiltFileYoungerThanSource(__filename); + + const latestFromStat = ({ mtimeMs, ctimeMs, birthtimeMs }) => + Math.max(mtimeMs, Math.max(ctimeMs, birthtimeMs)); + const source = built.replace(/\/dist\//, "/src/").replace(/js$/, "ts"); + const builtDate = latestFromStat(await FS.stat(built)); + const sourceDate = latestFromStat(await FS.stat(source)); + if (sourceDate > builtDate) throw new Error(`Build result is out of date on ${source}`); +}; diff --git a/tdrive/backend/node/src/cli/utils/exec-ag.ts b/tdrive/backend/node/src/cli/utils/exec-ag.ts new file mode 100644 index 000000000..e3c69a8b6 --- /dev/null +++ b/tdrive/backend/node/src/cli/utils/exec-ag.ts @@ -0,0 +1,64 @@ +import { spawnCheckingExitCode } from "../../utils/exec"; + +const runSearch = async (regex, folder, ignoreFolder?, useNull: boolean = false) => { + // const result = await spawnCheckingExitCode("ag", ["--count", "--case-sensitive", "--print0", "--fixed-strings", label, folder]); + // --ignore-dir not taken into acount .... + const result = await spawnCheckingExitCode("ag", [ + ...(useNull ? ["--print0"] : []), + ...regex, + folder, + ]); + if (result.code !== 0) { + if (result.stderr === "") return []; + throw new Error(`Unexpected result from ag: ${result.code}: ${result.stderr}${result.stdout}`); + } + return (result.stdout as string) + .split(useNull ? "\x00" : "\n") + .filter(x => x && (!ignoreFolder || !x.startsWith(ignoreFolder))); +}; + +/** Try to run `ag`, returns its version string, false if it wasn't found, and throws for any other error */ +export const getAgVersion = async () => { + try { + const result = await spawnCheckingExitCode("ag", ["--version"]); + return result.stdout; + } catch (e) { + if (e.code === "ENOENT") return false; + throw e; + } +}; + +// Poor man's regexp maker... don't put any symbols in label names ok... +export const runSearchLiteral = async (label, folder, ignoreFolder?) => + ( + await runSearch( + ["--count", "--case-sensitive", `\\b${label.replace(/\./g, "\\.")}\\b`], + folder, + ignoreFolder, + true, + ) + ).map(line => line.split(":")[0]); + +export const runRegexp = async (regex: RegExp, folder: string, ignoreFolder?: string) => + ( + await runSearch( + [regex.ignoreCase ? "--ignore-case" : "--case-sensitive", regex.source], + folder, + ignoreFolder, + ) + ) + .map(row => { + const match = /^([^:]+):([^:]+):\s*(.*)$/.exec(row); + if (!match) { + console.error(`Error matching row: ${JSON.stringify(row)}`); + return null; + } + const [_, file, _lineNumber, line] = match; + const innerMatch = regex.exec(line); + if (!innerMatch) { + console.error(`Error matching row for inner: ${JSON.stringify(row)}`); + return null; + } + return { file, line, match: innerMatch.groups?.result || innerMatch[0] }; + }) + .filter(x => x); diff --git a/tdrive/backend/node/src/cli/utils/non-plaform-command-yargs-builder.ts b/tdrive/backend/node/src/cli/utils/non-plaform-command-yargs-builder.ts new file mode 100644 index 000000000..32aeb047b --- /dev/null +++ b/tdrive/backend/node/src/cli/utils/non-plaform-command-yargs-builder.ts @@ -0,0 +1,7 @@ +/** Spread into yargs command builder to deactivate platform-specific generic functions */ +export const NonPlatformCommandYargsBuilder = { + quietConfigSummary: { + default: true, + hidden: true, + }, +}; diff --git a/tdrive/backend/node/src/cli/utils/text.ts b/tdrive/backend/node/src/cli/utils/text.ts new file mode 100644 index 000000000..417a5904c --- /dev/null +++ b/tdrive/backend/node/src/cli/utils/text.ts @@ -0,0 +1,120 @@ +import { inspect } from "util"; + +/** Similar to the arguments of `console.log` and friends, but returns a stri */ +const logToStr = (...stuff: any): string => + stuff.map((x: any) => (typeof x == "string" ? x : inspect(x))).join(" "); + +const wrap = (text: string, left: string, right = left) => left + text + right; + +export function box(line: string): string { + return [ + wrap("".padEnd(line.length + 2, "─"), "┌", "┐"), + wrap(wrap(line, " "), "│"), + wrap("".padEnd(line.length + 2, "─"), "└", "┘"), + ].join("\n"); +} + +/** make a string filestable representation of provided data */ +export function printTable( + data: any[][], + cols: string[], + withRowNumbers = true, + withCountRow = true, + cellMapper?: (cell: any, col: number, row: number) => string, +) { + if (withRowNumbers) { + cols = ["#"].concat(cols); + const maxWidth = (data.length + "").length; + data = data.map((obj, index) => ({ ...obj, "#": index.toString().padStart(maxWidth) })); + } + const strData = [cols].concat( + data.map((obj, y) => + cols.map((col, x) => + cellMapper + ? cellMapper(obj[col], x - (withRowNumbers ? 1 : 0), y) ?? "" + : obj[col] == null + ? "" + : "" + obj[col], + ), + ), + ); + const colWidths = cols.map((_, x) => + strData.reduce((acc, row) => Math.max(acc, row[x].length), 0), + ); + const asciiArtRow = (l, f, m, r) => wrap(colWidths.map(w => "".padEnd(w + 2, f)).join(m), l, r); + const output = []; + + output.push(asciiArtRow("┌", "─", "┬", "┐")); + strData.forEach((row, y) => { + output.push(wrap(row.map((cell, x) => wrap(cell.padEnd(colWidths[x]), " ")).join("│"), "│")); + if (y === 0 && strData.length > 1) output.push(asciiArtRow("├", "─", "┼", "┤")); + }); + if (withCountRow) { + output.push(asciiArtRow("├", "─", "┴", "┤")); + const count = ` ${strData.length - 1} row${strData.length == 2 ? "" : "s"} `; + output.push( + wrap( + count.padStart(colWidths.reduce((acc, w) => acc + w + 2, 0) + colWidths.length - 1), + "│", + ), + ); + output.push(asciiArtRow("└", "─", "─", "┘")); + } else output.push(asciiArtRow("└", "─", "┴", "┘")); + return output.join("\n"); +} + +/** Show progress on the same line, until a \n ends the line */ +export function startProgress(prefix: string) { + let lastLen = 0; + return (text: string, leave: boolean = false) => { + if (lastLen) process.stderr.write("\r" + new Array(lastLen + 1).join(" ")); + process.stderr.write("\r" + prefix + text + (leave ? "\n" : "\r")); + lastLen = leave || text.endsWith("\n") ? 0 : prefix.length + text.length; + }; +} + +export class IndentedPrinter { + static DefaultIndent = " "; + private readonly indents: string[] = []; + private readonly lines: string[] = []; + private depth = 0; + toString() { + return this.lines.join("\n"); + } + private pushStr(str: string) { + str + .split("\n") + .forEach(str => + this.lines.push(this.indents.join("") + str.replace(/\n/g, "\n" + this.indents.join(""))), + ); + } + push(...stuff: any) { + this.pushStr(logToStr(...stuff)); + } + private enter(prefix: string, str: string) { + this.pushStr(str); + this.indents.push(prefix); + this.depth++; + } + private leave() { + this.indents.pop(); + this.depth--; + } + private appendToPreviousStr(str: string) { + this.lines[this.lines.length - 1] = this.lines[this.lines.length - 1] + str; + } + appendToPrevious(...stuff: any) { + this.appendToPreviousStr(logToStr(...stuff)); + } + private async insideStr(prefix: string, str: string, code: () => Promise): Promise { + this.enter(prefix, str); + try { + return await code(); + } finally { + this.leave(); + } + } + async inside(prefix: string, stuff: any[], code: () => Promise): Promise { + return this.insideStr(prefix, logToStr(...stuff), code); + } +} diff --git a/tdrive/backend/node/src/utils/exec.ts b/tdrive/backend/node/src/utils/exec.ts new file mode 100644 index 000000000..27d012a59 --- /dev/null +++ b/tdrive/backend/node/src/utils/exec.ts @@ -0,0 +1,64 @@ +import util from "util"; +import { exec } from "child_process"; + +export const execPromise = util.promisify(exec); + +import OS from "os"; +const openCommandForPlatform = () => { + switch (OS.platform()) { + case "darwin": + return "open"; + case "win32": + return "start"; + case "linux": + return "xdg-open"; + default: + throw new Error(`Platform ${OS.platform()} is not supported.`); + } +}; +const veryPoorManShellEscape = str => "'" + str.replace(/'/g, "'\"'\"'") + "'"; + +export const openWithSystemViewer = (file: string) => + execPromise([openCommandForPlatform(), veryPoorManShellEscape(file)].join(" ")); + +import { spawn as spawnWithCB } from "child_process"; + +export const spawn = (cmd: string, args: string[] = []) => { + const reader = stream => + new Promise((resolve, reject) => { + const bufs = []; + let len = 0; + stream.on("error", reject); + stream.on("finish", () => { + resolve(Buffer.concat(bufs, len).toString("utf-8")); + }); + stream.on("data", data => { + bufs.push(data); + len += data.length; + }); + }); + const process = spawnWithCB(cmd, args); + const stdout = reader(process.stdout), + stderr = reader(process.stderr); + const processPromise = new Promise((resolve, reject) => { + process.on("error", reject); + process.on("close", resolve); + }); + process.stdin.end(); + return Promise.all([processPromise, stdout, stderr]).then(([code, stdout, stderr]) => ({ + code, + stdout, + stderr, + })); +}; + +export const spawnCheckingExitCode = (cmd: string, args: string[] = []) => + spawn(cmd, args).then(execResult => { + if (execResult.code != 0) + throw new Error( + `Error running ${cmd} ${JSON.stringify(args)}: exit with ${execResult.code}. Output:\n${ + execResult.stdout + }\n${execResult.stderr}`, + ); + return execResult; + });