Bulk scope test recorder (#2383)

1. Add support for new scope facets to a particular language. eg:
`scopeSupportFacets\typescript.ts`
2. Issue first command to show all unimplemented facets 
   * Select language
* New untitled document opens with all `supported` facets for the
language that are missing `.scope` files
3. Edit document with any number of fixtures
4. Issues second command to create multiple scope test fixtures

```
[[typescript]]

[command] - A command, for example Talon spoken command or bash
hello
---
```



Thoughts:
* Do we want to close the document after it's read? If yes do we have
something on the ide for this already or do I need to add it?
* I'm not one hundred percent sure on the command identifiers
* Regarding docs. Should I add a new heading in the exist [test case
recorder
file](https://github.com/cursorless-dev/cursorless/blob/ec694b754cd7169f792e6d2bb4028ab17a83bf4e/docs/contributing/test-case-recorder.md)
or should I start a new one for this format?

## Checklist

- [/] I have added
[tests](https://www.cursorless.org/docs/contributing/test-case-recorder/)
- [ ] I have updated the
[docs](https://github.com/cursorless-dev/cursorless/tree/main/docs) and
[cheatsheet](https://github.com/cursorless-dev/cursorless/tree/main/cursorless-talon/src/cheatsheet)
- [/] I have not broken the cheatsheet

---------

Co-authored-by: Pokey Rule <[email protected]>

cursorless-talon-dev/src/cursorless_dev.talon

@@ -10,6 +10,10 @@ tag: user.cursorless
     user.run_rpc_command("cursorless.recordTestCase")
 {user.cursorless_homophone} record one:
     user.run_rpc_command("cursorless.recordOneTestCaseThenPause")
+{user.cursorless_homophone} record scope:
+    user.run_rpc_command("cursorless.recordScopeTests.showUnimplementedFacets")
+{user.cursorless_homophone} save scope:
+    user.run_rpc_command("cursorless.recordScopeTests.saveActiveDocument")
 {user.cursorless_homophone} pause:
     user.run_rpc_command("cursorless.pauseRecording")
 {user.cursorless_homophone} resume:

packages/common/src/cursorlessCommandIds.ts

@@ -40,6 +40,8 @@ export const cursorlessCommandIds = [
   "cursorless.recordTestCase",
   "cursorless.recordOneTestCaseThenPause",
   "cursorless.resumeRecording",
+  "cursorless.recordScopeTests.showUnimplementedFacets",
+  "cursorless.recordScopeTests.saveActiveDocument",
   "cursorless.showCheatsheet",
   "cursorless.showDocumentation",
   "cursorless.showQuickPick",
@@ -70,6 +72,12 @@ export const cursorlessCommandDescriptions: Record<
   ["cursorless.resumeRecording"]: new VisibleCommand(
     "Resume test case recording",
   ),
+  ["cursorless.recordScopeTests.showUnimplementedFacets"]: new VisibleCommand(
+    "Bulk record unimplemented scope facets",
+  ),
+  ["cursorless.recordScopeTests.saveActiveDocument"]: new VisibleCommand(
+    "Bulk save scope tests for the active document",
+  ),
   ["cursorless.showDocumentation"]: new VisibleCommand("Show documentation"),
   ["cursorless.showScopeVisualizer"]: new VisibleCommand(
     "Show the scope visualizer",

packages/common/src/index.ts

@@ -69,6 +69,7 @@ export * from "./testUtil/shouldUpdateFixtures";
 export * from "./testUtil/TestCaseSnapshot";
 export * from "./testUtil/serializeTestFixture";
 export * from "./testUtil/asyncSafety";
+export * from "./testUtil/getScopeTestPathsRecursively";
 export * from "./util/typeUtils";
 export * from "./ide/types/hatStyles.types";
 export * from "./errors";

packages/common/src/scopeSupportFacets/languageScopeSupport.ts

@@ -1,3 +1,4 @@
+import type { StringRecord } from "../types/StringRecord";
 import { cScopeSupport } from "./c";
 import { clojureScopeSupport } from "./clojure";
 import { cppScopeSupport } from "./cpp";
@@ -27,36 +28,34 @@ import { typescriptreactScopeSupport } from "./typescriptreact";
 import { xmlScopeSupport } from "./xml";
 import { yamlScopeSupport } from "./yaml";
 
-export const languageScopeSupport: Record<
-  string,
-  LanguageScopeSupportFacetMap
-> = {
-  c: cScopeSupport,
-  clojure: clojureScopeSupport,
-  cpp: cppScopeSupport,
-  csharp: csharpScopeSupport,
-  css: cssScopeSupport,
-  go: goScopeSupport,
-  html: htmlScopeSupport,
-  java: javaScopeSupport,
-  javascript: javascriptScopeSupport,
-  javascriptreact: javascriptScopeSupport,
-  json: jsonScopeSupport,
-  jsonc: jsoncScopeSupport,
-  jsonl: jsonlScopeSupport,
-  latex: latexScopeSupport,
-  lua: luaScopeSupport,
-  markdown: markdownScopeSupport,
-  php: phpScopeSupport,
-  python: pythonScopeSupport,
-  ruby: rubyScopeSupport,
-  rust: rustScopeSupport,
-  scala: scalaScopeSupport,
-  scm: scmScopeSupport,
-  scss: scssScopeSupport,
-  talon: talonScopeSupport,
-  typescript: typescriptScopeSupport,
-  typescriptreact: typescriptreactScopeSupport,
-  xml: xmlScopeSupport,
-  yaml: yamlScopeSupport,
-};
+export const languageScopeSupport: StringRecord<LanguageScopeSupportFacetMap> =
+  {
+    c: cScopeSupport,
+    clojure: clojureScopeSupport,
+    cpp: cppScopeSupport,
+    csharp: csharpScopeSupport,
+    css: cssScopeSupport,
+    go: goScopeSupport,
+    html: htmlScopeSupport,
+    java: javaScopeSupport,
+    javascript: javascriptScopeSupport,
+    javascriptreact: javascriptScopeSupport,
+    json: jsonScopeSupport,
+    jsonc: jsoncScopeSupport,
+    jsonl: jsonlScopeSupport,
+    latex: latexScopeSupport,
+    lua: luaScopeSupport,
+    markdown: markdownScopeSupport,
+    php: phpScopeSupport,
+    python: pythonScopeSupport,
+    ruby: rubyScopeSupport,
+    rust: rustScopeSupport,
+    scala: scalaScopeSupport,
+    scm: scmScopeSupport,
+    scss: scssScopeSupport,
+    talon: talonScopeSupport,
+    typescript: typescriptScopeSupport,
+    typescriptreact: typescriptreactScopeSupport,
+    xml: xmlScopeSupport,
+    yaml: yamlScopeSupport,
+  };

packages/common/src/testUtil/getScopeTestPathsRecursively.ts

@@ -0,0 +1,101 @@
+import { readFileSync } from "node:fs";
+import { groupBy, type Dictionary } from "lodash";
+import {
+  getScopeTestConfigPaths,
+  getScopeTestPaths,
+  type ScopeTestPath,
+} from "./getFixturePaths";
+
+export interface ScopeTestConfig {
+  imports?: string[];
+  skip?: boolean;
+}
+
+export function getScopeTestPathsRecursively(): ScopeTestPath[] {
+  const configPaths = getScopeTestConfigPaths();
+  const configs = readConfigFiles(configPaths);
+  const testPathsRaw = getScopeTestPaths();
+  const languagesRaw = groupBy(testPathsRaw, (test) => test.languageId);
+  const result: ScopeTestPath[] = [];
+
+  // Languages without any tests still needs to be included in case they have an import
+  for (const languageId of Object.keys(configs)) {
+    if (!languagesRaw[languageId]) {
+      languagesRaw[languageId] = [];
+    }
+  }
+
+  for (const languageId of Object.keys(languagesRaw)) {
+    const config = configs[languageId];
+
+    // This 'language' only exists to be imported by other
+    if (config?.skip) {
+      continue;
+    }
+
+    const testPathsForLanguage: ScopeTestPath[] = [];
+    addTestPathsForLanguageRecursively(
+      languagesRaw,
+      configs,
+      testPathsForLanguage,
+      new Set(),
+      languageId,
+    );
+    for (const test of testPathsForLanguage) {
+      const name =
+        languageId === test.languageId
+          ? test.name
+          : `${test.name.replace(`/${test.languageId}/`, `/${languageId}/`)} (${test.languageId})`;
+      result.push({
+        ...test,
+        languageId,
+        name,
+      });
+    }
+  }
+
+  return result;
+}
+
+function addTestPathsForLanguageRecursively(
+  languages: Dictionary<ScopeTestPath[]>,
+  configs: Record<string, ScopeTestConfig | undefined>,
+  result: ScopeTestPath[],
+  usedLanguageIds: Set<string>,
+  languageId: string,
+): void {
+  if (usedLanguageIds.has(languageId)) {
+    return;
+  }
+
+  if (!languages[languageId]) {
+    throw Error(`No test paths found for language ${languageId}`);
+  }
+
+  result.push(...languages[languageId]);
+  usedLanguageIds.add(languageId);
+
+  const config = configs[languageId];
+  const importLanguageIds = config?.imports ?? [];
+
+  for (const langImport of importLanguageIds) {
+    addTestPathsForLanguageRecursively(
+      languages,
+      configs,
+      result,
+      usedLanguageIds,
+      langImport,
+    );
+  }
+}
+
+function readConfigFiles(
+  configPaths: { languageId: string; path: string }[],
+): Record<string, ScopeTestConfig> {
+  const result: Record<string, ScopeTestConfig> = {};
+  for (const p of configPaths) {
+    const content = readFileSync(p.path, "utf8");
+    result[p.languageId] = JSON.parse(content) as ScopeTestConfig;
+  }
+  return result;
+}

packages/common/src/types/StringRecord.ts

@@ -0,0 +1 @@
+export type StringRecord<T> = Partial<Record<string, T>>;

packages/cursorless-engine/src/ScopeTestRecorder.ts

@@ -0,0 +1,138 @@
+import {
+  ScopeSupportFacetLevel,
+  getScopeTestPathsRecursively,
+  getScopeTestsDirPath,
+  groupBy,
+  languageScopeSupport,
+  scopeSupportFacetInfos,
+  showInfo,
+  type IDE,
+  type ScopeSupportFacet,
+} from "@cursorless/common";
+import * as fs from "node:fs";
+import * as fsPromises from "node:fs/promises";
+import * as path from "node:path";
+
+export class ScopeTestRecorder {
+  constructor(private ide: IDE) {
+    this.showUnimplementedFacets = this.showUnimplementedFacets.bind(this);
+    this.saveActiveDocument = this.saveActiveDocument.bind(this);
+  }
+
+  async showUnimplementedFacets() {
+    const languageId = await this.languageSelection();
+
+    if (languageId == null) {
+      return;
+    }
+
+    const supportedScopeFacets = getSupportedScopeFacets(languageId);
+    const existingScopeTestFacets = getExistingScopeFacetTest(languageId);
+
+    const missingScopeFacets = supportedScopeFacets.filter(
+      (facet) => !existingScopeTestFacets.has(facet),
+    );
+
+    let currentSnippetPlaceholder = 1;
+    const missingScopeFacetRows = missingScopeFacets.map(
+      (facet) =>
+        `[${facet}] - ${scopeSupportFacetInfos[facet].description}\n$${currentSnippetPlaceholder++}\n---\n`,
+    );
+    const header = `[[${languageId}]]\n\n`;
+    const snippetText = `${header}${missingScopeFacetRows.join("\n")}`;
+
+    const editor = await this.ide.openUntitledTextDocument({
+      language: "markdown",
+    });
+
+    const editableEditor = this.ide.getEditableTextEditor(editor);
+    await editableEditor.insertSnippet(snippetText);
+  }
+
+  async saveActiveDocument() {
+    const text = this.ide.activeTextEditor?.document.getText() ?? "";
+    const matchLanguageId = text.match(/^\[\[(\w+)\]\]\n/);
+
+    if (matchLanguageId == null) {
+      throw Error(`Can't match language id`);
+    }
+
+    const languageId = matchLanguageId[1];
+    const restText = text.slice(matchLanguageId[0].length);
+
+    const parts = restText
+      .split(/^---$/gm)
+      .map((p) => p.trimStart())
+      .filter(Boolean);
+
+    const facetsToAdd: { facet: string; content: string }[] = [];
+
+    for (const part of parts) {
+      const match = part.match(/^\[([\w.]+)\].*\n([\s\S]*)$/);
+      const facet = match?.[1];
+      const content = match?.[2] ?? "";
+
+      if (facet == null) {
+        throw Error(`Invalid pattern '${part}'`);
+      }
+
+      if (!content.trim()) {
+        continue;
+      }
+
+      facetsToAdd.push({ facet, content });
+    }
+
+    const langDirectory = path.join(getScopeTestsDirPath(), languageId);
+
+    await fsPromises.mkdir(langDirectory, { recursive: true });
+
+    for (const { facet, content } of facetsToAdd) {
+      const fullContent = `${content}---\n`;
+      let filePath = path.join(langDirectory, `${facet}.scope`);
+      let i = 2;
+
+      while (fs.existsSync(filePath)) {
+        filePath = path.join(langDirectory, `${facet}${i++}.scope`);
+      }
+
+      await fsPromises.writeFile(filePath, fullContent, "utf-8");
+    }
+
+    await showInfo(
+      this.ide.messages,
+      "scopeTestsSaved",
+      `${facetsToAdd.length} scope tests saved for language '${languageId}`,
+    );
+  }
+
+  private languageSelection() {
+    const languageIds = Object.keys(languageScopeSupport);
+    languageIds.sort();
+    return this.ide.showQuickPick(languageIds, {
+      title: "Select language to record scope tests for",
+    });
+  }
+}
+
+function getSupportedScopeFacets(languageId: string): ScopeSupportFacet[] {
+  const scopeSupport = languageScopeSupport[languageId];
+
+  if (scopeSupport == null) {
+    throw Error(`Missing scope support for language '${languageId}'`);
+  }
+
+  const scopeFacets = Object.keys(scopeSupport) as ScopeSupportFacet[];
+
+  return scopeFacets.filter(
+    (facet) => scopeSupport[facet] === ScopeSupportFacetLevel.supported,
+  );
+}
+
+function getExistingScopeFacetTest(languageId: string): Set<string> {
+  const testPaths = getScopeTestPathsRecursively();
+  const languages = groupBy(testPaths, (test) => test.languageId);
+  const testPathsForLanguage = languages.get(languageId) ?? [];
+  const facets = testPathsForLanguage.map((test) => test.facet);
+  return new Set(facets);
+}

packages/cursorless-engine/src/index.ts

@@ -11,3 +11,4 @@ export * from "./CommandRunner";
 export * from "./CommandHistory";
 export * from "./CommandHistoryAnalyzer";
 export * from "./util/grammarHelpers";
+export * from "./ScopeTestRecorder";