Add preference to ignore files in workspace functions

packages/ai-workspace-agent/package.json

@@ -21,7 +21,9 @@
     "@theia/navigator": "1.55.0",
     "@theia/terminal": "1.55.0",
     "@theia/ai-core": "1.55.0",
-    "@theia/ai-chat": "1.55.0"
+    "@theia/ai-chat": "1.55.0",
+    "ignore": "^6.0.0",
+    "minimatch": "^9.0.0"
   },
   "publishConfig": {
     "access": "public"

packages/ai-workspace-agent/src/browser/frontend-module.ts

@@ -18,8 +18,11 @@ import { ChatAgent } from '@theia/ai-chat/lib/common';
 import { Agent, ToolProvider } from '@theia/ai-core/lib/common';
 import { WorkspaceAgent } from './workspace-agent';
 import { FileContentFunction, GetWorkspaceDirectoryStructure, GetWorkspaceFileList, WorkspaceFunctionScope } from './functions';
+import { PreferenceContribution } from '@theia/core/lib/browser';
+import { WorkspacePreferencesSchema } from './workspace-preferences';
 
 export default new ContainerModule(bind => {
+    bind(PreferenceContribution).toConstantValue({ schema: WorkspacePreferencesSchema });
     bind(WorkspaceAgent).toSelf().inSingletonScope();
     bind(Agent).toService(WorkspaceAgent);
     bind(ChatAgent).toService(WorkspaceAgent);

packages/ai-workspace-agent/src/browser/functions.ts

@@ -20,12 +20,27 @@ import { FileService } from '@theia/filesystem/lib/browser/file-service';
 import { FileStat } from '@theia/filesystem/lib/common/files';
 import { WorkspaceService } from '@theia/workspace/lib/browser';
 import { FILE_CONTENT_FUNCTION_ID, GET_WORKSPACE_DIRECTORY_STRUCTURE_FUNCTION_ID, GET_WORKSPACE_FILE_LIST_FUNCTION_ID } from '../common/functions';
+import ignore from 'ignore';
+import { Minimatch } from 'minimatch';
+import { PreferenceService } from '@theia/core/lib/browser';
+import { CONSIDER_GITIGNORE_PREF, USER_EXCLUDE_PATTERN_PREF } from './workspace-preferences';
 
 @injectable()
 export class WorkspaceFunctionScope {
+    protected readonly GITIGNORE_FILE_NAME = '.gitignore';
+
     @inject(WorkspaceService)
     protected workspaceService: WorkspaceService;
 
+    @inject(FileService)
+    protected fileService: FileService;
+
+    @inject(PreferenceService)
+    protected preferences: PreferenceService;
+
+    private gitignoreMatcher: ReturnType<typeof ignore> | undefined;
+    private gitignoreWatcherInitialized = false;
+
     async getWorkspaceRoot(): Promise<URI> {
         const wsRoots = await this.workspaceService.roots;
         if (wsRoots.length === 0) {
@@ -39,16 +54,70 @@ export class WorkspaceFunctionScope {
             throw new Error('Access outside of the workspace is not allowed');
         }
     }
-    /**
-     * Determines whether a given file or directory should be excluded from workspace operations.
-     *
-     * @param stat - The `FileStat` object representing the file or directory to check.
-     * @returns `true` if the file or directory should be excluded, `false` otherwise.
-     */
-    shouldExclude(stat: FileStat): boolean {
-        const excludedFolders = ['node_modules', 'lib'];
-        return stat.resource.path.base.startsWith('.') || excludedFolders.includes(stat.resource.path.base);
+
+    private async initializeGitignoreWatcher(workspaceRoot: URI): Promise<void> {
+        if (this.gitignoreWatcherInitialized) {
+            return;
+        }
+
+        const gitignoreUri = workspaceRoot.resolve(this.GITIGNORE_FILE_NAME);
+        this.fileService.watch(gitignoreUri);
+
+        this.fileService.onDidFilesChange(async event => {
+            if (event.contains(gitignoreUri)) {
+                this.gitignoreMatcher = undefined;
+            }
+        });
+
+        this.gitignoreWatcherInitialized = true;
     }
+
+    async shouldExclude(stat: FileStat): Promise<boolean> {
+        const shouldConsiderGitIgnore = this.preferences.get(CONSIDER_GITIGNORE_PREF, false);
+        const userExcludePatterns = this.preferences.get<string[]>(USER_EXCLUDE_PATTERN_PREF, []);
+
+        if (this.isUserExcluded(stat.resource.path.base, userExcludePatterns)) {
+            return true;
+        }
+        const workspaceRoot = await this.getWorkspaceRoot();
+        if (shouldConsiderGitIgnore && await this.isGitIgnored(stat, workspaceRoot)) {
+            return true;
+        }
+
+        return false;
+    }
+
+    protected isUserExcluded(fileName: string, userExcludePatterns: string[]): boolean {
+        return userExcludePatterns.some(pattern => new Minimatch(pattern, { dot: true }).match(fileName));
+    }
+
+    protected async isGitIgnored(stat: FileStat, workspaceRoot: URI): Promise<boolean> {
+        await this.initializeGitignoreWatcher(workspaceRoot);
+
+        const gitignoreUri = workspaceRoot.resolve(this.GITIGNORE_FILE_NAME);
+
+        try {
+            const fileStat = await this.fileService.resolve(gitignoreUri);
+            if (fileStat) {
+                if (!this.gitignoreMatcher) {
+                    const gitignoreContent = await this.fileService.read(gitignoreUri);
+                    this.gitignoreMatcher = ignore().add(gitignoreContent.value);
+                }
+                const relativePath = workspaceRoot.relative(stat.resource);
+                if (relativePath) {
+                    const relativePathStr = relativePath.toString() + (stat.isDirectory ? '/' : '');
+                    if (this.gitignoreMatcher.ignores(relativePathStr)) {
+                        return true;
+                    }
+                }
+            }
+        } catch {
+            // If .gitignore does not exist or cannot be read, continue without error
+        }
+
+        return false;
+    }
+
 }
 
 @injectable()
@@ -88,7 +157,7 @@ export class GetWorkspaceDirectoryStructure implements ToolProvider {
 
         if (stat && stat.isDirectory && stat.children) {
             for (const child of stat.children) {
-                if (!child.isDirectory || this.workspaceScope.shouldExclude(child)) { continue; };
+                if (!child.isDirectory || await this.workspaceScope.shouldExclude(child)) { continue; };
                 const path = `${prefix}${child.resource.path.base}/`;
                 result.push(path);
                 result.push(...await this.buildDirectoryStructure(child.resource, `${path}`));
@@ -225,12 +294,15 @@ export class GetWorkspaceFileList implements ToolProvider {
         const result: string[] = [];
 
         if (stat && stat.isDirectory) {
-            if (this.workspaceScope.shouldExclude(stat)) {
+            if (await this.workspaceScope.shouldExclude(stat)) {
                 return result;
             }
             const children = await this.fileService.resolve(uri);
             if (children.children) {
                 for (const child of children.children) {
+                    if (await this.workspaceScope.shouldExclude(child)) {
+                        continue;
+                    };
                     const relativePath = workspaceRootUri.relative(child.resource);
                     if (relativePath) {
                         result.push(relativePath.toString());

packages/ai-workspace-agent/src/browser/workspace-preferences.ts

@@ -0,0 +1,41 @@
+// *****************************************************************************
+// Copyright (C) 2024 EclipseSource GmbH.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import { PreferenceSchema } from '@theia/core/lib/browser/preferences/preference-contribution';
+
+export const CONSIDER_GITIGNORE_PREF = 'ai-features.workspaceFunctions.considerGitIgnore';
+export const USER_EXCLUDE_PATTERN_PREF = 'ai-features.workspaceFunctions.userExcludes';
+
+export const WorkspacePreferencesSchema: PreferenceSchema = {
+    type: 'object',
+    properties: {
+        [CONSIDER_GITIGNORE_PREF]: {
+            type: 'boolean',
+            title: 'Consider .gitignore',
+            description: 'If enabled, excludes files/folders specified in a global .gitignore file (expected location is the workspace root).',
+            default: false
+        },
+        [USER_EXCLUDE_PATTERN_PREF]: {
+            type: 'array',
+            title: 'Excluded File Patterns',
+            description: 'List of patterns (glob or regex) for files/folders to exclude.',
+            default: ['node_modules', 'lib', '.*'],
+            items: {
+                type: 'string'
+            }
+        }
+    }
+};

yarn.lock

@@ -6953,6 +6953,11 @@ ignore@^5.0.4, ignore@^5.2.0, ignore@^5.2.4:
   resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.1.tgz#5073e554cd42c5b33b394375f538b8593e34d4ef"
   integrity sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==
 
+ignore@^6.0.0:
+  version "6.0.2"
+  resolved "https://registry.yarnpkg.com/ignore/-/ignore-6.0.2.tgz#77cccb72a55796af1b6d2f9eb14fa326d24f4283"
+  integrity sha512-InwqeHHN2XpumIkMvpl/DCJVrAHgCsG5+cn1XlnLWGwtZBm8QJfSusItfrwx81CTp5agNZqpKU2J/ccC5nGT4A==
+
 image-size@~0.5.0:
   version "0.5.5"
   resolved "https://registry.yarnpkg.com/image-size/-/image-size-0.5.5.tgz#09dfd4ab9d20e29eb1c3e80b8990378df9e3cb9c"