feat: invalidate persisted cache

application/di/init.go

@@ -1,5 +1,5 @@
 /*
- * © 2022-2024 Snyk Limited
+ * © 2024 Snyk Limited
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

application/server/server.go

@@ -19,6 +19,7 @@ package server
 import (
 	"context"
 	"fmt"
+	"github.com/snyk/snyk-ls/domain/snyk/persistence"
 	"os"
 	"runtime"
 	"strings"
@@ -187,8 +188,13 @@ func workspaceDidChangeWorkspaceFoldersHandler(srv *jrpc2.Server) jrpc2.Handler
 
 		logger.Info().Msg("RECEIVING")
 		defer logger.Info().Msg("SENDING")
-		workspace.Get().ChangeWorkspaceFolders(bgCtx, params)
-		command.HandleFolders(bgCtx, srv, di.Notifier())
+		changedFolders := workspace.Get().ChangeWorkspaceFolders(params)
+		command.HandleFolders(bgCtx, srv, di.Notifier(), di.ScanPersister())
+		if config.CurrentConfig().IsAutoScanEnabled() {
+			for _, f := range changedFolders {
+				f.ScanFolder(ctx)
+			}
+		}
 		return nil, nil
 	})
 }
@@ -386,6 +392,12 @@ func initializedHandler(srv *jrpc2.Server) handler.Func {
 			logger.Error().Err(err).Msg("Scan initialization error, canceling scan")
 			return nil, nil
 		}
+		command.HandleFolders(context.Background(), srv, di.Notifier(), di.ScanPersister())
+
+		// Check once for expired cache in same thread before triggering a scan.
+		// Start a periodic go routine to check for the expired cache afterwards
+		deleteExpiredCache()
+		go periodicallyCheckForExpiredCache()
 
 		autoScanEnabled := c.IsAutoScanEnabled()
 		if autoScanEnabled {
@@ -400,11 +412,26 @@ func initializedHandler(srv *jrpc2.Server) handler.Func {
 		}
 
 		logger.Debug().Msg("trying to get trusted status for untrusted folders")
-		go command.HandleFolders(context.Background(), srv, di.Notifier())
 		return nil, nil
 	})
 }
 
+func deleteExpiredCache() {
+	w := workspace.Get()
+	var folderList []string
+	for _, f := range w.Folders() {
+		folderList = append(folderList, f.Path())
+	}
+	di.ScanPersister().Clear(folderList, true)
+}
+
+func periodicallyCheckForExpiredCache() {
+	for {
+		deleteExpiredCache()
+		time.Sleep(time.Duration(persistence.ExpirationInSeconds) * time.Second)
+	}
+}
+
 func addWorkspaceFolders(c *config.Config, params types.InitializeParams, w *workspace.Workspace) {
 	const method = "addWorkspaceFolders"
 	if len(params.WorkspaceFolders) > 0 {
@@ -545,8 +572,8 @@ func textDocumentDidOpenHandler() jrpc2.Handler {
 			di.Notifier().Send(diagnosticParams)
 		}
 
-		if scanner, ok := di.Scanner().(scanner.PackageScanner); ok {
-			scanner.ScanPackages(context.Background(), config.CurrentConfig(), filePath, "")
+		if sc, ok := di.Scanner().(scanner.PackageScanner); ok {
+			sc.ScanPackages(context.Background(), config.CurrentConfig(), filePath, "")
 		}
 		return nil, nil
 	})

domain/ide/command/folder_handler.go

@@ -19,6 +19,7 @@ package command
 import (
 	"context"
 	"fmt"
+	"github.com/snyk/snyk-ls/domain/snyk/persistence"
 	gitconfig "github.com/snyk/snyk-ls/internal/git_config"
 	noti "github.com/snyk/snyk-ls/internal/notification"
 
@@ -32,8 +33,9 @@ import (
 const DoTrust = "Trust folders and continue"
 const DontTrust = "Don't trust folders"
 
-func HandleFolders(ctx context.Context, srv types.Server, notifier noti.Notifier) {
+func HandleFolders(ctx context.Context, srv types.Server, notifier noti.Notifier, persister persistence.ScanSnapshotPersister) {
 	go sendFolderConfigsNotification(notifier)
+	initScanPersister(persister)
 	HandleUntrustedFolders(ctx, srv)
 }
 
@@ -53,6 +55,19 @@ func sendFolderConfigsNotification(notifier noti.Notifier) {
 	notifier.Send(folderConfigsParam)
 }
 
+func initScanPersister(persister persistence.ScanSnapshotPersister) {
+	logger := config.CurrentConfig().Logger().With().Str("method", "initScanPersister").Logger()
+	w := workspace.Get()
+	var folderList []string
+	for _, f := range w.Folders() {
+		folderList = append(folderList, f.Path())
+	}
+	err := persister.Init(folderList)
+	if err != nil {
+		logger.Error().Err(err).Msg("could not initialize scan persister")
+	}
+}
+
 func HandleUntrustedFolders(ctx context.Context, srv types.Server) {
 	w := workspace.Get()
 	// debounce requests from overzealous clients (Eclipse, I'm looking at you)

domain/ide/workspace/workspace.go

@@ -174,18 +174,17 @@ func (w *Workspace) ScanWorkspace(ctx context.Context) {
 
 // ChangeWorkspaceFolders clears the "Removed" folders, adds the "New" folders,
 // and starts an automatic scan if auto-scans are enabled.
-func (w *Workspace) ChangeWorkspaceFolders(ctx context.Context, params types.DidChangeWorkspaceFoldersParams) {
+func (w *Workspace) ChangeWorkspaceFolders(params types.DidChangeWorkspaceFoldersParams) []*Folder {
 	for _, folder := range params.Event.Removed {
 		w.RemoveFolder(uri.PathFromUri(folder.Uri))
 	}
-
+	var changedWorkspaceFolders []*Folder
 	for _, folder := range params.Event.Added {
 		f := NewFolder(w.c, uri.PathFromUri(folder.Uri), folder.Name, w.scanner, w.hoverService, w.scanNotifier, w.notifier, w.scanPersister)
 		w.AddFolder(f)
-		if config.CurrentConfig().IsAutoScanEnabled() {
-			f.ScanFolder(ctx)
-		}
+		changedWorkspaceFolders = append(changedWorkspaceFolders, f)
 	}
+	return changedWorkspaceFolders
 }
 
 func (w *Workspace) Clear() {

domain/ide/workspace/workspace_test.go

@@ -73,7 +73,7 @@ func Test_TrustFoldersAndScan_shouldAddFoldersToTrustedFoldersAndTriggerScan(t *
 	}, time.Second, time.Millisecond, "scanner should be called after trust is granted")
 }
 
-func Test_AddAndRemoveFoldersAndTriggerScan(t *testing.T) {
+func Test_AddAndRemoveFoldersAndReturnFolderList(t *testing.T) {
 	c := testutil.UnitTest(t)
 	const trustedDummy = "trustedDummy"
 	const untrustedDummy = "untrustedDummy"
@@ -101,14 +101,10 @@ func Test_AddAndRemoveFoldersAndTriggerScan(t *testing.T) {
 		},
 	}}
 
-	w.ChangeWorkspaceFolders(context.Background(), params)
-
+	folderList := w.ChangeWorkspaceFolders(params)
 	assert.Nil(t, w.GetFolderContaining(toBeRemoved))
 
-	// one call for one trusted folder
-	assert.Eventually(t, func() bool {
-		return sc.Calls() == 1
-	}, time.Second, time.Millisecond, "scanner should be called after trust is granted")
+	assert.Len(t, folderList, 2)
 }
 
 func Test_Get(t *testing.T) {

domain/snyk/persistence/file_operation.go

@@ -0,0 +1,129 @@
+/*
+ * © 2024 Snyk Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package persistence
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/snyk/snyk-ls/domain/snyk"
+	"github.com/snyk/snyk-ls/internal/product"
+	"github.com/snyk/snyk-ls/internal/vcs"
+)
+
+func (g *GitPersistenceProvider) persistToDisk(cacheDir string, folderHashedPath hashedFolderPath, commitHash string, p product.Product, inputToCache []snyk.Issue) error {
+	filePath := getLocalFilePath(cacheDir, folderHashedPath, commitHash, p)
+	data, err := json.Marshal(inputToCache)
+	if err != nil {
+		return err
+	}
+	g.logger.Debug().Msg("persisting scan results in file " + filePath)
+	return os.WriteFile(filePath, data, 0644)
+}
+
+func (g *GitPersistenceProvider) getPersistedFiles(cacheDir string) (persistedFiles []string, err error) {
+	entries, err := os.ReadDir(cacheDir)
+	if err != nil {
+		return persistedFiles, err
+	}
+
+	for _, entry := range entries {
+		fileName := entry.Name()
+		if !strings.HasSuffix(fileName, ".json") {
+			continue
+		}
+		s := strings.Split(fileName, ".")
+		if len(s) == 5 {
+			persistedFiles = append(persistedFiles, fileName)
+		}
+	}
+	return persistedFiles, nil
+}
+
+func (g *GitPersistenceProvider) ensureCacheDirExists(folderPath string) (string, error) {
+	g.logger.Debug().Msg("attempting to determine .git folder path")
+	cacheDir, err := g.snykCacheDir(folderPath)
+	if err != nil {
+		return "", err
+	}
+
+	if _, err = os.Stat(cacheDir); os.IsNotExist(err) {
+		err = os.Mkdir(cacheDir, 0700)
+		if err != nil {
+			return "", err
+		}
+	}
+	return cacheDir, nil
+}
+
+func (g *GitPersistenceProvider) snykCacheDir(folderPath string) (string, error) {
+	gitFolder, err := vcs.GitRepoFolderPath(g.logger, folderPath)
+	if err != nil {
+		return "", err
+	}
+
+	cacheDir := filepath.Join(gitFolder, CacheFolder)
+
+	return cacheDir, nil
+}
+
+func getLocalFilePath(cacheDir string, folderPathHash hashedFolderPath, commitHash string, p product.Product) string {
+	productName := p.ToProductCodename()
+	return filepath.Join(cacheDir, fmt.Sprintf("%s.%s.%s.%s.json", SchemaVersion, folderPathHash, commitHash, productName))
+}
+
+func (g *GitPersistenceProvider) snapshotExistsOnDisk(cacheDir string, hash hashedFolderPath, commitHash string, p product.Product) bool {
+	filePath := getLocalFilePath(cacheDir, hash, commitHash, p)
+	if _, err := os.Stat(filePath); os.IsNotExist(err) {
+		return false
+	}
+	return true
+}
+
+func (g *GitPersistenceProvider) deleteFile(fullPath string) error {
+	g.logger.Debug().Msgf("deleting cached scan file %s", fullPath)
+	err := os.Remove(fullPath)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (g *GitPersistenceProvider) isExpired(schemaVersion string, fullFilePath string) bool {
+	// if file has incorrect schema we just delete it
+	if schemaVersion != SchemaVersion {
+		return true
+	}
+
+	// Check last modified date
+	fileInfo, err := os.Stat(fullFilePath)
+	if err != nil {
+		g.logger.Error().Err(err).Msg("couldn't stat file " + fullFilePath)
+		return true
+	}
+
+	// If elapsed time is > ExpirationInSeconds, delete the file
+	if time.Since(fileInfo.ModTime()) > time.Duration(ExpirationInSeconds)*time.Second {
+		return true
+	}
+
+	return false
+}