Go 1.19 and generics

.github/workflows/ci.yaml

@@ -36,11 +36,17 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.17
+          go-version: 1.19
+      - name: Init API module
+        working-directory: go
+        run: |
+          sudo apt update -qq && sudo apt install -yqq libdevmapper-dev
+          go mod init github.com/sysflow-telemetry/sf-apis/go || true
+          go mod tidy
       - name: Lint go API
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.47.1
+          version: v1.51.1
           working-directory: go
           args: --disable=errcheck --build-tags=exclude_graphdriver_btrfs
   lint-python-api:

CHANGELOG.md

@@ -15,12 +15,17 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 ## [Unreleased]
 
-## [0.5.2] - 2023-07-25
+## [0.6.0] - 2023-09-30
 
 ### Changed
 
 - Make Python packages globally installed in base sfnb notebook
+- Go 1.19 and refactor of plugin interfaces to support generics
 
+### Security
+
+- CVE-2022-32149: Denial of service in golang.org/x/text/language (updated to 0.3.8)
+- CVE-2022-29526: golang.org/x/sys/unix has Incorrect privilege reporting in syscall (updated to 0.0.0-20220412211240-33da011f77ad)
 
 ## [0.5.1] - 2023-06-07
 
@@ -221,8 +226,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 - First release candidate with basic set of SysFlow APIs (C++ and Python).
 
-[Unreleased]: https://github.com/sysflow-telemetry/sf-apis/compare/0.5.2...HEAD
-[0.5.2]: https://github.com/sysflow-telemetry/sf-apis/compare/0.5.1...0.5.2
+[Unreleased]: https://github.com/sysflow-telemetry/sf-apis/compare/0.6.0...HEAD
+[0.6.0]: https://github.com/sysflow-telemetry/sf-apis/compare/0.5.1...0.6.0
 [0.5.1]: https://github.com/sysflow-telemetry/sf-apis/compare/0.5.0...0.5.1
 [0.5.0]: https://github.com/sysflow-telemetry/sf-apis/compare/0.4.4...0.5.0
 [0.4.4]: https://github.com/sysflow-telemetry/sf-apis/compare/0.4.3...0.4.4

README.md

@@ -6,7 +6,7 @@
 
 # Supported tags and respective `Dockerfile` links
 
--	[`0.5.2`, `latest`](https://github.com/sysflow-telemetry/sf-apis/tree/0.5.2), [`edge`](https://github.com/sysflow-telemetry/sf-apis/tree/master), [`dev`](https://github.com/sysflow-telemetry/sf-apis/tree/dev)
+-	[`0.6.0`, `latest`](https://github.com/sysflow-telemetry/sf-apis/tree/0.6.0), [`edge`](https://github.com/sysflow-telemetry/sf-apis/tree/master), [`dev`](https://github.com/sysflow-telemetry/sf-apis/tree/dev)
 
 # Quick reference
 

go/go.mod

@@ -1,6 +1,6 @@
 module github.com/sysflow-telemetry/sf-apis/go
 
-go 1.17
+go 1.19
 
 require (
 	github.com/actgardner/gogen-avro/v7 v7.3.1
@@ -22,9 +22,9 @@ require (
 	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/subosito/gotenv v1.2.0 // indirect
-	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
-	golang.org/x/text v0.3.7 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect	
+	golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
+	golang.org/x/text v0.3.8 // indirect
 	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
 	gopkg.in/ini.v1 v1.66.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

go/ioutils/ioutils.go

@@ -21,40 +21,90 @@
 package ioutils
 
 import (
-	"io/ioutil"
 	"os"
 	"path/filepath"
 )
 
 // ListFilePaths lists file paths with extension fileExt in path if
 // path is a valid directory, otherwise, it returns path if path is
 // a valid path and has extension fileExt.
-func ListFilePaths(path string, fileExt string) ([]string, error) {
+func ListFilePaths(path string, fileExts ...string) ([]string, error) {
 	var paths []string
 	if fi, err := os.Stat(path); os.IsNotExist(err) {
 		return paths, err
 	} else if fi.IsDir() {
-		var files []os.FileInfo
+		var entries []os.DirEntry
 		var err error
-		if files, err = ioutil.ReadDir(path); err != nil {
+		if entries, err = os.ReadDir(path); err != nil {
 			return paths, err
 		}
-		for _, file := range files {
-			if filepath.Ext(file.Name()) == fileExt {
-				f := path + "/" + file.Name()
+		for _, entry := range entries {
+			if entry.IsDir() {
+				continue
+			}
+			if len(fileExts) > 0 {
+				for _, fileExt := range fileExts {
+					if filepath.Ext(entry.Name()) == fileExt {
+						f := path + "/" + entry.Name()
+						paths = append(paths, f)
+					}
+				}
+			} else {
+				f := path + "/" + entry.Name()
 				paths = append(paths, f)
 			}
 		}
-		return paths, nil
 	} else {
-		if filepath.Ext(path) == fileExt {
-			return append(paths, path), nil
+		for _, fileExt := range fileExts {
+			if filepath.Ext(path) == fileExt {
+				return append(paths, path), nil
+			}
+		}
+	}
+	return paths, nil
+}
+
+// ListRecursiveFilePaths recursively lists file paths with extension
+// fileExt in path if path is a valid directory, otherwise, it returns
+// path if path is a valid path and has extension fileExt.
+func ListRecursiveFilePaths(path string, fileExts ...string) ([]string, error) {
+	var paths []string
+	if fi, err := os.Stat(path); os.IsNotExist(err) {
+		return paths, err
+	} else if fi.IsDir() {
+		err := filepath.Walk(path,
+			func(p string, info os.FileInfo, err error) error {
+				if err != nil {
+					return err
+				}
+				if info.IsDir() {
+					return nil
+				}
+				if len(fileExts) > 0 {
+					for _, fileExt := range fileExts {
+						if filepath.Ext(info.Name()) == fileExt {
+							paths = append(paths, p)
+						}
+					}
+				} else {
+					paths = append(paths, p)
+				}
+				return nil
+			})
+		if err != nil {
+			return paths, err
+		}
+	} else {
+		for _, fileExt := range fileExts {
+			if filepath.Ext(path) == fileExt {
+				return append(paths, path), nil
+			}
 		}
-		return paths, nil
 	}
+	return paths, nil
 }
 
-//FileExists checks whether a file exists and whether it is a directory.
+// FileExists checks whether a file exists and whether it is a directory.
 func FileExists(filename string) (bool, bool) {
 	info, err := os.Stat(filename)
 	if os.IsNotExist(err) {

go/logger/logger.go

@@ -23,7 +23,6 @@ package logger
 import (
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"os"
 	"strings"
@@ -42,6 +41,9 @@ const (
 	QUIET
 )
 
+// Perf logger string.
+const perf string = "Perf"
+
 func (d LogLevel) String() string {
 	return [...]string{"Trace", "Info", "Warn", "Error", "Health", "Quiet"}[d]
 }
@@ -62,6 +64,7 @@ var (
 	Warn   *log.Logger
 	Error  *log.Logger
 	Health *log.Logger
+	Perf   *log.Logger
 )
 
 // InitLoggers initialize utility loggers with default i/o streams.
@@ -70,17 +73,17 @@ func InitLoggers(level LogLevel) {
 	case TRACE:
 		initLoggers(os.Stdout, os.Stdout, os.Stdout, os.Stderr, os.Stdout)
 	case INFO:
-		initLoggers(ioutil.Discard, os.Stdout, os.Stdout, os.Stderr, os.Stdout)
+		initLoggers(io.Discard, os.Stdout, os.Stdout, os.Stderr, os.Stdout)
 	case WARN:
-		initLoggers(ioutil.Discard, ioutil.Discard, os.Stdout, os.Stderr, os.Stdout)
+		initLoggers(io.Discard, io.Discard, os.Stdout, os.Stderr, os.Stdout)
 	case ERROR:
-		initLoggers(ioutil.Discard, ioutil.Discard, ioutil.Discard, os.Stderr, os.Stdout)
+		initLoggers(io.Discard, io.Discard, io.Discard, os.Stderr, os.Stdout)
 	case HEALTH:
-		initLoggers(ioutil.Discard, ioutil.Discard, ioutil.Discard, ioutil.Discard, os.Stdout)
+		initLoggers(io.Discard, io.Discard, io.Discard, io.Discard, os.Stdout)
 	case QUIET:
-		initLoggers(ioutil.Discard, ioutil.Discard, ioutil.Discard, ioutil.Discard, ioutil.Discard)
+		initLoggers(io.Discard, io.Discard, io.Discard, io.Discard, io.Discard)
 	default:
-		initLoggers(ioutil.Discard, os.Stdout, os.Stdout, os.Stderr, os.Stdout)
+		initLoggers(io.Discard, os.Stdout, os.Stdout, os.Stderr, os.Stdout)
 	}
 }
 
@@ -110,4 +113,24 @@ func initLoggers(
 	Health = log.New(healthHandle,
 		fmt.Sprintf("[%s] ", HEALTH),
 		log.Ldate|log.Ltime|log.Lshortfile)
+
+	SetPerfLogger(false)
+}
+
+// SetPerfLogger changes the state of the performance logger. This logger is independent of log levels (default: disabled).
+func SetPerfLogger(enabled bool) {
+	var iowriter io.Writer
+	if enabled {
+		iowriter = os.Stdout
+	} else {
+		iowriter = io.Discard
+	}
+	Perf = log.New(iowriter,
+		fmt.Sprintf("[%s] ", perf),
+		log.Ldate|log.Ltime|log.Lshortfile)
+}
+
+// IsEnabled checks whether a logger is enabled.
+func IsEnabled(logger *log.Logger) bool {
+	return logger != nil && logger.Writer() != io.Discard
 }

go/plugins/pipeline.go

@@ -32,6 +32,6 @@ type SFPipeline interface {
 	GetNumProcessors() int
 	GetNumHandlers() int
 	GetPluginCache() SFPluginCache
-	GetChannel(name string)(interface{}, error)
+	GetChannel(name string) (interface{}, error)
 	Print()
 }

go/plugins/types.go

@@ -38,12 +38,7 @@ type CtxSysFlow struct {
 	GraphletID uint64
 }
 
-// CtxSFChannel defines a Contextual SysFlow channel for data transfer.
-type CtxSFChannel struct {
-	In chan *CtxSysFlow
-}
-
-// SFChannel defines a SysFlow channel for data transfer.
-type SFChannel struct {
-	In chan *sfgo.SysFlow
+// Channel type
+type Channel[R any] struct {
+	In chan R
 }

go/secrets/secrets.go

@@ -23,7 +23,6 @@ package secrets
 import (
 	"encoding/base64"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"strings"
 
@@ -76,7 +75,7 @@ func (s *Secrets) read(secret string) (string, error) {
 	if v, ok := s.secrets[secret]; ok {
 		return v, nil
 	}
-	buf, err := ioutil.ReadFile(s.secretsDir + "/" + secret)
+	buf, err := os.ReadFile(s.secretsDir + "/" + secret)
 	if err != nil {
 		return sfgo.Zeros.String, fmt.Errorf("secret %s does not exist or cannot be read: %v", secret, err)
 	}
@@ -88,9 +87,9 @@ func (s *Secrets) read(secret string) (string, error) {
 // Checks if the given path is a directory. Returns nil if directory.
 func isDir(path string) error {
 	if fi, err := os.Stat(path); os.IsNotExist(err) {
-		return fmt.Errorf("Path %s not found", path)
+		return fmt.Errorf("path %s not found", path)
 	} else if !fi.Mode().IsDir() {
-		return fmt.Errorf("Path %s is not a directory", path)
+		return fmt.Errorf("path %s is not a directory", path)
 	}
 	return nil
 }

py3/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = sysflow-tools
-version = 0.5.2
+version = 0.6.0-rc1
 description = SysFlow APIs and utilities
 long_description = file:README.md
 long_description_content_type = text/markdown