Merge pull request #47 from basemind-ai/e2e-setup

feat: E2E testing setup

README.md

@@ -116,3 +116,18 @@ You will need to add the following files:
 We use gRPC and protobuf files. The proto files are located under the `proto` folder and the generated code is stored
 under the `gen` folder. We use the [buf](https://buf.build/product/cli) cli tool to generate the code, as well as lint
 and format the proto files (done via pre-commit).
+
+## Testing
+
+To test the project end-to-end do the following:
+
+1. If you haven't migrated the database, begin by executing `task migrations:apply`.
+2. Seed the database by executing `task e2e:seed`. You might want to first clean the database of previous
+   data by executing `task e2e:clean` before hand, but this is optional.
+3. The `e2e:seed` command will log into the terminal the applicationId for the application that has been created.
+   Copy this value into you clipboard, and now execute `task e2e:create-jwt <applicationId>`, where <applicationId> is
+   the value you copied to your clipboard.
+4. The `e2e:create-jwt` command will print an encoded JWT token into the terminal, copy this value and save it.
+5. Bring up the docker services with `docker compose up --build`.
+6. Once the docker services are up you can test using postman. Load the proto file for the `api-gateway` service in postman,
+   and set the metadata header for authorization in the following format - key `authorization`, value `bearer <jwt-token>`.

Taskfile.yaml

@@ -115,3 +115,18 @@ tasks:
         desc: Format the protobuf files
         cmds:
             - npx buf format
+    # testing
+    e2e:seed:
+        desc: Seed the local database with testing data
+        cmds:
+            - task: db:up
+            - go run e2e/main.go seed
+    e2e:clean:
+        desc: Clean the local database of all testing data
+        cmds:
+            - task: db:up
+            - go run e2e/main.go clean
+    e2e:create-jwt:
+        desc: Create a JWT from the given applicationId arg
+        cmds:
+            - go run e2e/main.go create-jwt {{.CLI_ARGS}}

e2e/cmd/clean.go

@@ -0,0 +1,58 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/basemind-ai/monorepo/shared/go/db"
+	"github.com/rs/zerolog/log"
+	"strings"
+
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	rootCmd.AddCommand(cleanCommand)
+}
+
+var cleanCommand = &cobra.Command{
+	Use:   "clean",
+	Short: "Cleans the local database",
+	Long:  `Execute this command to clean the local database`,
+	Run: func(cmd *cobra.Command, args []string) {
+		log.Info().Msg("creating db connection")
+
+		conn, connErr := db.CreateConnection(cmd.Context(), dbUrl)
+		if connErr != nil {
+			log.Fatal().Err(connErr).Msg("failed to connect to DB")
+		}
+
+		defer func() {
+			if err := conn.Close(cmd.Context()); err != nil {
+				log.Fatal().Err(err).Msg("failed to close DB connection")
+			}
+		}()
+
+		rows, queryErr := conn.Query(cmd.Context(), "SELECT schemaname, tablename FROM pg_catalog.pg_tables")
+		if queryErr != nil {
+			log.Fatal().Err(queryErr).Msg("failed to query tables from DB")
+		}
+
+		defer rows.Close()
+
+		var tables []string
+		for rows.Next() {
+			schemaName := ""
+			tableName := ""
+			if scanErr := rows.Scan(&schemaName, &tableName); scanErr != nil {
+				log.Fatal().Err(scanErr).Msg("failed to scan row")
+			}
+			if schemaName == "public" && !strings.Contains(tableName, "atlas") {
+				tables = append(tables, fmt.Sprintf("\"%s\"", tableName))
+			}
+		}
+		log.Info().Interface("rows", tables).Msg("cleaning the following tables")
+		if _, execErr := conn.Exec(cmd.Context(), fmt.Sprintf("TRUNCATE %v CASCADE;", strings.Join(tables, ", "))); execErr != nil {
+			log.Fatal().Err(execErr).Msg("failed to truncate tables")
+		}
+		log.Info().Msg("cleaned tables")
+	},
+}

e2e/cmd/create-jwt.go

@@ -0,0 +1,33 @@
+package cmd
+
+import (
+	"errors"
+	"github.com/basemind-ai/monorepo/shared/go/jwtutils"
+	"github.com/rs/zerolog/log"
+	"time"
+
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	rootCmd.AddCommand(createJwt)
+}
+
+var createJwt = &cobra.Command{
+	Use:   "create-jwt [flags] {applicationId}",
+	Short: "Creates a JWT token",
+	Long:  `Creates a JWT token from the passed in applicationId`,
+	Args: func(cmd *cobra.Command, args []string) error {
+		if len(args) < 1 {
+			return errors.New("an applicationId arg is required")
+		}
+		return nil
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		jwt, createErr := jwtutils.CreateJWT(time.Hour, []byte(secret), args[0])
+		if createErr != nil {
+			log.Fatal().Err(createErr).Msg("failed to create JWT")
+		}
+		log.Info().Str("jwt", jwt).Msg("created JWT token")
+	},
+}

e2e/cmd/root.go

@@ -0,0 +1,37 @@
+package cmd
+
+import (
+	"github.com/basemind-ai/monorepo/shared/go/logging"
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+// rootCmd represents the base command when called without any subcommands.
+var rootCmd = &cobra.Command{
+	Use:   "e2e",
+	Short: "E2E testing CLI",
+	Long:  `CLI tool for executing E2E test`,
+}
+
+var (
+	dbUrl  string
+	secret string
+)
+
+// Execute adds all child commands to the root command and sets flags appropriately.
+// This is called by main.main(). It only needs to happen once to the rootCmd.
+func Execute() {
+	err := rootCmd.Execute()
+	if err != nil {
+		os.Exit(1)
+	}
+}
+
+func init() {
+	logging.Configure(true)
+
+	// global flags
+	rootCmd.PersistentFlags().StringVar(&dbUrl, "dbUrl", "postgresql://basemind:basemind@localhost:5432/basemind", "the url of the DB to use")
+	createJwt.Flags().StringVarP(&secret, "secret", "s", "jeronimo", "Secret to use for JWT creation and decoding")
+}

e2e/cmd/seed.go

@@ -0,0 +1,41 @@
+package cmd
+
+import (
+	"github.com/basemind-ai/monorepo/e2e/factories"
+	"github.com/basemind-ai/monorepo/shared/go/db"
+	"github.com/rs/zerolog/log"
+
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	rootCmd.AddCommand(seedCommand)
+}
+
+var seedCommand = &cobra.Command{
+	Use:   "seed",
+	Short: "Seeds the database with test data",
+	Long:  `Execute this command to insert test data into the DB`,
+	Run: func(cmd *cobra.Command, args []string) {
+		log.Info().Msg("creating db connection")
+
+		conn, connErr := db.CreateConnection(cmd.Context(), dbUrl)
+		if connErr != nil {
+			log.Fatal().Err(connErr).Msg("failed to connect to DB")
+		}
+
+		defer func() {
+			if err := conn.Close(cmd.Context()); err != nil {
+				log.Fatal().Err(err).Msg("failed to close DB connection")
+			}
+		}()
+
+		promptConfig, applicationId, promptConfigCreateErr := factories.CreateApplicationPromptConfig(cmd.Context())
+		if promptConfigCreateErr != nil {
+			log.Fatal().Err(promptConfigCreateErr).Msg("failed to create application prompt config")
+		}
+
+		log.Info().Str("applicationId", applicationId).Msg("created application")
+		log.Info().Interface("promptConfig", promptConfig).Msg("created prompt config")
+	},
+}

e2e/factories/factories.go

@@ -0,0 +1,106 @@
+package factories
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	openaiconnector "github.com/basemind-ai/monorepo/gen/go/openai/v1"
+	"github.com/basemind-ai/monorepo/shared/go/datatypes"
+	"github.com/basemind-ai/monorepo/shared/go/db"
+)
+
+func CreatePromptMessages(systemMessage string, userMessage string) ([]byte, error) {
+	s, createPromptSystemMessageErr := datatypes.CreatePromptTemplateMessage(make([]string, 0), map[string]interface{}{
+		"content": systemMessage,
+		"role":    openaiconnector.OpenAIMessageRole_OPEN_AI_MESSAGE_ROLE_SYSTEM,
+	})
+	if createPromptSystemMessageErr != nil {
+		return nil, createPromptSystemMessageErr
+	}
+	u, createPromptUserMessageErr := datatypes.CreatePromptTemplateMessage([]string{"userInput"}, map[string]interface{}{
+		"content": userMessage,
+		"role":    openaiconnector.OpenAIMessageRole_OPEN_AI_MESSAGE_ROLE_USER,
+	})
+	if createPromptUserMessageErr != nil {
+		return nil, createPromptUserMessageErr
+	}
+
+	promptMessages, marshalErr := json.Marshal([]datatypes.PromptTemplateMessage{
+		*s, *u,
+	})
+	if marshalErr != nil {
+		return nil, marshalErr
+	}
+
+	return promptMessages, nil
+}
+
+func CreateModelParameters() ([]byte, error) {
+	modelParameters, marshalErr := json.Marshal(map[string]float32{
+		"temperature":       1,
+		"top_p":             1,
+		"max_tokens":        1,
+		"presence_penalty":  1,
+		"frequency_penalty": 1,
+	})
+	if marshalErr != nil {
+		return nil, marshalErr
+	}
+
+	return modelParameters, nil
+}
+
+func CreateApplicationPromptConfig(ctx context.Context) (*datatypes.ApplicationPromptConfig, string, error) {
+	project, projectCreateErr := db.GetQueries().CreateProject(ctx, db.CreateProjectParams{
+		Name:        "test",
+		Description: "test",
+	})
+
+	if projectCreateErr != nil {
+		return nil, "", projectCreateErr
+	}
+
+	systemMessage := "You are a helpful chat bot."
+	userMessage := "This is what the user asked for: {userInput}"
+
+	application, applicationCreateErr := db.GetQueries().CreateApplication(ctx, db.CreateApplicationParams{
+		ProjectID:   project.ID,
+		Name:        "test",
+		Description: "test",
+	})
+
+	if applicationCreateErr != nil {
+		return nil, "", applicationCreateErr
+	}
+
+	modelParams, modelParamsCreateErr := CreateModelParameters()
+
+	if modelParamsCreateErr != nil {
+		return nil, "", modelParamsCreateErr
+	}
+
+	promptMessages, promptMessagesCreateErr := CreatePromptMessages(systemMessage, userMessage)
+
+	if promptMessagesCreateErr != nil {
+		return nil, "", promptMessagesCreateErr
+	}
+
+	promptConfig, promptConfigCreateErr := db.GetQueries().CreatePromptConfig(ctx, db.CreatePromptConfigParams{
+		ModelType:         db.ModelTypeGpt35Turbo,
+		ModelVendor:       db.ModelVendorOPENAI,
+		ModelParameters:   modelParams,
+		PromptMessages:    promptMessages,
+		TemplateVariables: []string{"userInput"},
+		IsActive:          true,
+		ApplicationID:     application.ID,
+	})
+	if promptConfigCreateErr != nil {
+		return nil, "", promptConfigCreateErr
+	}
+
+	return &datatypes.ApplicationPromptConfig{
+		ApplicationID:    fmt.Sprintf("%x-%x-%x-%x-%x", application.ID.Bytes[0:4], application.ID.Bytes[4:6], application.ID.Bytes[6:8], application.ID.Bytes[8:10], application.ID.Bytes[10:16]),
+		ApplicationData:  application,
+		PromptConfigData: promptConfig,
+	}, fmt.Sprintf("%x-%x-%x-%x-%x", application.ID.Bytes[0:4], application.ID.Bytes[4:6], application.ID.Bytes[6:8], application.ID.Bytes[8:10], application.ID.Bytes[10:16]), nil
+}

e2e/go.mod

@@ -0,0 +1,14 @@
+module github.com/basemind-ai/monorepo/e2e
+
+go 1.21
+
+require github.com/spf13/cobra v1.7.0
+
+require (
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+)
+
+replace (
+	github.com/basemind-ai/monorepo v0.0.0 => ../
+)

e2e/go.sum

@@ -0,0 +1,10 @@
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
+github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

e2e/main.go

@@ -0,0 +1,7 @@
+package main
+
+import "github.com/basemind-ai/monorepo/e2e/cmd"
+
+func main() {
+	cmd.Execute()
+}

go.mod

@@ -96,3 +96,5 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
+
+replace github.com/basemind-ai/monorepo/e2e => ./e2e

go.work

@@ -0,0 +1,6 @@
+go 1.21
+
+use (
+	e2e
+	.
+)