Merge pull request #44 from synackd/postgres-endpoint-history

/endpoint-history: Support PostgreSQL

cmd/boot-script-service/boot_data.go

@@ -800,11 +800,19 @@ func updateCloudInit(d *bssTypes.CloudInit, p bssTypes.CloudInit) bool {
 }
 
 func updateEndpointAccessed(name string, accessType bssTypes.EndpointType) {
-	timestamp := strconv.FormatInt(time.Now().Unix(), 10)
-	key := fmt.Sprintf("%s/%s/%s", endpointAccessPfx, name, accessType)
-	if err := kvstore.Store(key, timestamp); err != nil {
-		log.Printf("Failed to store last access timestamp %s to key %s: %s",
-			timestamp, key, err)
+	if useSQL {
+		err := bssdb.LogEndpointAccess(name, accessType)
+		if err != nil {
+			log.Printf("Failed to store last access timestamp for endpoint=%q name=%q to postgres DB: %s",
+				accessType, name, err)
+		}
+	} else {
+		timestamp := strconv.FormatInt(time.Now().Unix(), 10)
+		key := fmt.Sprintf("%s/%s/%s", endpointAccessPfx, name, accessType)
+		if err := kvstore.Store(key, timestamp); err != nil {
+			log.Printf("Failed to store last access timestamp %s to key %s: %s",
+				timestamp, key, err)
+		}
 	}
 }
 

cmd/boot-script-service/cloudInitAPI.go

@@ -293,9 +293,17 @@ func endpointHistoryGetAPI(w http.ResponseWriter, r *http.Request) {
 		lastAccessTypeStruct = bssTypes.EndpointType(endpoint)
 	}
 
-	accesses, err := SearchEndpointAccessed(name, lastAccessTypeStruct)
+	var (
+		accesses []bssTypes.EndpointAccess
+		err      error
+	)
+	if useSQL {
+		accesses, err = bssdb.SearchEndpointAccesses(name, bssTypes.EndpointType(endpoint))
+	} else {
+		accesses, err = SearchEndpointAccessed(name, lastAccessTypeStruct)
+	}
 	if err != nil {
-		errMsg := fmt.Sprintf("Failed to search for name: %s, endpoint: %s", name, endpoint)
+		errMsg := fmt.Sprintf("Failed to search for name=%q, endpoint=%q: %v", name, endpoint, err)
 		base.SendProblemDetailsGeneric(w, http.StatusInternalServerError, errMsg)
 		log.Printf("BSS request failed: %s", errMsg)
 		return

cmd/boot-script-service/default_api.go

@@ -899,9 +899,7 @@ func BootscriptGet(w http.ResponseWriter, r *http.Request) {
 				log.Printf("BSS request succeeded for %s", descr)
 
 				// Record the fact this was asked for.
-				if !useSQL {
-					updateEndpointAccessed(comp.ID, bssTypes.EndpointTypeBootscript)
-				}
+				updateEndpointAccessed(comp.ID, bssTypes.EndpointTypeBootscript)
 			}
 		} else {
 			log.Printf("BSS request failed writing response for %s: %s", descr, err.Error())

go.mod

@@ -17,11 +17,13 @@ require (
 )
 
 require (
+	github.com/Cray-HPE/hms-xname v1.3.0
 	github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18
 	github.com/OpenCHAMI/smd/v2 v2.12.15
 	github.com/go-chi/chi v1.5.1
 	github.com/go-chi/chi/v5 v5.0.11
 	github.com/golang-migrate/migrate/v4 v4.16.2
+	github.com/hashicorp/go-retryablehttp v0.7.4
 	github.com/lestrrat-go/jwx v1.2.28
 )
 
@@ -40,7 +42,6 @@ require (
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.4 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
 	github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 // indirect
 	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect

go.sum

@@ -10,6 +10,8 @@ github.com/Cray-HPE/hms-s3 v1.9.2 h1:qZ/yTTr1+r1slDldOsqkcF736bnsiZ/002JVO7LGD/0
 github.com/Cray-HPE/hms-s3 v1.9.2/go.mod h1:p5pVsMDeOdXKssd9qyFtXo4ztrn2lhD04nrO8+NOi7g=
 github.com/Cray-HPE/hms-securestorage v1.13.0 h1:ut6z9TMtCzL902f9NPxcbtkkDuk9zbX6E30pP8j3k6Q=
 github.com/Cray-HPE/hms-securestorage v1.13.0/go.mod h1:P4CMKqQVlx/lv+AdyEjNQubZw2FKNyo/IAtFNgQ3VuI=
+github.com/Cray-HPE/hms-xname v1.3.0 h1:DQmetMniubqcaL6Cxarz9+7KFfWGSEizIhfPHIgC3Gw=
+github.com/Cray-HPE/hms-xname v1.3.0/go.mod h1:XKdjQSzoTps5KDOE8yWojBTAWASGaS6LfRrVDxwTQO8=
 github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
 github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18 h1:oBPtXp9RVm9lk5zTmDLf+Vh21yDHpulBxUqGJQjwQCk=
 github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18/go.mod h1:ggNHWgLfW/WRXcE8ZZC4S7UwHif16HVmyowOCWdNSN8=

internal/postgres/postgres.go → internal/postgres/bootparams.go

@@ -1,4 +1,4 @@
-// Copyright © 2023 Triad National Security, LLC. All rights reserved.
+// Copyright © 2024 Triad National Security, LLC. All rights reserved.
 //
 // This program was produced under U.S. Government contract 89233218CNA000001
 // for Los Alamos National Laboratory (LANL), which is operated by Triad
@@ -16,20 +16,15 @@ package postgres
 import (
 	"database/sql"
 	"fmt"
-	"regexp"
 	"strings"
 
 	"github.com/OpenCHAMI/bss/pkg/bssTypes"
 	"github.com/docker/distribution/uuid"
 	"github.com/jmoiron/sqlx"
-	"github.com/jmoiron/sqlx/reflectx"
+	"github.com/Cray-HPE/hms-xname/xnames"
 	_ "github.com/lib/pq"
 )
 
-const (
-	xNameRegex = `^x([0-9]{1,4})c([0-7])(s([0-9]{1,4}))?b([0])(n([0-9]{1,4}))?$`
-)
-
 type Node struct {
 	Id      string `json:"id"`
 	BootMac string `json:"boot_mac,omitempty"`
@@ -69,71 +64,6 @@ type bgbc struct {
 	Bc BootConfig
 }
 
-// makeKey creates a key from a key and subkey.  If key is not empty, it will
-// be prepended with a '/' if it does not already start with one.  If subkey is
-// not empty, it will be appended with a '/' if it does not already end with
-// one.  The two will be concatenated with no '/' between them.
-func makeKey(key, subkey string) string {
-	ret := key
-	if key != "" && key[0] != '/' {
-		ret = "/" + key
-	}
-	if subkey != "" {
-		if subkey[0] != '/' {
-			ret += "/"
-		}
-		ret += subkey
-	}
-	return ret
-}
-
-// tagToColName extracts the field name from the JSON struct tag. Replace - with
-// _.
-// E.g: From `json:"params,omitempty"` comes `params`.
-func tagToColName(tag string) string {
-	re := regexp.MustCompile(`json:"([a-z0-9-]+)(?:,[a-z0-9-]+)*"`)
-	colName := re.FindString(tag)
-	return strings.Replace(colName, "-", "_", -1)
-}
-
-// fieldNameToColName converts the struct field name (in Pascal case) into
-// the format for the database column (in snake case).
-func fieldNameToColName(fieldName string) string {
-	firstCap := regexp.MustCompile(`(.)([A-Z][a-z]+)`)
-	allCaps := regexp.MustCompile(`([a-z0-9])([A-Z])`)
-	colName := firstCap.ReplaceAllString(fieldName, `${1}_${2}`)
-	colName = allCaps.ReplaceAllString(colName, `${1}_${2}`)
-	return strings.ToLower(colName)
-}
-
-// Connect opens a new connections to a Postgres database and ensures it is reachable.
-// If not, an error is thrown.
-func Connect(host string, port uint, dbName, user, password string, ssl bool, extraDbOpts string) (BootDataDatabase, error) {
-	var (
-		sslmode string
-		bddb    BootDataDatabase
-	)
-	if ssl {
-		sslmode = "verify-full"
-	} else {
-		sslmode = "disable"
-	}
-	connStr := fmt.Sprintf("user=%s password=%s host=%s port=%d dbname=%s sslmode=%s", user, password, host, port, dbName, sslmode)
-	if extraDbOpts != "" {
-		connStr += " " + extraDbOpts
-	}
-	db, err := sqlx.Connect("postgres", connStr)
-	if err != nil {
-		return bddb, err
-	}
-	// Create a new mapper which will use the struct field tag "json" instead of "db",
-	// and ignore extra JSON config values, e.g. "omitempty".
-	db.Mapper = reflectx.NewMapperTagFunc("json", fieldNameToColName, tagToColName)
-	bddb.DB = db
-
-	return bddb, err
-}
-
 // NewNode creates a new Node and populates it with the boot MAC address (converts to lower case),
 // XName, and NID specified.  Before returning the Node, its ID is populated with a new unique
 // identifier.
@@ -716,63 +646,6 @@ func (bddb BootDataDatabase) getConfigsWithNodes(nodeIds []string) (map[bgbc][]N
 	return bgbcToN, err
 }
 
-func stringSliceToSql(ss []string) string {
-	if len(ss) == 0 {
-		return "('')"
-	}
-	sep := ""
-	s := "("
-	for i, st := range ss {
-		s += sep + fmt.Sprintf("'%s'", st)
-		if i == len(ss)-1 {
-			sep = ""
-		} else {
-			sep = ", "
-		}
-	}
-	s += ")"
-	return s
-}
-
-func int32SliceToSql(is []int32) string {
-	sep := ""
-	s := "("
-	for i, in := range is {
-		s += sep + fmt.Sprintf("%d", in)
-		if i == len(is)-1 {
-			sep = ""
-		} else {
-			sep = ", "
-		}
-	}
-	s += ")"
-	return s
-}
-
-// Return the intersection of a and b (matches) and those elements in b but not in a (exclusions).
-func getMatches(a, b []string) (matches, exclusions []string) {
-	for _, aVal := range a {
-		aInB := false
-		for _, bVal := range b {
-			if aVal == bVal {
-				matches = append(matches, aVal)
-				aInB = true
-				break
-			}
-		}
-		if !aInB {
-			exclusions = append(exclusions, aVal)
-		}
-	}
-	return matches, exclusions
-}
-
-// Close calls the Close() method on the database object within the BootDataDatabase. If it errs, an
-// error is returned.
-func (bddb BootDataDatabase) Close() error {
-	return bddb.DB.Close()
-}
-
 // addBootConfigByGroup adds one or more BootConfig/BootGroup to the boot data database, assuming
 // that the list of names are names for node groups, if it doesn't already exist. If an error occurs
 // during any of the SQL queries, it is returned.
@@ -1539,7 +1412,6 @@ func (bddb BootDataDatabase) Add(bp bssTypes.BootParams) (result map[string]stri
 	var (
 		groupNames []string
 		xNames     []string
-		reXName    = regexp.MustCompile(xNameRegex)
 		nodesToAdd []Node
 	)
 
@@ -1559,8 +1431,10 @@ func (bddb BootDataDatabase) Add(bp bssTypes.BootParams) (result map[string]stri
 	case len(bp.Hosts) > 0:
 		// Check each host to see if it is an XName or a node group name.
 		for _, name := range bp.Hosts {
-			match := reXName.FindString(name)
-			if match == "" {
+			xnameRaw := xnames.FromString(name)
+			if xnameRaw == nil {
+				groupNames = append(groupNames, name)
+			} else if _, ok := xnameRaw.(xnames.Node); !ok {
 				groupNames = append(groupNames, name)
 			} else {
 				xNames = append(xNames, name)
@@ -1668,12 +1542,13 @@ func (bddb BootDataDatabase) Delete(bp bssTypes.BootParams) (nodesDeleted, bcsDe
 		var (
 			groupNames []string
 			xNames     []string
-			reXName    = regexp.MustCompile(xNameRegex)
 		)
 		// Check each host to see if it is an XName or a node group name.
 		for _, name := range bp.Hosts {
-			match := reXName.FindString(name)
-			if match == "" {
+			xnameRaw := xnames.FromString(name)
+			if xnameRaw == nil {
+				groupNames = append(groupNames, name)
+			} else if _, ok := xnameRaw.(xnames.Node); !ok {
 				groupNames = append(groupNames, name)
 			} else {
 				xNames = append(xNames, name)