Add endpoints for enhanced dashboarding view

Fix project-alvarium#17

* Fetch stack annotations alongside app annotations
* Add endpoint for retrieving hosts
* Add endpoint for retrieving confidence scores, and
  ability to filter by layer (default is app)

Signed-off-by: Ali Amin <[email protected]>
Signed-off-by: Ali Amin <[email protected]>

internal/db/arango.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/arangodb/go-driver"
 	"github.com/arangodb/go-driver/http"
+	"github.com/project-alvarium/alvarium-sdk-go/pkg/contracts"
 	"github.com/project-alvarium/alvarium-sdk-go/pkg/interfaces"
 	"github.com/project-alvarium/scoring-apps-go/internal/config"
 	"github.com/project-alvarium/scoring-apps-go/pkg/documents"
@@ -88,7 +89,7 @@ func (c *ArangoClient) QueryScore(ctx context.Context, key string) (documents.Sc
 	}
 	defer cursor.Close()
 
-	//There should only be one document returned here
+	// There should only be one document returned here
 	var score documents.Score
 	for {
 		_, err := cursor.ReadDocument(ctx, &score)
@@ -106,26 +107,132 @@ func (c *ArangoClient) QueryAnnotations(ctx context.Context, key string) ([]docu
 	if err != nil {
 		return nil, err
 	}
-	query := "FOR a in annotations FILTER a.dataRef == @key RETURN a"
+	appLayerQuery := `
+			LET stackAnnotations = (
+    				FOR a IN annotations FILTER a.dataRef == @key 
+                                	LET hostAnnotation = (
+						FOR hostAn in annotations 
+						FILTER a.host == hostAn.host 
+							AND (hostAn.layer == @host OR hostAn.layer == @os) 
+				      		RETURN hostAn
+			          	)
+                                        LET tagAnnotation = (
+				        	FOR tagAn IN annotations 
+				      		FILTER a.tag == tagAn.tag AND tagAn.layer == @cicd  
+				      		RETURN tagAn
+			                )
+                                RETURN DISTINCT APPEND(tagAnnotation, hostAnnotation)
+                        )
+			LET appAnnotations = (FOR a IN annotations FILTER a.dataRef == @key RETURN a)
+			RETURN FLATTEN(APPEND(appAnnotations, stackAnnotations))
+			`
 	bindVars := map[string]interface{}{
-		"key": key,
+		"key":  key,
+		"cicd": string(contracts.CiCd),
+		"os":   string(contracts.Os),
+		"host": string(contracts.Host),
 	}
-	cursor, err := db.Query(ctx, query, bindVars)
+	cursor, err := db.Query(ctx, appLayerQuery, bindVars)
 	if err != nil {
 		return nil, err
 	}
 	defer cursor.Close()
 
 	var annotations []documents.Annotation
 	for {
-		var doc documents.Annotation
-		_, err := cursor.ReadDocument(ctx, &doc)
+		_, err := cursor.ReadDocument(ctx, &annotations)
 		if driver.IsNoMoreDocuments(err) {
 			break
 		} else if err != nil {
 			return nil, err
 		}
-		annotations = append(annotations, doc)
 	}
+
 	return annotations, nil
 }
+
+func (c *ArangoClient) QueryScoreByLayer(ctx context.Context, key string, layer contracts.LayerType) ([]documents.Score, error) {
+	db, err := c.instance.Database(ctx, c.cfg.DatabaseName)
+	if err != nil {
+		return nil, err
+	}
+
+	var query string
+	switch layer {
+	case contracts.Application:
+		query = `FOR s IN scores FILTER s.dataRef == @key AND s.layer == @layer RETURN [s]`
+	case contracts.CiCd:
+		query = `FOR appScore IN scores FILTER appScore.dataRef == @key 
+				LET cicdScore = (
+					FOR s IN scores FILTER 
+					s.layer == @layer AND s.tag ANY IN appScore.tag 
+					RETURN s 
+				)
+				RETURN cicdScore `
+	case contracts.Os, contracts.Host:
+		query = `FOR a in annotations FILTER a.dataRef == @key LIMIT 1
+				LET scores = (FOR s IN scores FILTER s.layer == @layer AND a.host IN s.tag RETURN s)
+				RETURN scores`
+
+	}
+	bindVars := map[string]interface{}{
+		"key":   key,
+		"layer": layer,
+	}
+	cursor, err := db.Query(ctx, query, bindVars)
+	if err != nil {
+		return nil, err
+	}
+	defer cursor.Close()
+
+	// There may be multiple documents returned here, for example
+	// if a data score is affected by 2 workloads (2 CICD scores)
+	// then two scores will be returned
+	var scores []documents.Score
+	for {
+		_, err := cursor.ReadDocument(ctx, &scores)
+		if driver.IsNoMoreDocuments(err) {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+	}
+
+	return scores, nil
+}
+
+func (c *ArangoClient) FetchHosts(ctx context.Context) ([]string, error) {
+	db, err := c.instance.Database(ctx, c.cfg.DatabaseName)
+	if err != nil {
+		return nil, err
+	}
+
+	query := `FOR a IN annotations LET hosts = (a.host) RETURN DISTINCT hosts`
+	cursor, err := db.Query(ctx, query, nil)
+	if err != nil {
+		return nil, err
+	}
+	defer cursor.Close()
+
+	var hosts []string
+	for {
+		// returning 1 result will expect a string value,
+		// if multiple values, expects a []string value
+		if cursor.Count() > 1 {
+			_, err = cursor.ReadDocument(ctx, &hosts)
+		} else {
+			var host string
+			_, err = cursor.ReadDocument(ctx, &host)
+			if err == nil {
+				hosts = append(hosts, host)
+			}
+		}
+		if driver.IsNoMoreDocuments(err) {
+			break
+		} else if err != nil {
+			return nil, err
+		}
+	}
+
+	return hosts, nil
+}

internal/hashprovider/types.go

@@ -17,11 +17,12 @@ package hashprovider
 import (
 	crypto "crypto/sha256"
 	"encoding/hex"
+	"strings"
 )
 
 func DeriveHash(data []byte) string {
 	h := crypto.Sum256(data)
 	hashEncoded := make([]byte, hex.EncodedLen(len(h)))
 	hex.Encode(hashEncoded, h[:])
-	return string(hashEncoded)
+	return strings.ToUpper(string(hashEncoded))
 }

internal/populator-api/routes.go

@@ -16,13 +16,16 @@ package populator_api
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"log/slog"
 	"net/http"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/gorilla/mux"
+	"github.com/project-alvarium/alvarium-sdk-go/pkg/contracts"
 	"github.com/project-alvarium/alvarium-sdk-go/pkg/interfaces"
 	"github.com/project-alvarium/scoring-apps-go/internal/db"
 	"github.com/project-alvarium/scoring-apps-go/internal/hashprovider"
@@ -41,22 +44,32 @@ func LoadRestRoutes(r *mux.Router, dbArango *db.ArangoClient, dbMongo *db.MongoP
 	r.HandleFunc("/",
 		func(w http.ResponseWriter, r *http.Request) {
 			getIndexHandler(w, r, logger)
-		}).Methods(http.MethodGet)
+		}).Methods(http.MethodGet, http.MethodOptions)
 
 	r.HandleFunc("/data/{limit:[0-9]+}",
 		func(w http.ResponseWriter, r *http.Request) {
-			getSampleDataHandler(w, r, dbMongo, logger)
-		}).Methods(http.MethodGet)
+			getSampleDataHandler(w, r, dbMongo, dbArango, logger)
+		}).Methods(http.MethodGet, http.MethodOptions)
 
 	r.HandleFunc("/data/count",
 		func(w http.ResponseWriter, r *http.Request) {
 			getDocumentCountHandler(w, r, dbMongo, logger)
-		}).Methods(http.MethodGet)
+		}).Methods(http.MethodGet, http.MethodOptions)
 
 	r.HandleFunc("/data/{id}/annotations",
 		func(w http.ResponseWriter, r *http.Request) {
 			getAnnotationsHandler(w, r, dbMongo, dbArango, logger)
-		}).Methods(http.MethodGet)
+		}).Methods(http.MethodGet, http.MethodOptions)
+
+	r.HandleFunc("/data/{id}/confidence",
+		func(w http.ResponseWriter, r *http.Request) {
+			getDataConfidence(w, r, dbMongo, dbArango, logger)
+		}).Methods(http.MethodGet, http.MethodOptions)
+
+	r.HandleFunc("/hosts",
+		func(w http.ResponseWriter, r *http.Request) {
+			getHosts(w, r, dbArango, logger)
+		}).Methods(http.MethodGet, http.MethodOptions)
 }
 
 func getIndexHandler(w http.ResponseWriter, r *http.Request, logger interfaces.Logger) {
@@ -89,7 +102,13 @@ func getDocumentCountHandler(w http.ResponseWriter, r *http.Request, dbMongo *db
 	w.Write(b)
 }
 
-func getSampleDataHandler(w http.ResponseWriter, r *http.Request, dbMongo *db.MongoProvider, logger interfaces.Logger) {
+func getSampleDataHandler(
+	w http.ResponseWriter,
+	r *http.Request,
+	dbMongo *db.MongoProvider,
+	dbArango *db.ArangoClient,
+	logger interfaces.Logger,
+) {
 	defer r.Body.Close()
 
 	vars := mux.Vars(r)
@@ -100,16 +119,50 @@ func getSampleDataHandler(w http.ResponseWriter, r *http.Request, dbMongo *db.Mo
 		w.Write([]byte(err.Error()))
 		return
 	}
+
 	results, err := dbMongo.QueryMostRecent(r.Context(), limit)
 	if err != nil {
 		logger.Error(err.Error())
 		w.WriteHeader(http.StatusInternalServerError)
 		w.Write([]byte(err.Error()))
 		return
 	}
+
+	// Applying a host filter on the data if supplied
+
+	host := r.URL.Query().Get("host")
 	var viewModels []responses.DataViewModel
-	for _, i := range results {
-		viewModels = append(viewModels, models.ViewModelFromMongoRecord(i))
+	for _, record := range results {
+		// skip host filter if not supplied
+		if host == "" {
+			viewModels = append(viewModels, models.ViewModelFromMongoRecord(record))
+			continue
+		}
+		// Current approach is getting the dataRef by hashing
+		// the mongo record, then fetching annotations by that
+		// dataRef and finding their host
+		sampleData := models.SampleFromMongoRecord(record)
+		b, _ := json.Marshal(sampleData)
+		key := hashprovider.DeriveHash(b)
+
+		annotations, err := dbArango.QueryAnnotations(r.Context(), key)
+		if err != nil {
+			logger.Error("failed to filter data by hosts : " + err.Error())
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte(err.Error()))
+			return
+		}
+		if len(annotations) == 0 {
+			err := errors.New("failed to filter data by hosts : annotations required to find hosts")
+			logger.Error(err.Error())
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte(err.Error()))
+			return
+		}
+
+		if strings.EqualFold(annotations[0].Host, host) {
+			viewModels = append(viewModels, models.ViewModelFromMongoRecord(record))
+		}
 	}
 
 	response := responses.DataListResponse{
@@ -167,3 +220,86 @@ func getAnnotationsHandler(w http.ResponseWriter, r *http.Request, dbMongo *db.M
 	w.WriteHeader(http.StatusOK)
 	w.Write(b)
 }
+
+func getDataConfidence(
+	w http.ResponseWriter,
+	r *http.Request,
+	dbMongo *db.MongoProvider,
+	dbArango *db.ArangoClient,
+	logger interfaces.Logger,
+) {
+	defer r.Body.Close()
+
+	vars := mux.Vars(r)
+	id := vars["id"]
+
+	layerRaw := r.URL.Query().Get("layer")
+	var layer contracts.LayerType
+	if layerRaw == "" {
+		layer = contracts.Application
+	} else {
+		layer = contracts.LayerType(layerRaw)
+	}
+
+	if !layer.Validate() {
+		w.WriteHeader(http.StatusBadRequest)
+		w.Write([]byte("Bad layer value: " + layerRaw))
+		return
+	}
+
+	record, err := dbMongo.FetchById(r.Context(), id)
+	if err != nil {
+		logger.Error(err.Error())
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(err.Error()))
+		return
+	}
+
+	data := models.SampleFromMongoRecord(record)
+	b, _ := json.Marshal(data)
+	key := hashprovider.DeriveHash(b)
+
+	scores, err := dbArango.QueryScoreByLayer(r.Context(), key, layer)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(err.Error()))
+		return
+	}
+	s, err := json.Marshal(scores)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(err.Error()))
+		return
+	}
+	w.Header().Add(headerKeyContentType, headerValueJson)
+	w.Header().Add(headerCORS, headerCORSValue)
+	w.WriteHeader(http.StatusOK)
+	w.Write(s)
+}
+
+func getHosts(
+	w http.ResponseWriter,
+	r *http.Request,
+	dbArango *db.ArangoClient,
+	logger interfaces.Logger,
+) {
+	hosts, err := dbArango.FetchHosts(r.Context())
+	if err != nil {
+		logger.Error(err.Error())
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(err.Error()))
+		return
+	}
+
+	payload, err := json.Marshal(hosts)
+	if err != nil {
+		logger.Error(err.Error())
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(err.Error()))
+		return
+	}
+	w.Header().Add(headerKeyContentType, headerValueJson)
+	w.Header().Add(headerCORS, headerCORSValue)
+	w.WriteHeader(http.StatusOK)
+	w.Write(payload)
+}