Add endpoints for enhanced dashboarding view #18
Conversation
internal/db/arango.go
Outdated
| query := ` | ||
| LET stackAnnotations = ( | ||
| FOR a IN annotations FILTER a.dataRef == @key | ||
| LET hostAnnotation = ( | ||
| FOR hostAn in annotations | ||
| FILTER a.host == hostAn.host | ||
| AND (hostAn.layer == @host OR hostAn.layer == @os) | ||
| RETURN hostAn | ||
| ) | ||
| LET tagAnnotation = ( | ||
| FOR tagAn IN annotations | ||
| FILTER a.tag == tagAn.tag AND tagAn.layer == @cicd | ||
| RETURN tagAn | ||
| ) | ||
| RETURN DISTINCT APPEND(tagAnnotation, hostAnnotation) | ||
| ) | ||
| LET appAnnotations = (FOR a IN annotations FILTER a.dataRef == @key RETURN a) | ||
| RETURN FLATTEN(APPEND(appAnnotations, stackAnnotations)) | ||
| ` |
There was a problem hiding this comment.
This part of stackAnnotations won't return the host layer annotations because there is no direct relationship between the App layer annotations and Host layer annotations.
Instead, we should use the OS layer annotations to fetch the Host layer annotations.
| query := ` | |
| LET stackAnnotations = ( | |
| FOR a IN annotations FILTER a.dataRef == @key | |
| LET hostAnnotation = ( | |
| FOR hostAn in annotations | |
| FILTER a.host == hostAn.host | |
| AND (hostAn.layer == @host OR hostAn.layer == @os) | |
| RETURN hostAn | |
| ) | |
| LET tagAnnotation = ( | |
| FOR tagAn IN annotations | |
| FILTER a.tag == tagAn.tag AND tagAn.layer == @cicd | |
| RETURN tagAn | |
| ) | |
| RETURN DISTINCT APPEND(tagAnnotation, hostAnnotation) | |
| ) | |
| LET appAnnotations = (FOR a IN annotations FILTER a.dataRef == @key RETURN a) | |
| RETURN FLATTEN(APPEND(appAnnotations, stackAnnotations)) | |
| ` | |
| query := ` | |
| LET stackAnnotations = ( | |
| FOR a IN annotations | |
| FILTER a.dataRef == @key | |
| LET osHostAnnotations = ( | |
| FOR osAn IN annotations | |
| FILTER a.host == osAn.host AND osAn.layer == @os | |
| LET hostAnnotations = ( | |
| FOR hostAn IN annotations | |
| FILTER hostAn.tag == osAn.tag AND hostAn.layer == @host | |
| RETURN hostAn | |
| ) | |
| RETURN FLATTEN(APPEND([osAn], hostAnnotations)) | |
| ) | |
| LET cicdAnnotations = ( | |
| FOR tagAn IN annotations | |
| FILTER a.tag == tagAn.tag AND tagAn.layer == @cicd | |
| RETURN tagAn | |
| ) | |
| RETURN DISTINCT FLATTEN(APPEND(cicdAnnotations, osHostAnnotations)) | |
| ) | |
| LET appAnnotations = ( | |
| FOR a IN annotations | |
| FILTER a.dataRef == @key | |
| RETURN a | |
| ) | |
| RETURN FLATTEN(APPEND(appAnnotations, stackAnnotations)) | |
| ` |
| query = `FOR appScore IN scores FILTER appScore.dataRef == @key | ||
| LET cicdScore = ( | ||
| FOR s IN scores FILTER | ||
| s.layer == @layer AND s.tag ANY IN appScore.tag |
There was a problem hiding this comment.
There can exist multiple score nodes for cicd layer, we should sort by their timestamp and limit the selection to one score.
| s.layer == @layer AND s.tag ANY IN appScore.tag | |
| s.layer == @layer AND s.tag ANY IN appScore.tag SORT s.timestamp DESC LIMIT 1 |
There was a problem hiding this comment.
@michaelehab I am assuming this is due to failed pipelines, or multiple pipelines producing the same binary/image. I am wondering though does this affect having a piece of data passing through multiple workloads deployed from different pipeliens ?
internal/db/arango.go
Outdated
| RETURN cicdScore ` | ||
| case contracts.Os, contracts.Host: | ||
| query = `FOR a in annotations FILTER a.dataRef == @key LIMIT 1 | ||
| LET scores = (FOR s IN scores FILTER s.layer == @layer AND a.host IN s.tag RETURN s) |
There was a problem hiding this comment.
I believe the same thing can happen with Host and Os scores
| LET scores = (FOR s IN scores FILTER s.layer == @layer AND a.host IN s.tag RETURN s) | |
| LET scores = (FOR s IN scores FILTER s.layer == @layer AND a.host IN s.tag SORT s.timestamp DESC LIMIT 1 RETURN s) | |
There was a problem hiding this comment.
I'd have the same comment as above.
internal/db/arango.go
Outdated
| query := `FOR a IN annotations LET hosts = (a.host) RETURN DISTINCT hosts` | ||
| cursor, err := db.Query(ctx, query, nil) |
There was a problem hiding this comment.
GIven that Host annotations have the machine id from '/etc/machine-id' as the value for the "Host" field, we should be only selecting hosts from app layer annotations, or exclude host layer annotations, I'm not sure which one is better in our case. @tsconn23 What do you think?
| query := `FOR a IN annotations LET hosts = (a.host) RETURN DISTINCT hosts` | |
| cursor, err := db.Query(ctx, query, nil) | |
| query := `FOR a IN annotations FILTER a.layer == @app LET hosts = (a.host) RETURN DISTINCT hosts` | |
| bindVars := map[string]interface{}{ | |
| "app": string(contracts.Application), | |
| } | |
| cursor, err := db.Query(ctx, query, bindVars) |
Fix project-alvarium#17 * Fetch stack annotations alongside app annotations * Add endpoint for retrieving hosts * Add endpoint for retrieving confidence scores, and ability to filter by layer (default is app) Signed-off-by: Ali Amin <[email protected]>
| b, _ := json.Marshal(sampleData) | ||
| key := hashprovider.DeriveHash(b) | ||
|
|
||
| annotations, err := dbArango.QueryAnnotations(r.Context(), key) |
There was a problem hiding this comment.
We'll need to trim the key here to avoid any issues with the db query.
Fix #17