Skip to content

Commit

Permalink
Merge pull request #58 from anchore/logging-improvements
Browse files Browse the repository at this point in the history 
fix: ensure objects referenced in the report exist
  • Loading branch information
@bradleyjones
bradleyjones authored Jun 16, 2023
2 parents 910eb1e + 03cab9f commit 44023bc
Show file tree
Hide file tree
Showing 3 changed files with 349 additions and 4 deletions.
1 change: 1 addition & 0 deletions pkg/inventory/ecs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ func fetchTasksFromCluster(client ecsiface.ECSAPI, cluster string) ([]*string, e
}

func fetchServicesFromCluster(client ecsiface.ECSAPI, cluster string) ([]*string, error) {
defer tracker.TrackFunctionTime(time.Now(), fmt.Sprintf("Fetching services from cluster: %s", cluster))
input := &ecs.ListServicesInput{
Cluster: aws.String(cluster),
}
Expand Down
76 changes: 75 additions & 1 deletion pkg/inventory/report.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@ func GetInventoryReportsForRegion(region string, anchoreDetails connection.Ancho
err = HandleReport(report, anchoreDetails, quiet, dryRun)
if err != nil {
logger.Log.Error("Failed to report inventory for cluster", err)
jsonReport, _ := json.Marshal(report)
logger.Log.Error("Failed payload", fmt.Errorf("report %s", jsonReport))
}
}
}(*cluster)
Expand All @@ -99,6 +101,78 @@ func GetInventoryReportsForRegion(region string, anchoreDetails connection.Ancho
return nil
}

// ensures that the referenced objects in the report exist, and if not, creates them.
// e.g. if a service is referenced in a task, but the service is not present in the report, create the service with minimal metadata
//
// NOTE: in the future, this can be removed if the enterprise API is updated to accept reports with missing objects and create them on
// the server side
func ensureReferencedObjectsExist(report reporter.Report) reporter.Report {
updatedReport := report

serviceARNs := map[string]bool{}
for _, service := range report.Services {
serviceARNs[service.ARN] = true
}

taskARNs := map[string]bool{}
for _, task := range report.Tasks {
taskARNs[task.ARN] = true
}

// Ensure all services referenced in tasks exist in the report
for _, task := range report.Tasks {
if task.ServiceARN != "" {
if _, ok := serviceARNs[task.ServiceARN]; !ok {
// Service not present in report, create it
updatedReport.Services = append(updatedReport.Services, reporter.Service{
ARN: task.ServiceARN,
})
logger.Log.Warn(
"Service referenced in task not present in report, adding minimal service to report",
"service",
task.ServiceARN,
)
}
}
}

// Ensure all tasks referenced in containers exist in the report
for _, container := range report.Containers {
if _, ok := taskARNs[container.TaskARN]; !ok {
// Task not present in report, create it
updatedReport.Tasks = append(updatedReport.Tasks, reporter.Task{
ARN: container.TaskARN,
TaskDefARN: "UNKNOWN", // NOTE TaskDefARN is not a nullable field in the db, so we need to provide a value
ServiceARN: "",
})
logger.Log.Warn(
"Task referenced in container not present in report, adding minimal task to report",
"task",
container.TaskARN,
)
}
}

// If the report has services, ensure tasks that are not part of a service reference an "UNKNOWN" placeholder service
// so the enterprise API will accept the report
addUnknownService := false
if len(report.Services) > 0 {
for i, task := range updatedReport.Tasks {
if task.ServiceARN == "" {
updatedReport.Tasks[i].ServiceARN = "UNKNOWN"
if !addUnknownService {
updatedReport.Services = append(updatedReport.Services, reporter.Service{
ARN: "UNKNOWN",
})
addUnknownService = true
}
}
}
}

return updatedReport
}

// GetInventoryReportForCluster is an atomic method for getting in-use image results, for a cluster
func GetInventoryReportForCluster(clusterARN string, ecsClient ecsiface.ECSAPI) (reporter.Report, error) {
defer tracker.TrackFunctionTime(time.Now(), fmt.Sprintf("Getting Inventory Report for cluster: %s", clusterARN))
Expand Down Expand Up @@ -148,5 +222,5 @@ func GetInventoryReportForCluster(clusterARN string, ecsClient ecsiface.ECSAPI)
logger.Log.Info("Found containers in cluster", "cluster", clusterARN, "containerCount", len(containers))
}

return report, nil
return ensureReferencedObjectsExist(report), nil
}
276 changes: 273 additions & 3 deletions pkg/inventory/report_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
package inventory

import (
"fmt"
"testing"

"github.com/stretchr/testify/assert"

"github.com/anchore/ecs-inventory/pkg/reporter"
)

func TestGetInventoryReportForCluster(t *testing.T) {
Expand All @@ -13,6 +14,275 @@ func TestGetInventoryReportForCluster(t *testing.T) {
report, err := GetInventoryReportForCluster("cluster-1", mockSvc)

assert.NoError(t, err)
fmt.Println(report)
// assert.Equal(t, 3, len(report.Containers))
assert.Equal(t, 4, len(report.Containers))
}

func Test_ensureReferencedObjectsExist(t *testing.T) {
type args struct {
report reporter.Report
}
tests := []struct {
name string
args args
want reporter.Report
}{
{
name: "no missing objects",
args: args{
report: reporter.Report{
Containers: []reporter.Container{
{
ARN: "arn:aws:ecs:us-east-1:123456789012:container/container-1",
TaskARN: "arn:aws:ecs:us-east-1:123456789012:task/cluster-1/12345678-1234-1234-1234-000000000000",
ImageDigest: "sha256:1234567890123456789012345678901234567890123456789012345678901234",
ImageTag: "latest",
},
},
Tasks: []reporter.Task{
{
ARN: "arn:aws:ecs:us-east-1:123456789012:task/cluster-1/12345678-1234-1234-1234-000000000000",
TaskDefARN: "arn:aws:ecs:us-east-1:123456789012:task-definition/taskdef-1",
ServiceARN: "arn:aws:ecs:us-east-1:123456789012:service/cluster-1/service-1",
Tags: map[string]string{
"tag1": "value1",
},
},
},
Services: []reporter.Service{
{
ARN: "arn:aws:ecs:us-east-1:123456789012:service/cluster-1/service-1",
Tags: map[string]string{
"tag1": "value1",
},
},
},
Timestamp: "2023-06-15T00:00:00Z",
ClusterARN: "arn:aws:ecs:us-east-1:123456789012:cluster/cluster-1",
},
},
want: reporter.Report{
Containers: []reporter.Container{
{
ARN: "arn:aws:ecs:us-east-1:123456789012:container/container-1",
TaskARN: "arn:aws:ecs:us-east-1:123456789012:task/cluster-1/12345678-1234-1234-1234-000000000000",
ImageDigest: "sha256:1234567890123456789012345678901234567890123456789012345678901234",
ImageTag: "latest",
},
},
Tasks: []reporter.Task{
{
ARN: "arn:aws:ecs:us-east-1:123456789012:task/cluster-1/12345678-1234-1234-1234-000000000000",
TaskDefARN: "arn:aws:ecs:us-east-1:123456789012:task-definition/taskdef-1",
ServiceARN: "arn:aws:ecs:us-east-1:123456789012:service/cluster-1/service-1",
Tags: map[string]string{
"tag1": "value1",
},
},
},
Services: []reporter.Service{
{
ARN: "arn:aws:ecs:us-east-1:123456789012:service/cluster-1/service-1",
Tags: map[string]string{
"tag1": "value1",
},
},
},
Timestamp: "2023-06-15T00:00:00Z",
ClusterARN: "arn:aws:ecs:us-east-1:123456789012:cluster/cluster-1",
},
},
{
name: "missing service object",
args: args{
report: reporter.Report{
Containers: []reporter.Container{
{
ARN: "arn:aws:ecs:us-east-1:123456789012:container/container-1",
TaskARN: "arn:aws:ecs:us-east-1:123456789012:task/cluster-1/12345678-1234-1234-1234-000000000000",
ImageDigest: "sha256:1234567890123456789012345678901234567890123456789012345678901234",
ImageTag: "latest",
},
},
Tasks: []reporter.Task{
{
ARN: "arn:aws:ecs:us-east-1:123456789012:task/cluster-1/12345678-1234-1234-1234-000000000000",
TaskDefARN: "arn:aws:ecs:us-east-1:123456789012:task-definition/taskdef-1",
ServiceARN: "arn:aws:ecs:us-east-1:123456789012:service/cluster-1/service-1",
Tags: map[string]string{
"tag1": "value1",
},
},
},
Services: []reporter.Service{},
Timestamp: "2023-06-15T00:00:00Z",
ClusterARN: "arn:aws:ecs:us-east-1:123456789012:cluster/cluster-1",
},
},
want: reporter.Report{
Containers: []reporter.Container{
{
ARN: "arn:aws:ecs:us-east-1:123456789012:container/container-1",
TaskARN: "arn:aws:ecs:us-east-1:123456789012:task/cluster-1/12345678-1234-1234-1234-000000000000",
ImageDigest: "sha256:1234567890123456789012345678901234567890123456789012345678901234",
ImageTag: "latest",
},
},
Tasks: []reporter.Task{
{
ARN: "arn:aws:ecs:us-east-1:123456789012:task/cluster-1/12345678-1234-1234-1234-000000000000",
TaskDefARN: "arn:aws:ecs:us-east-1:123456789012:task-definition/taskdef-1",
ServiceARN: "arn:aws:ecs:us-east-1:123456789012:service/cluster-1/service-1",
Tags: map[string]string{
"tag1": "value1",
},
},
},
Services: []reporter.Service{
{
ARN: "arn:aws:ecs:us-east-1:123456789012:service/cluster-1/service-1",
},
},
Timestamp: "2023-06-15T00:00:00Z",
ClusterARN: "arn:aws:ecs:us-east-1:123456789012:cluster/cluster-1",
},
},
{
name: "missing task object",
args: args{
report: reporter.Report{
Containers: []reporter.Container{
{
ARN: "arn:aws:ecs:us-east-1:123456789012:container/container-1",
TaskARN: "arn:aws:ecs:us-east-1:123456789012:task/cluster-1/12345678-1234-1234-1234-000000000000",
ImageDigest: "sha256:1234567890123456789012345678901234567890123456789012345678901234",
ImageTag: "latest",
},
},
Tasks: []reporter.Task{},
Timestamp: "2023-06-15T00:00:00Z",
ClusterARN: "arn:aws:ecs:us-east-1:123456789012:cluster/cluster-1",
},
},
want: reporter.Report{
Containers: []reporter.Container{
{
ARN: "arn:aws:ecs:us-east-1:123456789012:container/container-1",
TaskARN: "arn:aws:ecs:us-east-1:123456789012:task/cluster-1/12345678-1234-1234-1234-000000000000",
ImageDigest: "sha256:1234567890123456789012345678901234567890123456789012345678901234",
ImageTag: "latest",
},
},
Tasks: []reporter.Task{
{
ARN: "arn:aws:ecs:us-east-1:123456789012:task/cluster-1/12345678-1234-1234-1234-000000000000",
TaskDefARN: "UNKNOWN",
},
},
Timestamp: "2023-06-15T00:00:00Z",
ClusterARN: "arn:aws:ecs:us-east-1:123456789012:cluster/cluster-1",
},
},
{
name: "mix of standalone tasks and tasks in services",
args: args{
report: reporter.Report{
Containers: []reporter.Container{
{
ARN: "arn:aws:ecs:us-east-1:123456789012:container/container-1",
TaskARN: "arn:aws:ecs:us-east-1:123456789012:task/cluster-1/12345678-1234-1234-1234-000000000000",
ImageDigest: "sha256:1234567890123456789012345678901234567890123456789012345678901234",
ImageTag: "latest",
},
{
ARN: "arn:aws:ecs:us-east-1:123456789012:container/container-2",
TaskARN: "arn:aws:ecs:us-east-1:123456789012:task/cluster-1/12345678-1234-1234-1234-000000000001",
ImageDigest: "sha256:1234567890123456789012345678901234567890123456789012345678901234",
ImageTag: "latest",
},
},
Tasks: []reporter.Task{
{
ARN: "arn:aws:ecs:us-east-1:123456789012:task/cluster-1/12345678-1234-1234-1234-000000000000",
TaskDefARN: "arn:aws:ecs:us-east-1:123456789012:task-definition/taskdef-1",
ServiceARN: "arn:aws:ecs:us-east-1:123456789012:service/cluster-1/service-1",
Tags: map[string]string{
"tag1": "value1",
},
},
{
ARN: "arn:aws:ecs:us-east-1:123456789012:task/cluster-1/12345678-1234-1234-1234-000000000001",
TaskDefARN: "arn:aws:ecs:us-east-1:123456789012:task-definition/taskdef-2",
Tags: map[string]string{
"tag1": "value1",
},
},
},
Services: []reporter.Service{
{
ARN: "arn:aws:ecs:us-east-1:123456789012:service/cluster-1/service-1",
Tags: map[string]string{
"tag1": "value1",
},
},
},
Timestamp: "2023-06-15T00:00:00Z",
ClusterARN: "arn:aws:ecs:us-east-1:123456789012:cluster/cluster-1",
},
},
want: reporter.Report{
Containers: []reporter.Container{
{
ARN: "arn:aws:ecs:us-east-1:123456789012:container/container-1",
TaskARN: "arn:aws:ecs:us-east-1:123456789012:task/cluster-1/12345678-1234-1234-1234-000000000000",
ImageDigest: "sha256:1234567890123456789012345678901234567890123456789012345678901234",
ImageTag: "latest",
},
{
ARN: "arn:aws:ecs:us-east-1:123456789012:container/container-2",
TaskARN: "arn:aws:ecs:us-east-1:123456789012:task/cluster-1/12345678-1234-1234-1234-000000000001",
ImageDigest: "sha256:1234567890123456789012345678901234567890123456789012345678901234",
ImageTag: "latest",
},
},
Tasks: []reporter.Task{
{
ARN: "arn:aws:ecs:us-east-1:123456789012:task/cluster-1/12345678-1234-1234-1234-000000000000",
TaskDefARN: "arn:aws:ecs:us-east-1:123456789012:task-definition/taskdef-1",
ServiceARN: "arn:aws:ecs:us-east-1:123456789012:service/cluster-1/service-1",
Tags: map[string]string{
"tag1": "value1",
},
},
{
ARN: "arn:aws:ecs:us-east-1:123456789012:task/cluster-1/12345678-1234-1234-1234-000000000001",
TaskDefARN: "arn:aws:ecs:us-east-1:123456789012:task-definition/taskdef-2",
ServiceARN: "UNKNOWN",
Tags: map[string]string{
"tag1": "value1",
},
},
},
Services: []reporter.Service{
{
ARN: "arn:aws:ecs:us-east-1:123456789012:service/cluster-1/service-1",
Tags: map[string]string{
"tag1": "value1",
},
},
{
ARN: "UNKNOWN",
Tags: nil,
},
},
Timestamp: "2023-06-15T00:00:00Z",
ClusterARN: "arn:aws:ecs:us-east-1:123456789012:cluster/cluster-1",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := ensureReferencedObjectsExist(tt.args.report)
assert.Equal(t, tt.want, got)
})
}
}

0 comments on commit 44023bc

Please sign in to comment.