Skip to content

Commit

Permalink
init
Browse files Browse the repository at this point in the history
  • Loading branch information
@dcaravel
dcaravel committed Jan 14, 2025
1 parent 111f6f5 commit 8963385
Show file tree
Hide file tree
Showing 3 changed files with 124 additions and 48 deletions.
31 changes: 27 additions & 4 deletions database/pgsql/rhelv2_layer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"github.com/stackrox/rox/pkg/utils"
"github.com/stackrox/scanner/database"
"github.com/stackrox/scanner/database/metrics"
"github.com/stackrox/scanner/pkg/env"
)

func (pgSQL *pgSQL) InsertRHELv2Layer(layer *database.RHELv2Layer) error {
Expand Down Expand Up @@ -46,11 +47,18 @@ func (pgSQL *pgSQL) InsertRHELv2Layer(layer *database.RHELv2Layer) error {
func (pgSQL *pgSQL) insertRHELv2Layer(tx *sql.Tx, layer *database.RHELv2Layer) error {
defer metrics.ObserveQueryTime("insertRHELv2Layer", "layer", time.Now())

_, err := tx.Exec(insertRHELv2Layer, layer.Hash, layer.ParentHash, layer.Dist, pq.Array(layer.CPEs), layer.Lineage, layer.ParentLineage)
var lineage string
var parentLineage string
if env.RHLineage.Enabled() {
lineage = layer.Lineage
parentLineage = layer.ParentLineage
}

_, err := tx.Exec(insertRHELv2Layer, layer.Hash, layer.ParentHash, layer.Dist, pq.Array(layer.CPEs), lineage, parentLineage)
return err
}

func (pgSQL *pgSQL) insertRHELv2Packages(tx *sql.Tx, layer string, pkgs []*database.RHELv2Package, lineage string) error {
func (pgSQL *pgSQL) insertRHELv2Packages(tx *sql.Tx, layer string, pkgs []*database.RHELv2Package, layerLineage string) error {
// Sort packages to avoid potential deadlock.
// Sort by the unique index (name, version, module, arch).
sort.SliceStable(pkgs, func(i, j int) bool {
Expand Down Expand Up @@ -80,6 +88,11 @@ func (pgSQL *pgSQL) insertRHELv2Packages(tx *sql.Tx, layer string, pkgs []*datab
}
}

var lineage string
if env.RHLineage.Enabled() {
lineage = layerLineage
}

for _, pkg := range pkgs {
if pkg.Name == "" {
continue
Expand Down Expand Up @@ -112,7 +125,12 @@ func (pgSQL *pgSQL) GetRHELv2Layers(layerHash, layerLineage string) ([]*database
return nil, handleError("GetRHELv2Layers.Begin()", err)
}

rows, err := tx.Query(searchRHELv2Layers, layerHash, layerLineage)
var lineage string
if env.RHLineage.Enabled() {
lineage = layerLineage
}

rows, err := tx.Query(searchRHELv2Layers, layerHash, lineage)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -177,7 +195,12 @@ func (pgSQL *pgSQL) populatePackages(tx *sql.Tx, layers []*database.RHELv2Layer)
func (pgSQL *pgSQL) getPackagesByLayer(tx *sql.Tx, layer *database.RHELv2Layer) error {
defer metrics.ObserveQueryTime("getRHELv2Layers", "packagesByLayer", time.Now())

rows, err := tx.Query(searchRHELv2Package, layer.Hash, layer.Lineage)
var lineage string
if env.RHLineage.Enabled() {
lineage = layer.Lineage
}

rows, err := tx.Query(searchRHELv2Package, layer.Hash, lineage)
if err != nil {
return err
}
Expand Down
134 changes: 90 additions & 44 deletions database/pgsql/rhelv2_layer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"testing"

"github.com/stackrox/scanner/database"
"github.com/stackrox/scanner/pkg/env"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
Expand Down Expand Up @@ -199,31 +200,12 @@ func TestGetRHELv2Layers(t *testing.T) {
// TestRHELv2LayerLineage verifies that data for duplicate layers with different parent
// layers (lineage) is pulled correctly.
func TestRHELv2LayerLineage(t *testing.T) {
datastore, err := openDatabaseForTest("RHELv2LayerLineage", false)
if err != nil {
t.Error(err)
return
}
defer datastore.Close()

// Two 'fake' images will be created, each with 3 layers, the DB will resemble:
// id | hash | parent_hash | dist | cpes | lineage | parent_lineage
// ----+-----------------+-----------------+--------+----------------+-----------+----------------
// 1 | sha256:base | | rhel:8 | | |
// 2 | sha256:layer1-a | sha256:base | rhel:8 | {cpe-a,cpe2-a} | lineage |
// 3 | sha256:layer1-b | sha256:base | rhel:8 | {cpe-b,cpe2-b} | lineage |
// 4 | sha256:leaf | sha256:layer1-a | rhel:8 | | lineage-a | lineage
// 5 | sha256:leaf | sha256:layer1-b | rhel:8 | | lineage-b | lineage

// base layers
base := &database.RHELv2Layer{
Hash: "sha256:base",
Dist: "rhel:8",
}

err = datastore.InsertRHELv2Layer(base)
require.NoError(t, err)

layer1a := &database.RHELv2Layer{
Hash: "sha256:layer1-a",
Lineage: "lineage",
Expand All @@ -250,11 +232,6 @@ func TestRHELv2LayerLineage(t *testing.T) {
CPEs: []string{"cpe-b", "cpe2-b"},
}

err = datastore.InsertRHELv2Layer(layer1a)
require.NoError(t, err)
err = datastore.InsertRHELv2Layer(layer1b)
require.NoError(t, err)

leafa := &database.RHELv2Layer{
Hash: "sha256:leaf", // for this test all leafs should have same digest
Lineage: "lineage-a", // lineage is specific to layer A
Expand All @@ -268,34 +245,103 @@ func TestRHELv2LayerLineage(t *testing.T) {
leafb.Lineage = "lineage-b"
leafb.ParentHash = "sha256:layer1-b"

err = datastore.InsertRHELv2Layer(leafa)
require.NoError(t, err)
err = datastore.InsertRHELv2Layer(leafb)
require.NoError(t, err)
prepDataStore := func(t *testing.T, name string) *pgSQL {
datastore, err := openDatabaseForTest("RHELv2LayerLineage_enabled", false)
require.NoError(t, err)

err = datastore.InsertRHELv2Layer(base)
require.NoError(t, err)
err = datastore.InsertRHELv2Layer(layer1a)
require.NoError(t, err)
err = datastore.InsertRHELv2Layer(layer1b)
require.NoError(t, err)
err = datastore.InsertRHELv2Layer(leafa)
require.NoError(t, err)
err = datastore.InsertRHELv2Layer(leafb)
require.NoError(t, err)

return datastore
}

assertLayersEqual := func(t *testing.T, expected, actual *database.RHELv2Layer) {
assertLayersEqual := func(t *testing.T, expected, actual *database.RHELv2Layer, skipLineage bool) {
resetPackageIDs(actual)
assert.Equal(t, expected.Hash, actual.Hash, "Hash mismatch")
assert.Equal(t, expected.Lineage, actual.Lineage, "Lineage mismatch")
assert.Equal(t, expected.CPEs, actual.CPEs, "CPEs mistmatch")
assert.Equal(t, expected.Pkgs, actual.Pkgs, "Pkgs mismatch")
}

layers, err := datastore.GetRHELv2Layers("sha256:leaf", "lineage-a")
require.NoError(t, err)
require.Len(t, layers, 3)

assertLayersEqual(t, base, layers[0])
assertLayersEqual(t, layer1a, layers[1])
assertLayersEqual(t, leafa, layers[2])
expectedLineage := expected.Lineage
if skipLineage {
expectedLineage = ""
}
assert.Equal(t, expectedLineage, actual.Lineage, "Lineage mismatch")
}

layers, err = datastore.GetRHELv2Layers("sha256:leaf", "lineage-b")
require.NoError(t, err)
require.Len(t, layers, 3)
t.Run("enabled", func(t *testing.T) {
t.Setenv(env.RHLineage.EnvVar(), "true")

datastore := prepDataStore(t, "RHELv2LayerLineage_enabled")
defer datastore.Close()

// The DB will resemble:
// id | hash | parent_hash | dist | cpes | lineage | parent_lineage
// ----+-----------------+-----------------+--------+----------------+-----------+----------------
// 1 | sha256:base | | rhel:8 | | |
// 2 | sha256:layer1-a | sha256:base | rhel:8 | {cpe-a,cpe2-a} | lineage |
// 3 | sha256:layer1-b | sha256:base | rhel:8 | {cpe-b,cpe2-b} | lineage |
// 4 | sha256:leaf | sha256:layer1-a | rhel:8 | | lineage-a | lineage
// 5 | sha256:leaf | sha256:layer1-b | rhel:8 | | lineage-b | lineage

layers, err := datastore.GetRHELv2Layers("sha256:leaf", "lineage-a")
require.NoError(t, err)
require.Len(t, layers, 3)

assertLayersEqual(t, base, layers[0], false)
assertLayersEqual(t, layer1a, layers[1], false)
assertLayersEqual(t, leafa, layers[2], false)

layers, err = datastore.GetRHELv2Layers("sha256:leaf", "lineage-b")
require.NoError(t, err)
require.Len(t, layers, 3)

assertLayersEqual(t, base, layers[0], false)
assertLayersEqual(t, layer1b, layers[1], false)
assertLayersEqual(t, leafb, layers[2], false)
})

t.Run("disable", func(t *testing.T) {
t.Setenv(env.RHLineage.EnvVar(), "false")

datastore := prepDataStore(t, "RHELv2LayerLineage_disabled")
defer datastore.Close()

// The DB will resemble:
// id | hash | parent_hash | dist | cpes | lineage | parent_lineage
// ----+-----------------+-----------------+--------+----------------+---------+----------------
// 1 | sha256:base | | rhel:8 | | |
// 2 | sha256:layer1-a | sha256:base | rhel:8 | {cpe-a,cpe2-a} | |
// 3 | sha256:layer1-b | sha256:base | rhel:8 | {cpe-b,cpe2-b} | |
// 4 | sha256:leaf | sha256:layer1-a | rhel:8 | | |
//
// Note: only the first leaf layer will be inserted (due to the insert
// query 'ON CONFLICT DO NOTHING' clause)

layers, err := datastore.GetRHELv2Layers("sha256:leaf", "lineage-a")
require.NoError(t, err)
require.Len(t, layers, 3)

assertLayersEqual(t, base, layers[0], true)
assertLayersEqual(t, layer1a, layers[1], true)
assertLayersEqual(t, leafa, layers[2], true)

layers, err = datastore.GetRHELv2Layers("sha256:leaf", "lineage-b")
require.NoError(t, err)
require.Len(t, layers, 3)

assertLayersEqual(t, base, layers[0], true)
assertLayersEqual(t, layer1a, layers[1], true) // the bug, would expect layer1b to be here
assertLayersEqual(t, leafb, layers[2], true)
})

assertLayersEqual(t, base, layers[0])
assertLayersEqual(t, layer1b, layers[1])
assertLayersEqual(t, leafb, layers[2])
}

// resetPackageIDs sets all package IDs to 0. Package IDs are DB sequence numbers
Expand Down
7 changes: 7 additions & 0 deletions pkg/env/list.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,4 +44,11 @@ var (
// LegacyNVDLoader when true will cause the loader to pull NVD data using
// the NVD Legacy Data Feeds, if false will pull from the NVD 2.0 API.
LegacyNVDLoader = RegisterBooleanSetting("ROX_LEGACY_NVD_LOADER", false)

// RHLineage when true will cause all parent layers (a.k.a lineage) to be considered when
// storing scan results for RHEL image layers.
//
// Setting this to false will cause known scan inaccuracies and should only be disabled as a
// temporary measure to address unforeseen stability issues.
RHLineage = RegisterBooleanSetting("ROX_RHEL_LINEAGE", true)
)

0 comments on commit 8963385

Please sign in to comment.