From d367d29002ecce4c43025b234916f75274c64630 Mon Sep 17 00:00:00 2001 From: rishav-karanjit Date: Fri, 10 Jan 2025 12:23:35 -0800 Subject: [PATCH 1/9] init --- .../go/TestsFromDafny-go/localcmc_test.go | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 AwsCryptographicMaterialProviders/runtimes/go/TestsFromDafny-go/localcmc_test.go diff --git a/AwsCryptographicMaterialProviders/runtimes/go/TestsFromDafny-go/localcmc_test.go b/AwsCryptographicMaterialProviders/runtimes/go/TestsFromDafny-go/localcmc_test.go new file mode 100644 index 000000000..81dd66b30 --- /dev/null +++ b/AwsCryptographicMaterialProviders/runtimes/go/TestsFromDafny-go/localcmc_test.go @@ -0,0 +1,39 @@ +package main + +import ( + "context" + "testing" + + "github.com/aws/aws-cryptographic-material-providers-library/mpl/awscryptographymaterialproviderssmithygenerated" + "github.com/aws/aws-cryptographic-material-providers-library/mpl/awscryptographymaterialproviderssmithygeneratedtypes" +) + +// Test setup +var ( + identifiers = []string{ + "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", + "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen", "seventeen", + "eighteen", "nineteen", "twenty", "twenty one", + } + idSize = len(identifiers) +) + +func TestLotsOfAdding(t *testing.T) { + client, err := awscryptographymaterialproviderssmithygenerated.NewClient(awscryptographymaterialproviderssmithygeneratedtypes.MaterialProvidersConfig{}) + if err != nil { + t.Fatal(err) + } + cache := awscryptographymaterialproviderssmithygeneratedtypes.CacheTypeMemberDefault{ + Value: awscryptographymaterialproviderssmithygeneratedtypes.DefaultCache{ + EntryCapacity: 100, + }, + } + test, err := client.CreateCryptographicMaterialsCache(context.Background(), awscryptographymaterialproviderssmithygeneratedtypes.CreateCryptographicMaterialsCacheInput{ + Cache: &cache, + }) + awscryptographymaterialproviderssmithygeneratedtypes.GetCacheEntryInput{ + Identifier: &identifiers[0], + } + test.GetCacheEntry() + // cache := mpl.CreateCryptographicMaterialsCache(100, 16*1024*1024) +} From 8c5a54b7cf6b1081fef5787584382f9884b644a6 Mon Sep 17 00:00:00 2001 From: rishav-karanjit Date: Fri, 10 Jan 2025 16:12:47 -0800 Subject: [PATCH 2/9] Add multithreading --- .../go/TestsFromDafny-go/localcmc_test.go | 76 +++++++++++++++++-- 1 file changed, 71 insertions(+), 5 deletions(-) diff --git a/AwsCryptographicMaterialProviders/runtimes/go/TestsFromDafny-go/localcmc_test.go b/AwsCryptographicMaterialProviders/runtimes/go/TestsFromDafny-go/localcmc_test.go index 81dd66b30..337bca87c 100644 --- a/AwsCryptographicMaterialProviders/runtimes/go/TestsFromDafny-go/localcmc_test.go +++ b/AwsCryptographicMaterialProviders/runtimes/go/TestsFromDafny-go/localcmc_test.go @@ -2,8 +2,13 @@ package main import ( "context" + "crypto/rand" + "math/big" + "sync" "testing" + "time" + "github.com/aws/aws-cryptographic-material-providers-library/mpl/awscryptographykeystoresmithygeneratedtypes" "github.com/aws/aws-cryptographic-material-providers-library/mpl/awscryptographymaterialproviderssmithygenerated" "github.com/aws/aws-cryptographic-material-providers-library/mpl/awscryptographymaterialproviderssmithygeneratedtypes" ) @@ -18,6 +23,41 @@ var ( idSize = len(identifiers) ) +func TestConcurrentCacheOperations(t *testing.T) { + // Similar to threadPoolSize = 10 + const numWorkers = 1 + // Similar to invocationCount = 300000 + const totalOperations = 1 + // Similar to timeOut = 10000 + timeout := time.After(1000 * time.Second) + // Create a WaitGroup to track all operations + var wg sync.WaitGroup + // Create buffered channel to control concurrent operations + ops := make(chan struct{}, totalOperations) + // Fill the ops channel with work items + for i := 0; i < totalOperations; i++ { + ops <- struct{}{} + } + close(ops) + // Launch worker goroutines + for i := 0; i < numWorkers; i++ { + wg.Add(1) + go TestLotsOfAdding(t) + } + // Wait for either completion or timeout + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + select { + case <-timeout: + t.Fatal("Test timed out") + case <-done: + // Test completed successfully + } +} + func TestLotsOfAdding(t *testing.T) { client, err := awscryptographymaterialproviderssmithygenerated.NewClient(awscryptographymaterialproviderssmithygeneratedtypes.MaterialProvidersConfig{}) if err != nil { @@ -25,15 +65,41 @@ func TestLotsOfAdding(t *testing.T) { } cache := awscryptographymaterialproviderssmithygeneratedtypes.CacheTypeMemberDefault{ Value: awscryptographymaterialproviderssmithygeneratedtypes.DefaultCache{ - EntryCapacity: 100, + EntryCapacity: 10, }, } test, err := client.CreateCryptographicMaterialsCache(context.Background(), awscryptographymaterialproviderssmithygeneratedtypes.CreateCryptographicMaterialsCacheInput{ Cache: &cache, }) - awscryptographymaterialproviderssmithygeneratedtypes.GetCacheEntryInput{ - Identifier: &identifiers[0], + if err != nil { + t.Fatal(err) + } + randIndex, err := rand.Int(rand.Reader, big.NewInt(int64(idSize))) + beaconKeyIdentifier := identifiers[randIndex.Int64()] + getCacheEntryInput := awscryptographymaterialproviderssmithygeneratedtypes.GetCacheEntryInput{ + Identifier: []byte(beaconKeyIdentifier), + } + _, err = test.GetCacheEntry(getCacheEntryInput) + if err != nil { + switch err.(type) { + case awscryptographymaterialproviderssmithygeneratedtypes.EntryDoesNotExist: + materials := awscryptographymaterialproviderssmithygeneratedtypes.MaterialsMemberBeaconKey{ + Value: awscryptographykeystoresmithygeneratedtypes.BeaconKeyMaterials{ + BeaconKeyIdentifier: beaconKeyIdentifier, + BeaconKey: []byte(beaconKeyIdentifier), + EncryptionContext: map[string]string{}, + }, + } + + putCacheEntryInput := awscryptographymaterialproviderssmithygeneratedtypes.PutCacheEntryInput{ + Identifier: []byte(beaconKeyIdentifier), + CreationTime: time.Now().Unix(), + ExpiryTime: time.Now().Unix() + 1, + Materials: &materials, + } + test.PutCacheEntry(putCacheEntryInput) + default: + t.Fatal(err) + } } - test.GetCacheEntry() - // cache := mpl.CreateCryptographicMaterialsCache(100, 16*1024*1024) } From 7c647ab046734652ba673625c177c80630339ea8 Mon Sep 17 00:00:00 2001 From: rishav-karanjit Date: Mon, 13 Jan 2025 14:18:04 -0800 Subject: [PATCH 3/9] Add TestConcurrentCacheOperations --- .../go/TestsFromDafny-go/localcmc_test.go | 65 +++++++++++-------- 1 file changed, 39 insertions(+), 26 deletions(-) diff --git a/AwsCryptographicMaterialProviders/runtimes/go/TestsFromDafny-go/localcmc_test.go b/AwsCryptographicMaterialProviders/runtimes/go/TestsFromDafny-go/localcmc_test.go index 337bca87c..5dd013bf1 100644 --- a/AwsCryptographicMaterialProviders/runtimes/go/TestsFromDafny-go/localcmc_test.go +++ b/AwsCryptographicMaterialProviders/runtimes/go/TestsFromDafny-go/localcmc_test.go @@ -24,44 +24,56 @@ var ( ) func TestConcurrentCacheOperations(t *testing.T) { - // Similar to threadPoolSize = 10 - const numWorkers = 1 - // Similar to invocationCount = 300000 - const totalOperations = 1 - // Similar to timeOut = 10000 - timeout := time.After(1000 * time.Second) + const numWorkers = 10 + totalOperations := 300000 + timeout := time.After(10 * time.Second) + go func() { + <-timeout + panic("operation timed out") + }() // Create a WaitGroup to track all operations var wg sync.WaitGroup // Create buffered channel to control concurrent operations - ops := make(chan struct{}, totalOperations) + ops := make(chan int, totalOperations) + errChannel := make(chan error, numWorkers) + for range numWorkers { + wg.Add(1) + go func() { + processWorker(ops, &wg, errChannel) + }() + } // Fill the ops channel with work items - for i := 0; i < totalOperations; i++ { - ops <- struct{}{} + for i := range totalOperations { + ops <- i } close(ops) - // Launch worker goroutines - for i := 0; i < numWorkers; i++ { - wg.Add(1) - go TestLotsOfAdding(t) - } - // Wait for either completion or timeout - done := make(chan struct{}) + // Wait for completion in a separate goroutine go func() { wg.Wait() - close(done) + close(errChannel) }() - select { - case <-timeout: - t.Fatal("Test timed out") - case <-done: - // Test completed successfully + // Check for errors in the main test goroutine + for err := range errChannel { + if err != nil { + t.Fatal(err) + } + } +} + +func processWorker(ops <-chan int, wg *sync.WaitGroup, errChannel chan<- error) { + defer wg.Done() + for range ops { + err := testLotsOfAdding() + if err != nil { + errChannel <- err + } } } -func TestLotsOfAdding(t *testing.T) { +func testLotsOfAdding() error { client, err := awscryptographymaterialproviderssmithygenerated.NewClient(awscryptographymaterialproviderssmithygeneratedtypes.MaterialProvidersConfig{}) if err != nil { - t.Fatal(err) + return err } cache := awscryptographymaterialproviderssmithygeneratedtypes.CacheTypeMemberDefault{ Value: awscryptographymaterialproviderssmithygeneratedtypes.DefaultCache{ @@ -72,7 +84,7 @@ func TestLotsOfAdding(t *testing.T) { Cache: &cache, }) if err != nil { - t.Fatal(err) + return err } randIndex, err := rand.Int(rand.Reader, big.NewInt(int64(idSize))) beaconKeyIdentifier := identifiers[randIndex.Int64()] @@ -99,7 +111,8 @@ func TestLotsOfAdding(t *testing.T) { } test.PutCacheEntry(putCacheEntryInput) default: - t.Fatal(err) + return err } } + return nil } From bd83af7646bfa7491ecbe5af9654ca3862a0852f Mon Sep 17 00:00:00 2001 From: rishav-karanjit Date: Mon, 13 Jan 2025 15:05:07 -0800 Subject: [PATCH 4/9] auto commit --- .github/workflows/library_go_tests.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/library_go_tests.yml b/.github/workflows/library_go_tests.yml index c6204c4b0..1312c2f71 100644 --- a/.github/workflows/library_go_tests.yml +++ b/.github/workflows/library_go_tests.yml @@ -97,3 +97,10 @@ jobs: shell: bash run: | make test_go + + - name: Test ${{ matrix.library }} + if: matrix.library == 'AwsCryptographicMaterialProviders' + working-directory: ./${{ matrix.library }}/runtimes/go/TestsFromDafny-go + shell: bash + run: | + go test \ No newline at end of file From 6bea3225ce390db248d2f3bd9459304775518965 Mon Sep 17 00:00:00 2001 From: rishav-karanjit Date: Mon, 13 Jan 2025 15:09:19 -0800 Subject: [PATCH 5/9] auto commit --- .github/workflows/library_go_tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/library_go_tests.yml b/.github/workflows/library_go_tests.yml index 1312c2f71..d489f7064 100644 --- a/.github/workflows/library_go_tests.yml +++ b/.github/workflows/library_go_tests.yml @@ -97,7 +97,8 @@ jobs: shell: bash run: | make test_go - + + # This is only to test AwsCryptographicMaterialProviders which has local cmc test inside it. - name: Test ${{ matrix.library }} if: matrix.library == 'AwsCryptographicMaterialProviders' working-directory: ./${{ matrix.library }}/runtimes/go/TestsFromDafny-go From d445338fa1028eb2429b3d6610a57ce69d71a9cd Mon Sep 17 00:00:00 2001 From: rishav-karanjit Date: Mon, 13 Jan 2025 15:33:04 -0800 Subject: [PATCH 6/9] go mod name change --- .../runtimes/go/TestsFromDafny-go/localcmc_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/AwsCryptographicMaterialProviders/runtimes/go/TestsFromDafny-go/localcmc_test.go b/AwsCryptographicMaterialProviders/runtimes/go/TestsFromDafny-go/localcmc_test.go index 5dd013bf1..2dd880329 100644 --- a/AwsCryptographicMaterialProviders/runtimes/go/TestsFromDafny-go/localcmc_test.go +++ b/AwsCryptographicMaterialProviders/runtimes/go/TestsFromDafny-go/localcmc_test.go @@ -8,9 +8,9 @@ import ( "testing" "time" - "github.com/aws/aws-cryptographic-material-providers-library/mpl/awscryptographykeystoresmithygeneratedtypes" - "github.com/aws/aws-cryptographic-material-providers-library/mpl/awscryptographymaterialproviderssmithygenerated" - "github.com/aws/aws-cryptographic-material-providers-library/mpl/awscryptographymaterialproviderssmithygeneratedtypes" + "github.com/aws/aws-cryptographic-material-providers-library/releases/go/mpl/awscryptographykeystoresmithygeneratedtypes" + "github.com/aws/aws-cryptographic-material-providers-library/releases/go/mpl/awscryptographymaterialproviderssmithygenerated" + "github.com/aws/aws-cryptographic-material-providers-library/releases/go/mpl/awscryptographymaterialproviderssmithygeneratedtypes" ) // Test setup From dadb2c0753c76ba7e79a0e3dee76f8bdfaeb7305 Mon Sep 17 00:00:00 2001 From: rishav-karanjit Date: Mon, 13 Jan 2025 15:36:35 -0800 Subject: [PATCH 7/9] formatting --- .github/workflows/library_go_tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/library_go_tests.yml b/.github/workflows/library_go_tests.yml index d489f7064..c36fb75e3 100644 --- a/.github/workflows/library_go_tests.yml +++ b/.github/workflows/library_go_tests.yml @@ -97,11 +97,11 @@ jobs: shell: bash run: | make test_go - - # This is only to test AwsCryptographicMaterialProviders which has local cmc test inside it. + + # This is only to test AwsCryptographicMaterialProviders which has local cmc test inside it. - name: Test ${{ matrix.library }} if: matrix.library == 'AwsCryptographicMaterialProviders' working-directory: ./${{ matrix.library }}/runtimes/go/TestsFromDafny-go shell: bash run: | - go test \ No newline at end of file + go test From 0f6a98b84838fffdcff1e212c83c848ddbcea771 Mon Sep 17 00:00:00 2001 From: rishav-karanjit Date: Tue, 14 Jan 2025 09:54:34 -0800 Subject: [PATCH 8/9] auto commit --- .github/workflows/library_go_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/library_go_tests.yml b/.github/workflows/library_go_tests.yml index c36fb75e3..272fb2617 100644 --- a/.github/workflows/library_go_tests.yml +++ b/.github/workflows/library_go_tests.yml @@ -99,7 +99,7 @@ jobs: make test_go # This is only to test AwsCryptographicMaterialProviders which has local cmc test inside it. - - name: Test ${{ matrix.library }} + - name: Test local CMC if: matrix.library == 'AwsCryptographicMaterialProviders' working-directory: ./${{ matrix.library }}/runtimes/go/TestsFromDafny-go shell: bash From 4f7908845c0f1c39caf59352004f15f24d105ad0 Mon Sep 17 00:00:00 2001 From: rishav-karanjit Date: Wed, 22 Jan 2025 16:31:47 -0800 Subject: [PATCH 9/9] Update with cache hit assertion --- .../go/TestsFromDafny-go/localcmc_test.go | 56 +++++++++++-------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/AwsCryptographicMaterialProviders/runtimes/go/TestsFromDafny-go/localcmc_test.go b/AwsCryptographicMaterialProviders/runtimes/go/TestsFromDafny-go/localcmc_test.go index 2dd880329..325e22d04 100644 --- a/AwsCryptographicMaterialProviders/runtimes/go/TestsFromDafny-go/localcmc_test.go +++ b/AwsCryptographicMaterialProviders/runtimes/go/TestsFromDafny-go/localcmc_test.go @@ -3,14 +3,15 @@ package main import ( "context" "crypto/rand" + "fmt" "math/big" "sync" "testing" "time" "github.com/aws/aws-cryptographic-material-providers-library/releases/go/mpl/awscryptographykeystoresmithygeneratedtypes" - "github.com/aws/aws-cryptographic-material-providers-library/releases/go/mpl/awscryptographymaterialproviderssmithygenerated" "github.com/aws/aws-cryptographic-material-providers-library/releases/go/mpl/awscryptographymaterialproviderssmithygeneratedtypes" + "github.com/aws/aws-cryptographic-material-providers-library/releases/go/mpl/test/awscryptographymaterialproviderssmithygenerated" ) // Test setup @@ -36,10 +37,26 @@ func TestConcurrentCacheOperations(t *testing.T) { // Create buffered channel to control concurrent operations ops := make(chan int, totalOperations) errChannel := make(chan error, numWorkers) + client, err := awscryptographymaterialproviderssmithygenerated.NewClient(awscryptographymaterialproviderssmithygeneratedtypes.MaterialProvidersConfig{}) + if err != nil { + panic(err) + } + cacheType := awscryptographymaterialproviderssmithygeneratedtypes.CacheTypeMemberDefault{ + Value: awscryptographymaterialproviderssmithygeneratedtypes.DefaultCache{ + EntryCapacity: 10, + }, + } + cache, err := client.CreateCryptographicMaterialsCache(context.Background(), awscryptographymaterialproviderssmithygeneratedtypes.CreateCryptographicMaterialsCacheInput{ + Cache: &cacheType, + }) + if err != nil { + panic(err) + } + for range numWorkers { wg.Add(1) go func() { - processWorker(ops, &wg, errChannel) + processWorker(ops, &wg, errChannel, cache) }() } // Fill the ops channel with work items @@ -60,38 +77,23 @@ func TestConcurrentCacheOperations(t *testing.T) { } } -func processWorker(ops <-chan int, wg *sync.WaitGroup, errChannel chan<- error) { +func processWorker(ops <-chan int, wg *sync.WaitGroup, errChannel chan<- error, cache awscryptographymaterialproviderssmithygeneratedtypes.ICryptographicMaterialsCache) { defer wg.Done() for range ops { - err := testLotsOfAdding() + err := testLotsOfAdding(cache) if err != nil { errChannel <- err } } } -func testLotsOfAdding() error { - client, err := awscryptographymaterialproviderssmithygenerated.NewClient(awscryptographymaterialproviderssmithygeneratedtypes.MaterialProvidersConfig{}) - if err != nil { - return err - } - cache := awscryptographymaterialproviderssmithygeneratedtypes.CacheTypeMemberDefault{ - Value: awscryptographymaterialproviderssmithygeneratedtypes.DefaultCache{ - EntryCapacity: 10, - }, - } - test, err := client.CreateCryptographicMaterialsCache(context.Background(), awscryptographymaterialproviderssmithygeneratedtypes.CreateCryptographicMaterialsCacheInput{ - Cache: &cache, - }) - if err != nil { - return err - } +func testLotsOfAdding(cache awscryptographymaterialproviderssmithygeneratedtypes.ICryptographicMaterialsCache) error { randIndex, err := rand.Int(rand.Reader, big.NewInt(int64(idSize))) beaconKeyIdentifier := identifiers[randIndex.Int64()] getCacheEntryInput := awscryptographymaterialproviderssmithygeneratedtypes.GetCacheEntryInput{ Identifier: []byte(beaconKeyIdentifier), } - _, err = test.GetCacheEntry(getCacheEntryInput) + op, err := cache.GetCacheEntry(getCacheEntryInput) if err != nil { switch err.(type) { case awscryptographymaterialproviderssmithygeneratedtypes.EntryDoesNotExist: @@ -106,13 +108,21 @@ func testLotsOfAdding() error { putCacheEntryInput := awscryptographymaterialproviderssmithygeneratedtypes.PutCacheEntryInput{ Identifier: []byte(beaconKeyIdentifier), CreationTime: time.Now().Unix(), - ExpiryTime: time.Now().Unix() + 1, + ExpiryTime: time.Now().Unix() + 100, Materials: &materials, } - test.PutCacheEntry(putCacheEntryInput) + err := cache.PutCacheEntry(putCacheEntryInput) + if err != nil { + return (err) + } default: return err } + } else { + if op.Materials.(*awscryptographymaterialproviderssmithygeneratedtypes.MaterialsMemberBeaconKey).Value.BeaconKeyIdentifier != beaconKeyIdentifier { + return fmt.Errorf("beacon key identifier mismatch: %s != %s", op.Materials.(*awscryptographymaterialproviderssmithygeneratedtypes.MaterialsMemberBeaconKey).Value.BeaconKeyIdentifier, beaconKeyIdentifier) + } + fmt.Printf("Cache hit with beacon key identifier %s \n", beaconKeyIdentifier) } return nil }