feat: add support for aws ecr tokens (#2650)

Signed-off-by: K Tamil Vanan <[email protected]>

examples/config-sync-ecr-credential-helper.json

@@ -0,0 +1,40 @@
+{
+  "distSpecVersion": "1.1.0",
+  "storage": {
+    "rootDirectory": "/tmp/zot",
+    "dedupe": false,
+    "storageDriver": {
+    "name": "s3",
+    "region": "REGION_NAME",
+    "bucket": "BUGKET_NAME",
+    "rootdirectory": "/ROOTDIR",
+    "secure": true,
+    "skipverify": false
+   }
+  },
+  "http": {
+    "address": "0.0.0.0",
+    "port": "8080"
+  },
+  "log": {
+    "level": "debug"
+  },
+  "extensions": {
+     "sync": {
+      "credentialsFile": "",
+      "DownloadDir": "/tmp/zot",
+      "registries": [
+        {
+          "urls": [
+            "https://ACCOUNTID.dkr.ecr.REGION.amazonaws.com"
+          ],
+          "onDemand": true,
+          "maxRetries": 5,
+          "retryDelay": "2m",
+          "credentialHelper": "ecr"
+        }
+      ]
+    }
+  }
+}
+

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/config v1.28.7
 	github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.15.22
 	github.com/aws/aws-sdk-go-v2/service/dynamodb v1.38.1
+	github.com/aws/aws-sdk-go-v2/service/ecr v1.36.6
 	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.8
 	github.com/aws/aws-secretsmanager-caching-go v1.2.0
 	github.com/aws/smithy-go v1.22.1
@@ -158,7 +159,6 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.24.10 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ebs v1.25.3 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.193.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ecr v1.36.6 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.25.3 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.7 // indirect

pkg/extensions/config/sync/config.go

@@ -23,15 +23,16 @@ type Config struct {
 }
 
 type RegistryConfig struct {
-	URLs         []string
-	PollInterval time.Duration
-	Content      []Content
-	TLSVerify    *bool
-	OnDemand     bool
-	CertDir      string
-	MaxRetries   *int
-	RetryDelay   *time.Duration
-	OnlySigned   *bool
+	URLs             []string
+	PollInterval     time.Duration
+	Content          []Content
+	TLSVerify        *bool
+	OnDemand         bool
+	CertDir          string
+	MaxRetries       *int
+	RetryDelay       *time.Duration
+	OnlySigned       *bool
+	CredentialHelper string
 }
 
 type Content struct {

pkg/extensions/sync/ecr_credential_helper.go

@@ -0,0 +1,168 @@
+//go:build sync
+// +build sync
+
+package sync
+
+import (
+	"context"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/ecr"
+
+	syncconf "zotregistry.dev/zot/pkg/extensions/config/sync"
+	"zotregistry.dev/zot/pkg/log"
+)
+
+// ECR tokens are valid for 12 hours. The ExpiryWindow variable is set to 1 hour,
+// meaning if the remaining validity of the token is less than 1 hour, it will be considered expired.
+const (
+	ExpiryWindow          int = 1
+	ECRURLSplitPartsCount int = 6
+	UsernameTokenParts    int = 2
+)
+
+var (
+	ErrInvalidURLFormat          = errors.New("invalid ECR URL is received")
+	ErrInvalidTokenFormat        = errors.New("invalid token format received from ECR")
+	ErrUnableToLoadAWSConfig     = errors.New("unable to load AWS config for region")
+	ErrUnableToGetECRAuthToken   = errors.New("unable to get ECR authorization token for account")
+	ErrUnableToDecodeECRToken    = errors.New("unable to decode ECR token")
+	ErrFailedToGetECRCredentials = errors.New("failed to get ECR credentials")
+)
+
+type ECRCredential struct {
+	username string
+	password string
+	expiry   time.Time
+	account  string
+	region   string
+}
+
+type ECRCredentialsHelper struct {
+	credentials map[string]ECRCredential
+	log         log.Logger
+}
+
+func NewECRCredentialHelper(log log.Logger) CredentialHelper {
+	return &ECRCredentialsHelper{
+		credentials: make(map[string]ECRCredential),
+		log:         log,
+	}
+}
+
+// extractAccountAndRegion extracts the account ID and region from the given ECR URL.
+// Example URL format: account.dkr.ecr.region.amazonaws.com.
+func extractAccountAndRegion(url string) (string, string, error) {
+	parts := strings.Split(url, ".")
+	if len(parts) < ECRURLSplitPartsCount {
+		return "", "", fmt.Errorf("%w: %s", ErrInvalidURLFormat, url)
+	}
+
+	accountID := parts[0] // First part is the account ID
+
+	region := parts[3] // Fourth part is the region
+
+	return accountID, region, nil
+}
+
+func getECRCredentials(remoteAddress string) (ECRCredential, error) {
+	// Extract account ID and region from the URL.
+	accountID, region, err := extractAccountAndRegion(remoteAddress)
+	if err != nil {
+		return ECRCredential{}, fmt.Errorf("%w %s: %w", ErrInvalidTokenFormat, remoteAddress, err)
+	}
+
+	// Load the AWS config for the specific region.
+	cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(region))
+	if err != nil {
+		return ECRCredential{}, fmt.Errorf("%w %s: %w", ErrUnableToLoadAWSConfig, region, err)
+	}
+
+	// Create an ECR client
+	ecrClient := ecr.NewFromConfig(cfg)
+
+	// Fetch the ECR authorization token.
+	ecrAuth, err := ecrClient.GetAuthorizationToken(context.TODO(), &ecr.GetAuthorizationTokenInput{
+		RegistryIds: []string{accountID}, // Filter by the account ID.
+	})
+	if err != nil {
+		return ECRCredential{}, fmt.Errorf("%w %s: %w", ErrUnableToGetECRAuthToken, accountID, err)
+	}
+
+	// Decode the base64-encoded ECR token.
+	authToken := *ecrAuth.AuthorizationData[0].AuthorizationToken
+
+	decodedToken, err := base64.StdEncoding.DecodeString(authToken)
+	if err != nil {
+		return ECRCredential{}, fmt.Errorf("%w: %w", ErrUnableToDecodeECRToken, err)
+	}
+
+	// Split the decoded token into username and password (username is "AWS").
+	tokenParts := strings.Split(string(decodedToken), ":")
+	if len(tokenParts) != UsernameTokenParts {
+		return ECRCredential{}, fmt.Errorf("%w", ErrInvalidTokenFormat)
+	}
+
+	expiry := *ecrAuth.AuthorizationData[0].ExpiresAt
+	username := tokenParts[0]
+	password := tokenParts[1]
+
+	return ECRCredential{username: username, password: password, expiry: expiry, account: accountID, region: region}, nil
+}
+
+// GetECRCredentials retrieves the ECR credentials (username and password) from AWS ECR.
+func (credHelper *ECRCredentialsHelper) GetCredentials(urls []string) (syncconf.CredentialsFile, error) {
+	ecrCredentials := make(syncconf.CredentialsFile)
+
+	for _, url := range urls {
+		remoteAddress := StripRegistryTransport(url)
+
+		ecrCred, err := getECRCredentials(remoteAddress)
+		if err != nil {
+			return syncconf.CredentialsFile{}, fmt.Errorf("%w %s: %w", ErrFailedToGetECRCredentials, url, err)
+		}
+		// Store the credentials in the map using the base URL as the key.
+		ecrCredentials[remoteAddress] = syncconf.Credentials{
+			Username: ecrCred.username,
+			Password: ecrCred.password,
+		}
+		credHelper.credentials[remoteAddress] = ecrCred
+	}
+
+	return ecrCredentials, nil
+}
+
+func (credHelper *ECRCredentialsHelper) IsCredentialsValid(remoteAddress string) bool {
+	expiry := credHelper.credentials[remoteAddress].expiry
+	expiryDuration := time.Duration(ExpiryWindow) * time.Hour
+
+	if time.Until(expiry) <= expiryDuration {
+		credHelper.log.Info().
+			Str("url", remoteAddress).
+			Msg("The credentials are close to expiring")
+
+		return false
+	}
+
+	credHelper.log.Info().
+		Str("url", remoteAddress).
+		Msg("The credentials are valid")
+
+	return true
+}
+
+func (credHelper *ECRCredentialsHelper) RefreshCredentials(remoteAddress string) (syncconf.Credentials, error) {
+	credHelper.log.Info().Str("url", remoteAddress).Msg("Refreshing the ECR credentials")
+
+	ecrCred, err := getECRCredentials(remoteAddress)
+	if err != nil {
+		return syncconf.Credentials{}, fmt.Errorf("%w %s: %w", ErrFailedToGetECRCredentials, remoteAddress, err)
+	}
+
+	return syncconf.Credentials{Username: ecrCred.username, Password: ecrCred.password}, nil
+}

pkg/extensions/sync/remote.go

@@ -44,6 +44,13 @@ func NewRemoteRegistry(client *client.Client, logger log.Logger) Remote {
 	return registry
 }
 
+func (registry *RemoteRegistry) SetUpstreamAuthConfig(username, password string) {
+	registry.context.DockerAuthConfig = &types.DockerAuthConfig{
+		Username: username,
+		Password: password,
+	}
+}
+
 func (registry *RemoteRegistry) GetContext() *types.SystemContext {
 	return registry.context
 }