Add new config option for DynamicHostPortRange

README.md

@@ -205,6 +205,7 @@ additional details on each available environment variable.
 | `NO_PROXY` | <For Linux: 169.254.169.254,169.254.170.2,/var/run/docker.sock &#124; For Windows: 169.254.169.254,169.254.170.2,\\.\pipe\docker_engine> | The HTTP traffic that should not be forwarded to the specified HTTP_PROXY. You must specify 169.254.169.254,/var/run/docker.sock to filter Amazon EC2 instance metadata and Docker daemon traffic from the proxy. | `null` | `null` |
 | `CREDENTIALS_FETCHER_HOST`   | `unix:///var/credentials-fetcher/socket/credentials_fetcher.sock` | Used to create a connection to the [credentials-fetcher daemon](https://github.com/aws/credentials-fetcher); to support gMSA on Linux. The default is fine for most users, only needs to be modified if user is configuring a custom credentials-fetcher socket path, ie, [CF_UNIX_DOMAIN_SOCKET_DIR](https://github.com/aws/credentials-fetcher#default-environment-variables). | `unix:///var/credentials-fetcher/socket/credentials_fetcher.sock` | Not Applicable |
 | `CREDENTIALS_FETCHER_SECRET_NAME_FOR_DOMAINLESS_GMSA`   | `secretmanager-secretname` | Used to support scaling option for gMSA on Linux [credentials-fetcher daemon](https://github.com/aws/credentials-fetcher). If user is configuring gMSA on a non-domain joined instance, they need to create an Active Directory user with access to retrieve principals for the gMSA account and store it in secrets manager | `secretmanager-secretname` | Not Applicable |
+| `ECS_DYNAMIC_HOST_PORT_RANGE` | `100-200` | This specifies the dynamic host port range that the agent uses to assign host ports from, for a container port range mapping. | Defined by `/proc/sys/net/ipv4/ip_local_port_range` | `49152-65535` |
 ### Persistence
 
 When you run the Amazon ECS Container Agent in production, its `datadir` should be persisted between runs of the Docker

agent/api/task/task.go

@@ -1860,7 +1860,7 @@ func (task *Task) dockerHostConfig(container *apicontainer.Container, dockerCont
 	if err != nil {
 		return nil, &apierrors.HostConfigError{Msg: err.Error()}
 	}
-	dockerPortMap, err := task.dockerPortMap(container)
+	dockerPortMap, err := task.dockerPortMap(container, cfg.DynamicHostPortRange)
 	if err != nil {
 		return nil, &apierrors.HostConfigError{Msg: fmt.Sprintf("error retrieving docker port map: %+v", err.Error())}
 	}
@@ -2334,7 +2334,7 @@ func (task *Task) dockerLinks(container *apicontainer.Container, dockerContainer
 
 var getHostPortRange = utils.GetHostPortRange
 
-func (task *Task) dockerPortMap(container *apicontainer.Container) (nat.PortMap, error) {
+func (task *Task) dockerPortMap(container *apicontainer.Container, dynamicHostPortRange string) (nat.PortMap, error) {
 	dockerPortMap := nat.PortMap{}
 	scContainer := task.GetServiceConnectContainer()
 	containerToCheck := container
@@ -2402,7 +2402,7 @@ func (task *Task) dockerPortMap(container *apicontainer.Container) (nat.PortMap,
 			// we will try to get a contiguous set of host ports from the ephemeral host port range.
 			// this is to ensure that docker maps host ports in a contiguous manner, and
 			// we are guaranteed to have the entire hostPortRange in a single network binding while sending this info to ECS.
-			hostPortRange, err := getHostPortRange(numberOfPorts, protocol)
+			hostPortRange, err := getHostPortRange(numberOfPorts, protocol, dynamicHostPortRange)
 			if err != nil {
 				// in the odd case where we're unable to find a contiguous set of host ports, we fall back to docker dynamic port
 				// assignment for the requested ContainerPortRange.

agent/api/task/task_test.go

@@ -282,7 +282,7 @@ func TestDockerHostConfigPortBinding(t *testing.T) {
 	testCases := []struct {
 		testName                      string
 		testTask                      *Task
-		getHostPortRange              func(numberOfPorts int, protocol string) (string, error)
+		getHostPortRange              func(numberOfPorts int, protocol string, dynamicHostPortRange string) (string, error)
 		expectedPortBinding           nat.PortMap
 		expectedContainerPortSet      map[int]struct{}
 		expectedContainerPortRangeMap map[string]string
@@ -303,7 +303,7 @@ func TestDockerHostConfigPortBinding(t *testing.T) {
 		{
 			testName: "2 port bindings, each with container port range, 1 found valid host port range, other didn't",
 			testTask: testTask2,
-			getHostPortRange: func(numberOfPorts int, protocol string) (string, error) {
+			getHostPortRange: func(numberOfPorts int, protocol string, dynamicHostPortRange string) (string, error) {
 				if numberOfPorts == 3 {
 					return "", errors.New("couldn't find host ports")
 				}
@@ -325,7 +325,7 @@ func TestDockerHostConfigPortBinding(t *testing.T) {
 		{
 			testName: "2 port bindings, one with container port range, other with singular container port",
 			testTask: testTask3,
-			getHostPortRange: func(numberOfPorts int, protocol string) (string, error) {
+			getHostPortRange: func(numberOfPorts int, protocol string, dynamicHostPortRange string) (string, error) {
 				return "155-157", nil
 			},
 			expectedPortBinding: nat.PortMap{

agent/config/config.go

@@ -596,6 +596,7 @@ func environmentConfig() (Config, error) {
 		EnableRuntimeStats:                  parseBooleanDefaultFalseConfig("ECS_ENABLE_RUNTIME_STATS"),
 		ShouldExcludeIPv6PortBinding:        parseBooleanDefaultTrueConfig("ECS_EXCLUDE_IPV6_PORTBINDING"),
 		WarmPoolsSupport:                    parseBooleanDefaultFalseConfig("ECS_WARM_POOLS_CHECK"),
+		DynamicHostPortRange:                parseDynamicHostPortRange("ECS_DYNAMIC_HOST_PORT_RANGE"),
 	}, err
 }
 

agent/config/config_test.go

@@ -19,13 +19,15 @@ package config
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"os"
 	"testing"
 	"time"
 
 	"github.com/aws/amazon-ecs-agent/agent/dockerclient"
 	"github.com/aws/amazon-ecs-agent/agent/ec2"
 	mock_ec2 "github.com/aws/amazon-ecs-agent/agent/ec2/mocks"
+	"github.com/aws/amazon-ecs-agent/agent/utils"
 
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/golang/mock/gomock"
@@ -160,6 +162,7 @@ func TestEnvironmentConfig(t *testing.T) {
 	defer setTestEnv("ECS_ENABLE_RUNTIME_STATS", "true")()
 	defer setTestEnv("ECS_EXCLUDE_IPV6_PORTBINDING", "true")()
 	defer setTestEnv("ECS_WARM_POOLS_CHECK", "false")()
+	defer setTestEnv("ECS_DYNAMIC_HOST_PORT_RANGE", "200-300")()
 	additionalLocalRoutesJSON := `["1.2.3.4/22","5.6.7.8/32"]`
 	setTestEnv("ECS_AWSVPC_ADDITIONAL_LOCAL_ROUTES", additionalLocalRoutesJSON)
 	setTestEnv("ECS_ENABLE_CONTAINER_METADATA", "true")
@@ -219,6 +222,7 @@ func TestEnvironmentConfig(t *testing.T) {
 	assert.True(t, conf.EnableRuntimeStats.Enabled(), "Wrong value for EnableRuntimeStats")
 	assert.True(t, conf.ShouldExcludeIPv6PortBinding.Enabled(), "Wrong value for ShouldExcludeIPv6PortBinding")
 	assert.False(t, conf.WarmPoolsSupport.Enabled(), "Wrong value for WarmPoolsSupport")
+	assert.Equal(t, "200-300", conf.DynamicHostPortRange)
 }
 
 func TestTrimWhitespaceWhenCreating(t *testing.T) {
@@ -494,6 +498,62 @@ func TestValidFormatParseEnvVariableDuration(t *testing.T) {
 	assert.Equal(t, 1*time.Second, duration, "Unexpected value parsed in parseEnvVariableDuration.")
 }
 
+func TestParseDynamicHostPortRange(t *testing.T) {
+	testCases := []struct {
+		testName                            string
+		testDynamicHostPortRangeVal         string
+		expectedPortRangeVal                string
+		expectedErrorDynamicHostPortRange   error
+		expectedErrorEphemeralHostPortRange error
+	}{
+		{
+			testName:                            "Parse DynamicHostPortRange for valid DynamicHostPortRange value",
+			testDynamicHostPortRangeVal:         "200-300",
+			expectedPortRangeVal:                "200-300",
+			expectedErrorDynamicHostPortRange:   nil,
+			expectedErrorEphemeralHostPortRange: nil,
+		},
+		{
+			testName:                            "Parse DynamicHostPortRange for Invalid DynamicHostPortRange value",
+			testDynamicHostPortRangeVal:         "test1",
+			expectedPortRangeVal:                "300-400",
+			expectedErrorDynamicHostPortRange:   errors.New("Invalid DynamicHostPortRange"),
+			expectedErrorEphemeralHostPortRange: nil,
+		},
+		{
+			testName:                            "Invalid DynamicHostPortRange value and error on getDynamicHostPortRange value",
+			testDynamicHostPortRangeVal:         "test2",
+			expectedPortRangeVal:                fmt.Sprintf("%d-%d", utils.DefaultPortRangeStart, utils.DefaultPortRangeEnd),
+			expectedErrorDynamicHostPortRange:   nil,
+			expectedErrorEphemeralHostPortRange: errors.New("Error getting EphemeralHostPortRange"),
+		},
+	}
+	defer func() {
+		getDynamicHostPortRange = utils.GetDynamicHostPortRange
+	}()
+	for _, tc := range testCases {
+		t.Run(tc.testName, func(t *testing.T) {
+			defer setTestRegion()()
+			defer setTestEnv("ECS_DYNAMIC_HOST_PORT_RANGE", tc.testDynamicHostPortRangeVal)()
+
+			if tc.expectedErrorDynamicHostPortRange != nil {
+				getDynamicHostPortRange = func() (start int, end int, err error) {
+					return 300, 400, nil
+				}
+			}
+
+			if tc.expectedErrorEphemeralHostPortRange != nil {
+				getDynamicHostPortRange = func() (start int, end int, err error) {
+					return 10, 20, errors.New("test default values")
+				}
+			}
+
+			dynamicHostPortRange := parseDynamicHostPortRange("ECS_DYNAMIC_HOST_PORT_RANGE")
+			assert.Equal(t, tc.expectedPortRangeVal, dynamicHostPortRange)
+		})
+	}
+}
+
 func TestInvalidTaskCleanupTimeoutOverridesToThreeHours(t *testing.T) {
 	defer setTestRegion()()
 	setTestEnv("ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION", "1ms")

agent/config/parse.go

@@ -23,8 +23,11 @@ import (
 	"time"
 
 	"github.com/aws/amazon-ecs-agent/agent/dockerclient"
+	"github.com/aws/amazon-ecs-agent/agent/utils"
+
 	"github.com/cihub/seelog"
 	cnitypes "github.com/containernetworking/cni/pkg/types"
+	"github.com/docker/go-connections/nat"
 )
 
 const (
@@ -372,3 +375,21 @@ func parseCgroupCPUPeriod() time.Duration {
 
 	return defaultCgroupCPUPeriod
 }
+
+var getDynamicHostPortRange = utils.GetDynamicHostPortRange
+
+func parseDynamicHostPortRange(dynamicHostPortRangeEnv string) string {
+	dynamicHostPortRange := os.Getenv(dynamicHostPortRangeEnv)
+	_, _, err := nat.ParsePortRangeToInt(dynamicHostPortRange)
+	if err != nil {
+		seelog.Warnf("Unable to read the dynamicHostPortRange value: %s", dynamicHostPortRange)
+		startHostPortRange, endHostPortRange, err := getDynamicHostPortRange()
+		if err != nil {
+			seelog.Warnf("Unable to read the ephemeral host port range, "+
+				"falling back to the default range: %v-%v", utils.DefaultPortRangeStart, utils.DefaultPortRangeEnd)
+			return fmt.Sprintf("%d-%d", utils.DefaultPortRangeStart, utils.DefaultPortRangeEnd)
+		}
+		return fmt.Sprintf("%d-%d", startHostPortRange, endHostPortRange)
+	}
+	return dynamicHostPortRange
+}

agent/config/types.go

@@ -33,6 +33,7 @@ type Config struct {
 	// ClusterArn is the Name or full ARN of a Cluster to register into. It has
 	// been deprecated (and will eventually be removed) in favor of Cluster
 	ClusterArn string `deprecated:"Please use Cluster instead"`
+
 	// Cluster can either be the Name or full ARN of a Cluster. This is the
 	// cluster the agent should register this ContainerInstance into. If this
 	// value is not set, it will default to "default"
@@ -359,4 +360,9 @@ type Config struct {
 	// WarmPoolsSupport specifies whether the agent should poll IMDS to check the target lifecycle state for a starting
 	// instance
 	WarmPoolsSupport BooleanDefaultFalse
+
+	// DynamicHostPortRange specifies the dynamic host port range that the agent
+	// uses to assign host ports from, for a container port range mapping.
+	// This defaults to the platform specific ephemeral host port range
+	DynamicHostPortRange string
 }

agent/utils/ephemeral_ports.go

@@ -21,7 +21,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/cihub/seelog"
+	"github.com/docker/go-connections/nat"
 )
 
 // From https://www.kernel.org/doc/html/latest//networking/ip-sysctl.html#ip-variables
@@ -89,22 +89,15 @@ func (pt *safePortTracker) GetLastAssignedHostPort() int {
 	return pt.lastAssignedHostPort
 }
 
-var dynamicHostPortRange = getDynamicHostPortRange
 var tracker safePortTracker
 
 // GetHostPortRange gets N contiguous host ports from the ephemeral host port range defined on the host.
-func GetHostPortRange(numberOfPorts int, protocol string) (string, error) {
+func GetHostPortRange(numberOfPorts int, protocol string, dynamicHostPortRange string) (string, error) {
 	portLock.Lock()
 	defer portLock.Unlock()
 
 	// get ephemeral port range, either default or if custom-defined
-	startHostPortRange, endHostPortRange, err := dynamicHostPortRange()
-	if err != nil {
-		seelog.Warnf("Unable to read the ephemeral host port range, falling back to the default range: %v-%v",
-			defaultPortRangeStart, defaultPortRangeEnd)
-		startHostPortRange, endHostPortRange = defaultPortRangeStart, defaultPortRangeEnd
-	}
-
+	startHostPortRange, endHostPortRange, _ := nat.ParsePortRangeToInt(dynamicHostPortRange)
 	start := startHostPortRange
 	end := endHostPortRange
 

agent/utils/ephemeral_ports_linux.go

@@ -23,17 +23,17 @@ import (
 )
 
 const (
-	// defaultPortRangeStart indicates the first port in ephemeral port range
-	defaultPortRangeStart = 49153
-	// defaultPortRangeEnd indicates the last port in ephemeral port range
-	defaultPortRangeEnd = 65535
 	// portRangeKernelParam is a kernel parameter that defines the ephemeral port range
 	portRangeKernelParam = "/proc/sys/net/ipv4/ip_local_port_range"
+	// DefaultPortRangeStart indicates the first port in ephemeral port range
+	DefaultPortRangeStart = 49153
+	// DefaultPortRangeEnd indicates the last port in ephemeral port range
+	DefaultPortRangeEnd = 65535
 )
 
-// getDynamicHostPortRange returns the ephemeral port range defined by the "/proc/sys/net/ipv4/ip_local_port_range"
+// GetDynamicHostPortRange returns the ephemeral port range defined by the "/proc/sys/net/ipv4/ip_local_port_range"
 // kernel parameter. Ref: https://github.com/moby/moby/blob/master/libnetwork/portallocator/portallocator_linux.go
-func getDynamicHostPortRange() (start int, end int, err error) {
+func GetDynamicHostPortRange() (start int, end int, err error) {
 	file, err := os.Open(portRangeKernelParam)
 	if err != nil {
 		return 0, 0, err

agent/utils/ephemeral_ports_test.go

@@ -76,6 +76,7 @@ func TestGetHostPortRange(t *testing.T) {
 	testCases := []struct {
 		testName                 string
 		numberOfPorts            int
+		testDynamicHostPortRange string
 		protocol                 string
 		expectedLastAssignedPort []int
 		numberOfRequests         int
@@ -84,6 +85,7 @@ func TestGetHostPortRange(t *testing.T) {
 		{
 			testName:                 "tcp protocol, contiguous hostPortRange found",
 			numberOfPorts:            10,
+			testDynamicHostPortRange: "40001-40080",
 			protocol:                 testTCPProtocol,
 			expectedLastAssignedPort: []int{40010},
 			numberOfRequests:         1,
@@ -92,6 +94,7 @@ func TestGetHostPortRange(t *testing.T) {
 		{
 			testName:                 "udp protocol, contiguous hostPortRange found",
 			numberOfPorts:            30,
+			testDynamicHostPortRange: "40001-40080",
 			protocol:                 testUDPProtocol,
 			expectedLastAssignedPort: []int{40040},
 			numberOfRequests:         1,
@@ -100,6 +103,7 @@ func TestGetHostPortRange(t *testing.T) {
 		{
 			testName:                 "2 requests for contiguous hostPortRange in succession, success",
 			numberOfPorts:            20,
+			testDynamicHostPortRange: "40001-40080",
 			protocol:                 testTCPProtocol,
 			expectedLastAssignedPort: []int{40060, 40000},
 			numberOfRequests:         2,
@@ -108,33 +112,28 @@ func TestGetHostPortRange(t *testing.T) {
 		{
 			testName:                 "contiguous hostPortRange after looping back, success",
 			numberOfPorts:            15,
+			testDynamicHostPortRange: "40001-40080",
 			protocol:                 testUDPProtocol,
 			expectedLastAssignedPort: []int{40015},
 			numberOfRequests:         1,
 			expectedError:            nil,
 		},
 		{
-			testName:         "contiguous hostPortRange not found",
-			numberOfPorts:    20,
-			protocol:         testTCPProtocol,
-			numberOfRequests: 1,
-			expectedError:    errors.New("20 contiguous host ports unavailable"),
+			testName:                 "contiguous hostPortRange not found",
+			numberOfPorts:            20,
+			testDynamicHostPortRange: "40001-40005",
+			protocol:                 testTCPProtocol,
+			numberOfRequests:         1,
+			expectedError:            errors.New("20 contiguous host ports unavailable"),
 		},
 	}
 
-	defer func() {
-		dynamicHostPortRange = getDynamicHostPortRange
-	}()
-
 	for _, tc := range testCases {
 		t.Run(tc.testName, func(t *testing.T) {
 			for i := 0; i < tc.numberOfRequests; i++ {
 				if tc.expectedError == nil {
-					dynamicHostPortRange = func() (start int, end int, err error) {
-						return 40001, 40080, nil
-					}
 
-					hostPortRange, err := GetHostPortRange(tc.numberOfPorts, tc.protocol)
+					hostPortRange, err := GetHostPortRange(tc.numberOfPorts, tc.protocol, tc.testDynamicHostPortRange)
 					assert.NoError(t, err)
 
 					numberOfHostPorts, err := getPortRangeLength(hostPortRange)
@@ -147,11 +146,7 @@ func TestGetHostPortRange(t *testing.T) {
 					// need to reset the tracker to avoid getting data from previous test cases
 					tracker.SetLastAssignedHostPort(0)
 
-					dynamicHostPortRange = func() (start int, end int, err error) {
-						return 40001, 40005, nil
-					}
-
-					hostPortRange, err := GetHostPortRange(tc.numberOfPorts, tc.protocol)
+					hostPortRange, err := GetHostPortRange(tc.numberOfPorts, tc.protocol, tc.testDynamicHostPortRange)
 					assert.Equal(t, tc.expectedError, err)
 					assert.Equal(t, "", hostPortRange)
 				}

agent/utils/ephemeral_ports_unsupported.go

@@ -17,13 +17,13 @@
 package utils
 
 const (
-	// defaultPortRangeStart indicates the first port in ephemeral port range
-	defaultPortRangeStart = 49153
-	// defaultPortRangeEnd indicates the last port in ephemeral port range
-	defaultPortRangeEnd = 65535
+	// DefaultPortRangeStart indicates the first port in ephemeral port range
+	DefaultPortRangeStart = 49153
+	// DefaultPortRangeEnd indicates the last port in ephemeral port range
+	DefaultPortRangeEnd = 65535
 )
 
-// getDynamicHostPortRange returns the default ephemeral port range
-func getDynamicHostPortRange() (start int, end int, err error) {
-	return defaultPortRangeStart, defaultPortRangeEnd, nil
+// GetDynamicHostPortRange returns the default ephemeral port range
+func GetDynamicHostPortRange() (start int, end int, err error) {
+	return DefaultPortRangeStart, DefaultPortRangeEnd, nil
 }