From f96a0614d3aea9abfab624a8469539057aaee775 Mon Sep 17 00:00:00 2001 From: SreeeS Date: Tue, 20 Dec 2022 12:48:09 -0600 Subject: [PATCH] Add new config option for DynamicHostPortRange --- README.md | 1 + agent/api/task/task.go | 6 +-- agent/api/task/task_test.go | 6 +-- agent/config/config.go | 1 + agent/config/config_test.go | 60 ++++++++++++++++++++++ agent/config/parse.go | 21 ++++++++ agent/config/types.go | 6 +++ agent/utils/ephemeral_ports.go | 13 ++--- agent/utils/ephemeral_ports_linux.go | 12 ++--- agent/utils/ephemeral_ports_test.go | 31 +++++------ agent/utils/ephemeral_ports_unsupported.go | 14 ++--- agent/utils/ephemeral_ports_windows.go | 14 ++--- 12 files changed, 131 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index 9e3d84d0210..73f09d6c880 100644 --- a/README.md +++ b/README.md @@ -205,6 +205,7 @@ additional details on each available environment variable. | `NO_PROXY` | | The HTTP traffic that should not be forwarded to the specified HTTP_PROXY. You must specify 169.254.169.254,/var/run/docker.sock to filter Amazon EC2 instance metadata and Docker daemon traffic from the proxy. | `null` | `null` | | `CREDENTIALS_FETCHER_HOST` | `unix:///var/credentials-fetcher/socket/credentials_fetcher.sock` | Used to create a connection to the [credentials-fetcher daemon](https://github.com/aws/credentials-fetcher); to support gMSA on Linux. The default is fine for most users, only needs to be modified if user is configuring a custom credentials-fetcher socket path, ie, [CF_UNIX_DOMAIN_SOCKET_DIR](https://github.com/aws/credentials-fetcher#default-environment-variables). | `unix:///var/credentials-fetcher/socket/credentials_fetcher.sock` | Not Applicable | | `CREDENTIALS_FETCHER_SECRET_NAME_FOR_DOMAINLESS_GMSA` | `secretmanager-secretname` | Used to support scaling option for gMSA on Linux [credentials-fetcher daemon](https://github.com/aws/credentials-fetcher). If user is configuring gMSA on a non-domain joined instance, they need to create an Active Directory user with access to retrieve principals for the gMSA account and store it in secrets manager | `secretmanager-secretname` | Not Applicable | +| `ECS_DYNAMIC_HOST_PORT_RANGE` | `100-200` | This specifies the dynamic host port range that the agent uses to assign host ports from, for a container port range mapping. | Defined by `/proc/sys/net/ipv4/ip_local_port_range` | `49152-65535` | ### Persistence When you run the Amazon ECS Container Agent in production, its `datadir` should be persisted between runs of the Docker diff --git a/agent/api/task/task.go b/agent/api/task/task.go index 369332c9906..d2f95d3b192 100644 --- a/agent/api/task/task.go +++ b/agent/api/task/task.go @@ -1860,7 +1860,7 @@ func (task *Task) dockerHostConfig(container *apicontainer.Container, dockerCont if err != nil { return nil, &apierrors.HostConfigError{Msg: err.Error()} } - dockerPortMap, err := task.dockerPortMap(container) + dockerPortMap, err := task.dockerPortMap(container, cfg.DynamicHostPortRange) if err != nil { return nil, &apierrors.HostConfigError{Msg: fmt.Sprintf("error retrieving docker port map: %+v", err.Error())} } @@ -2334,7 +2334,7 @@ func (task *Task) dockerLinks(container *apicontainer.Container, dockerContainer var getHostPortRange = utils.GetHostPortRange -func (task *Task) dockerPortMap(container *apicontainer.Container) (nat.PortMap, error) { +func (task *Task) dockerPortMap(container *apicontainer.Container, dynamicHostPortRange string) (nat.PortMap, error) { dockerPortMap := nat.PortMap{} scContainer := task.GetServiceConnectContainer() containerToCheck := container @@ -2402,7 +2402,7 @@ func (task *Task) dockerPortMap(container *apicontainer.Container) (nat.PortMap, // we will try to get a contiguous set of host ports from the ephemeral host port range. // this is to ensure that docker maps host ports in a contiguous manner, and // we are guaranteed to have the entire hostPortRange in a single network binding while sending this info to ECS. - hostPortRange, err := getHostPortRange(numberOfPorts, protocol) + hostPortRange, err := getHostPortRange(numberOfPorts, protocol, dynamicHostPortRange) if err != nil { // in the odd case where we're unable to find a contiguous set of host ports, we fall back to docker dynamic port // assignment for the requested ContainerPortRange. diff --git a/agent/api/task/task_test.go b/agent/api/task/task_test.go index 54b697eea45..b1e1cecf458 100644 --- a/agent/api/task/task_test.go +++ b/agent/api/task/task_test.go @@ -282,7 +282,7 @@ func TestDockerHostConfigPortBinding(t *testing.T) { testCases := []struct { testName string testTask *Task - getHostPortRange func(numberOfPorts int, protocol string) (string, error) + getHostPortRange func(numberOfPorts int, protocol string, dynamicHostPortRange string) (string, error) expectedPortBinding nat.PortMap expectedContainerPortSet map[int]struct{} expectedContainerPortRangeMap map[string]string @@ -303,7 +303,7 @@ func TestDockerHostConfigPortBinding(t *testing.T) { { testName: "2 port bindings, each with container port range, 1 found valid host port range, other didn't", testTask: testTask2, - getHostPortRange: func(numberOfPorts int, protocol string) (string, error) { + getHostPortRange: func(numberOfPorts int, protocol string, dynamicHostPortRange string) (string, error) { if numberOfPorts == 3 { return "", errors.New("couldn't find host ports") } @@ -325,7 +325,7 @@ func TestDockerHostConfigPortBinding(t *testing.T) { { testName: "2 port bindings, one with container port range, other with singular container port", testTask: testTask3, - getHostPortRange: func(numberOfPorts int, protocol string) (string, error) { + getHostPortRange: func(numberOfPorts int, protocol string, dynamicHostPortRange string) (string, error) { return "155-157", nil }, expectedPortBinding: nat.PortMap{ diff --git a/agent/config/config.go b/agent/config/config.go index 4b308bb53e5..d8c3d98f697 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -596,6 +596,7 @@ func environmentConfig() (Config, error) { EnableRuntimeStats: parseBooleanDefaultFalseConfig("ECS_ENABLE_RUNTIME_STATS"), ShouldExcludeIPv6PortBinding: parseBooleanDefaultTrueConfig("ECS_EXCLUDE_IPV6_PORTBINDING"), WarmPoolsSupport: parseBooleanDefaultFalseConfig("ECS_WARM_POOLS_CHECK"), + DynamicHostPortRange: parseDynamicHostPortRange("ECS_DYNAMIC_HOST_PORT_RANGE"), }, err } diff --git a/agent/config/config_test.go b/agent/config/config_test.go index f6b92f0c036..d902771ba9c 100644 --- a/agent/config/config_test.go +++ b/agent/config/config_test.go @@ -19,6 +19,7 @@ package config import ( "encoding/json" "errors" + "fmt" "os" "testing" "time" @@ -26,6 +27,7 @@ import ( "github.com/aws/amazon-ecs-agent/agent/dockerclient" "github.com/aws/amazon-ecs-agent/agent/ec2" mock_ec2 "github.com/aws/amazon-ecs-agent/agent/ec2/mocks" + "github.com/aws/amazon-ecs-agent/agent/utils" "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/golang/mock/gomock" @@ -160,6 +162,7 @@ func TestEnvironmentConfig(t *testing.T) { defer setTestEnv("ECS_ENABLE_RUNTIME_STATS", "true")() defer setTestEnv("ECS_EXCLUDE_IPV6_PORTBINDING", "true")() defer setTestEnv("ECS_WARM_POOLS_CHECK", "false")() + defer setTestEnv("ECS_DYNAMIC_HOST_PORT_RANGE", "200-300")() additionalLocalRoutesJSON := `["1.2.3.4/22","5.6.7.8/32"]` setTestEnv("ECS_AWSVPC_ADDITIONAL_LOCAL_ROUTES", additionalLocalRoutesJSON) setTestEnv("ECS_ENABLE_CONTAINER_METADATA", "true") @@ -219,6 +222,7 @@ func TestEnvironmentConfig(t *testing.T) { assert.True(t, conf.EnableRuntimeStats.Enabled(), "Wrong value for EnableRuntimeStats") assert.True(t, conf.ShouldExcludeIPv6PortBinding.Enabled(), "Wrong value for ShouldExcludeIPv6PortBinding") assert.False(t, conf.WarmPoolsSupport.Enabled(), "Wrong value for WarmPoolsSupport") + assert.Equal(t, "200-300", conf.DynamicHostPortRange) } func TestTrimWhitespaceWhenCreating(t *testing.T) { @@ -494,6 +498,62 @@ func TestValidFormatParseEnvVariableDuration(t *testing.T) { assert.Equal(t, 1*time.Second, duration, "Unexpected value parsed in parseEnvVariableDuration.") } +func TestParseDynamicHostPortRange(t *testing.T) { + testCases := []struct { + testName string + testDynamicHostPortRangeVal string + expectedPortRangeVal string + expectedErrorDynamicHostPortRange error + expectedErrorEphemeralHostPortRange error + }{ + { + testName: "Parse DynamicHostPortRange for valid DynamicHostPortRange value", + testDynamicHostPortRangeVal: "200-300", + expectedPortRangeVal: "200-300", + expectedErrorDynamicHostPortRange: nil, + expectedErrorEphemeralHostPortRange: nil, + }, + { + testName: "Parse DynamicHostPortRange for Invalid DynamicHostPortRange value", + testDynamicHostPortRangeVal: "test1", + expectedPortRangeVal: "300-400", + expectedErrorDynamicHostPortRange: errors.New("Invalid DynamicHostPortRange"), + expectedErrorEphemeralHostPortRange: nil, + }, + { + testName: "Invalid DynamicHostPortRange value and error on getDynamicHostPortRange value", + testDynamicHostPortRangeVal: "test2", + expectedPortRangeVal: fmt.Sprintf("%d-%d", utils.DefaultPortRangeStart, utils.DefaultPortRangeEnd), + expectedErrorDynamicHostPortRange: nil, + expectedErrorEphemeralHostPortRange: errors.New("Error getting EphemeralHostPortRange"), + }, + } + defer func() { + getDynamicHostPortRange = utils.GetDynamicHostPortRange + }() + for _, tc := range testCases { + t.Run(tc.testName, func(t *testing.T) { + defer setTestRegion()() + defer setTestEnv("ECS_DYNAMIC_HOST_PORT_RANGE", tc.testDynamicHostPortRangeVal)() + + if tc.expectedErrorDynamicHostPortRange != nil { + getDynamicHostPortRange = func() (start int, end int, err error) { + return 300, 400, nil + } + } + + if tc.expectedErrorEphemeralHostPortRange != nil { + getDynamicHostPortRange = func() (start int, end int, err error) { + return 10, 20, errors.New("test default values") + } + } + + dynamicHostPortRange := parseDynamicHostPortRange("ECS_DYNAMIC_HOST_PORT_RANGE") + assert.Equal(t, tc.expectedPortRangeVal, dynamicHostPortRange) + }) + } +} + func TestInvalidTaskCleanupTimeoutOverridesToThreeHours(t *testing.T) { defer setTestRegion()() setTestEnv("ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION", "1ms") diff --git a/agent/config/parse.go b/agent/config/parse.go index f8f018f3d41..773942333cc 100644 --- a/agent/config/parse.go +++ b/agent/config/parse.go @@ -23,8 +23,11 @@ import ( "time" "github.com/aws/amazon-ecs-agent/agent/dockerclient" + "github.com/aws/amazon-ecs-agent/agent/utils" + "github.com/cihub/seelog" cnitypes "github.com/containernetworking/cni/pkg/types" + "github.com/docker/go-connections/nat" ) const ( @@ -372,3 +375,21 @@ func parseCgroupCPUPeriod() time.Duration { return defaultCgroupCPUPeriod } + +var getDynamicHostPortRange = utils.GetDynamicHostPortRange + +func parseDynamicHostPortRange(dynamicHostPortRangeEnv string) string { + dynamicHostPortRange := os.Getenv(dynamicHostPortRangeEnv) + _, _, err := nat.ParsePortRangeToInt(dynamicHostPortRange) + if err != nil { + seelog.Warnf("Unable to read the dynamicHostPortRange value: %s", dynamicHostPortRange) + startHostPortRange, endHostPortRange, err := getDynamicHostPortRange() + if err != nil { + seelog.Warnf("Unable to read the ephemeral host port range, "+ + "falling back to the default range: %v-%v", utils.DefaultPortRangeStart, utils.DefaultPortRangeEnd) + return fmt.Sprintf("%d-%d", utils.DefaultPortRangeStart, utils.DefaultPortRangeEnd) + } + return fmt.Sprintf("%d-%d", startHostPortRange, endHostPortRange) + } + return dynamicHostPortRange +} diff --git a/agent/config/types.go b/agent/config/types.go index fa6da0fe6d7..68c6eee6cd5 100644 --- a/agent/config/types.go +++ b/agent/config/types.go @@ -33,6 +33,7 @@ type Config struct { // ClusterArn is the Name or full ARN of a Cluster to register into. It has // been deprecated (and will eventually be removed) in favor of Cluster ClusterArn string `deprecated:"Please use Cluster instead"` + // Cluster can either be the Name or full ARN of a Cluster. This is the // cluster the agent should register this ContainerInstance into. If this // value is not set, it will default to "default" @@ -359,4 +360,9 @@ type Config struct { // WarmPoolsSupport specifies whether the agent should poll IMDS to check the target lifecycle state for a starting // instance WarmPoolsSupport BooleanDefaultFalse + + // DynamicHostPortRange specifies the dynamic host port range that the agent + // uses to assign host ports from, for a container port range mapping. + // This defaults to the platform specific ephemeral host port range + DynamicHostPortRange string } diff --git a/agent/utils/ephemeral_ports.go b/agent/utils/ephemeral_ports.go index 421070a128b..25caaedb6ac 100644 --- a/agent/utils/ephemeral_ports.go +++ b/agent/utils/ephemeral_ports.go @@ -21,7 +21,7 @@ import ( "sync" "time" - "github.com/cihub/seelog" + "github.com/docker/go-connections/nat" ) // From https://www.kernel.org/doc/html/latest//networking/ip-sysctl.html#ip-variables @@ -89,22 +89,15 @@ func (pt *safePortTracker) GetLastAssignedHostPort() int { return pt.lastAssignedHostPort } -var dynamicHostPortRange = getDynamicHostPortRange var tracker safePortTracker // GetHostPortRange gets N contiguous host ports from the ephemeral host port range defined on the host. -func GetHostPortRange(numberOfPorts int, protocol string) (string, error) { +func GetHostPortRange(numberOfPorts int, protocol string, dynamicHostPortRange string) (string, error) { portLock.Lock() defer portLock.Unlock() // get ephemeral port range, either default or if custom-defined - startHostPortRange, endHostPortRange, err := dynamicHostPortRange() - if err != nil { - seelog.Warnf("Unable to read the ephemeral host port range, falling back to the default range: %v-%v", - defaultPortRangeStart, defaultPortRangeEnd) - startHostPortRange, endHostPortRange = defaultPortRangeStart, defaultPortRangeEnd - } - + startHostPortRange, endHostPortRange, _ := nat.ParsePortRangeToInt(dynamicHostPortRange) start := startHostPortRange end := endHostPortRange diff --git a/agent/utils/ephemeral_ports_linux.go b/agent/utils/ephemeral_ports_linux.go index fa00afa7dc3..b959326aef1 100644 --- a/agent/utils/ephemeral_ports_linux.go +++ b/agent/utils/ephemeral_ports_linux.go @@ -23,17 +23,17 @@ import ( ) const ( - // defaultPortRangeStart indicates the first port in ephemeral port range - defaultPortRangeStart = 49153 - // defaultPortRangeEnd indicates the last port in ephemeral port range - defaultPortRangeEnd = 65535 // portRangeKernelParam is a kernel parameter that defines the ephemeral port range portRangeKernelParam = "/proc/sys/net/ipv4/ip_local_port_range" + // DefaultPortRangeStart indicates the first port in ephemeral port range + DefaultPortRangeStart = 49153 + // DefaultPortRangeEnd indicates the last port in ephemeral port range + DefaultPortRangeEnd = 65535 ) -// getDynamicHostPortRange returns the ephemeral port range defined by the "/proc/sys/net/ipv4/ip_local_port_range" +// GetDynamicHostPortRange returns the ephemeral port range defined by the "/proc/sys/net/ipv4/ip_local_port_range" // kernel parameter. Ref: https://github.com/moby/moby/blob/master/libnetwork/portallocator/portallocator_linux.go -func getDynamicHostPortRange() (start int, end int, err error) { +func GetDynamicHostPortRange() (start int, end int, err error) { file, err := os.Open(portRangeKernelParam) if err != nil { return 0, 0, err diff --git a/agent/utils/ephemeral_ports_test.go b/agent/utils/ephemeral_ports_test.go index 9642174fd48..a8ef7c9b88f 100644 --- a/agent/utils/ephemeral_ports_test.go +++ b/agent/utils/ephemeral_ports_test.go @@ -76,6 +76,7 @@ func TestGetHostPortRange(t *testing.T) { testCases := []struct { testName string numberOfPorts int + testDynamicHostPortRange string protocol string expectedLastAssignedPort []int numberOfRequests int @@ -84,6 +85,7 @@ func TestGetHostPortRange(t *testing.T) { { testName: "tcp protocol, contiguous hostPortRange found", numberOfPorts: 10, + testDynamicHostPortRange: "40001-40080", protocol: testTCPProtocol, expectedLastAssignedPort: []int{40010}, numberOfRequests: 1, @@ -92,6 +94,7 @@ func TestGetHostPortRange(t *testing.T) { { testName: "udp protocol, contiguous hostPortRange found", numberOfPorts: 30, + testDynamicHostPortRange: "40001-40080", protocol: testUDPProtocol, expectedLastAssignedPort: []int{40040}, numberOfRequests: 1, @@ -100,6 +103,7 @@ func TestGetHostPortRange(t *testing.T) { { testName: "2 requests for contiguous hostPortRange in succession, success", numberOfPorts: 20, + testDynamicHostPortRange: "40001-40080", protocol: testTCPProtocol, expectedLastAssignedPort: []int{40060, 40000}, numberOfRequests: 2, @@ -108,33 +112,28 @@ func TestGetHostPortRange(t *testing.T) { { testName: "contiguous hostPortRange after looping back, success", numberOfPorts: 15, + testDynamicHostPortRange: "40001-40080", protocol: testUDPProtocol, expectedLastAssignedPort: []int{40015}, numberOfRequests: 1, expectedError: nil, }, { - testName: "contiguous hostPortRange not found", - numberOfPorts: 20, - protocol: testTCPProtocol, - numberOfRequests: 1, - expectedError: errors.New("20 contiguous host ports unavailable"), + testName: "contiguous hostPortRange not found", + numberOfPorts: 20, + testDynamicHostPortRange: "40001-40005", + protocol: testTCPProtocol, + numberOfRequests: 1, + expectedError: errors.New("20 contiguous host ports unavailable"), }, } - defer func() { - dynamicHostPortRange = getDynamicHostPortRange - }() - for _, tc := range testCases { t.Run(tc.testName, func(t *testing.T) { for i := 0; i < tc.numberOfRequests; i++ { if tc.expectedError == nil { - dynamicHostPortRange = func() (start int, end int, err error) { - return 40001, 40080, nil - } - hostPortRange, err := GetHostPortRange(tc.numberOfPorts, tc.protocol) + hostPortRange, err := GetHostPortRange(tc.numberOfPorts, tc.protocol, tc.testDynamicHostPortRange) assert.NoError(t, err) numberOfHostPorts, err := getPortRangeLength(hostPortRange) @@ -147,11 +146,7 @@ func TestGetHostPortRange(t *testing.T) { // need to reset the tracker to avoid getting data from previous test cases tracker.SetLastAssignedHostPort(0) - dynamicHostPortRange = func() (start int, end int, err error) { - return 40001, 40005, nil - } - - hostPortRange, err := GetHostPortRange(tc.numberOfPorts, tc.protocol) + hostPortRange, err := GetHostPortRange(tc.numberOfPorts, tc.protocol, tc.testDynamicHostPortRange) assert.Equal(t, tc.expectedError, err) assert.Equal(t, "", hostPortRange) } diff --git a/agent/utils/ephemeral_ports_unsupported.go b/agent/utils/ephemeral_ports_unsupported.go index 86c3d316fcc..f59dd3f5e4d 100644 --- a/agent/utils/ephemeral_ports_unsupported.go +++ b/agent/utils/ephemeral_ports_unsupported.go @@ -17,13 +17,13 @@ package utils const ( - // defaultPortRangeStart indicates the first port in ephemeral port range - defaultPortRangeStart = 49153 - // defaultPortRangeEnd indicates the last port in ephemeral port range - defaultPortRangeEnd = 65535 + // DefaultPortRangeStart indicates the first port in ephemeral port range + DefaultPortRangeStart = 49153 + // DefaultPortRangeEnd indicates the last port in ephemeral port range + DefaultPortRangeEnd = 65535 ) -// getDynamicHostPortRange returns the default ephemeral port range -func getDynamicHostPortRange() (start int, end int, err error) { - return defaultPortRangeStart, defaultPortRangeEnd, nil +// GetDynamicHostPortRange returns the default ephemeral port range +func GetDynamicHostPortRange() (start int, end int, err error) { + return DefaultPortRangeStart, DefaultPortRangeEnd, nil } diff --git a/agent/utils/ephemeral_ports_windows.go b/agent/utils/ephemeral_ports_windows.go index d33ba61b6bc..79d5473963c 100644 --- a/agent/utils/ephemeral_ports_windows.go +++ b/agent/utils/ephemeral_ports_windows.go @@ -5,14 +5,14 @@ package utils const ( // Ref: https://learn.microsoft.com/en-US/troubleshoot/windows-server/networking/default-dynamic-port-range-tcpip-chang - // defaultPortRangeStart indicates the first port in ephemeral port range - defaultPortRangeStart = 49152 - // defaultPortRangeEnd indicates the last port in ephemeral port range - defaultPortRangeEnd = 65535 + // DefaultPortRangeStart indicates the first port in ephemeral port range + DefaultPortRangeStart = 49152 + // DefaultPortRangeEnd indicates the last port in ephemeral port range + DefaultPortRangeEnd = 65535 ) -// getDynamicHostPortRange returns the default ephemeral port range on Windows. +// GetDynamicHostPortRange returns the default ephemeral port range on Windows. // TODO: instead of sticking to defaults, run netsh commands on the host to get the ranges. -func getDynamicHostPortRange() (start int, end int, err error) { - return defaultPortRangeStart, defaultPortRangeEnd, nil +func GetDynamicHostPortRange() (start int, end int, err error) { + return DefaultPortRangeStart, DefaultPortRangeEnd, nil }