From ba1c1e2eff8d6cc272c0de36257abc1a526d1fce Mon Sep 17 00:00:00 2001 From: chengyouling Date: Thu, 26 Sep 2024 17:50:38 +0800 Subject: [PATCH 1/2] [#4532] Fixed when the servicecomb engine storage pool is faulty, the registration and configuration centers cannot perform HA switchover problem --- .../center/client/ConfigCenterClient.java | 22 +++++ .../center/client/ConfigCenterManager.java | 17 ++++ .../center/client/ConfigCenterOperation.java | 8 ++ .../config/kie/client/KieClient.java | 16 ++++ .../config/kie/client/KieConfigManager.java | 22 +++++ .../config/kie/client/KieConfigOperation.java | 8 ++ .../client/common/AbstractAddressManager.java | 85 ++++--------------- .../common/AbstractAddressManagerTest.java | 12 +-- .../center/client/ServiceCenterClient.java | 12 +++ .../center/client/ServiceCenterOperation.java | 8 ++ .../center/client/ServiceCenterRawClient.java | 26 ++++-- .../client/ServiceCenterRegistration.java | 11 +++ 12 files changed, 162 insertions(+), 85 deletions(-) diff --git a/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterClient.java b/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterClient.java index cde37e920dd..4232c3db581 100644 --- a/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterClient.java +++ b/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterClient.java @@ -132,6 +132,28 @@ public QueryConfigurationsResponse queryConfigurations(QueryConfigurationsReques } } + @Override + public void checkAddressAvailable(QueryConfigurationsRequest request, String address) { + String dimensionsInfo = buildDimensionsInfo(request, true); + try { + String uri = address + "/configuration/items?dimensionsInfo=" + + HttpUtils.encodeURLParam(dimensionsInfo) + "&revision=" + request.getRevision(); + + Map headers = new HashMap<>(); + headers.put("x-environment", request.getEnvironment()); + HttpRequest httpRequest = new HttpRequest(uri, headers, null, + HttpRequest.GET); + + HttpResponse httpResponse = httpTransport.doRequest(httpRequest); + if (httpResponse.getStatusCode() == HttpStatus.SC_NOT_MODIFIED + || httpResponse.getStatusCode() == HttpStatus.SC_OK) { + addressManager.recoverIsolatedAddress(address); + } + } catch (Exception e) { + LOGGER.error("check config center isolation address {} available error!", address); + } + } + private String buildDimensionsInfo(QueryConfigurationsRequest request, boolean withVersion) { String result = request.getServiceName() + DEFAULT_APP_SEPARATOR diff --git a/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterManager.java b/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterManager.java index a5a4f90d31a..5b8c434045c 100644 --- a/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterManager.java +++ b/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterManager.java @@ -17,6 +17,7 @@ package org.apache.servicecomb.config.center.client; +import java.util.List; import java.util.Map; import org.apache.servicecomb.config.center.client.model.ConfigCenterConfiguration; @@ -63,6 +64,7 @@ public void setQueryConfigurationsRequest(QueryConfigurationsRequest queryConfig public void startConfigCenterManager() { this.startTask(new PollConfigurationTask(0)); + this.startTask(new CheckConfigCenterAddressTask()); } class PollConfigurationTask implements Task { @@ -91,4 +93,19 @@ public void execute() { } } } + + class CheckConfigCenterAddressTask implements Task { + @Override + public void execute() { + List isolationAddresses = configCenterAddressManager.getIsolationAddresses(); + if (isolationAddresses.isEmpty()) { + return; + } + for (String address : isolationAddresses) { + configCenterClient.checkAddressAvailable(queryConfigurationsRequest, address); + } + startTask(new BackOffSleepTask(configCenterConfiguration.getRefreshIntervalInMillis(), + new CheckConfigCenterAddressTask())); + } + } } diff --git a/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterOperation.java b/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterOperation.java index d05967692ec..43b30c74831 100644 --- a/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterOperation.java +++ b/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterOperation.java @@ -30,4 +30,12 @@ public interface ConfigCenterOperation { * @throws OperationException If some problems happened to contact service center or non http 200 returned. */ QueryConfigurationsResponse queryConfigurations(QueryConfigurationsRequest request, String address); + + /** + * Check config center isolation address available + * + * @param request queryConfigurationsRequest + * @param address isolation address + */ + void checkAddressAvailable(QueryConfigurationsRequest request, String address); } diff --git a/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieClient.java b/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieClient.java index 50cf2b873b9..e85a7c65874 100644 --- a/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieClient.java +++ b/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieClient.java @@ -19,6 +19,7 @@ import com.google.common.eventbus.EventBus; +import java.io.IOException; import java.io.StringReader; import java.nio.charset.StandardCharsets; import java.util.Collections; @@ -113,6 +114,21 @@ public ConfigurationsResponse queryConfigurations(ConfigurationsRequest request, } } + @Override + public void checkAddressAvailable(ConfigurationsRequest request, String address) { + String url = buildUrl(request, address); + HttpRequest httpRequest = new HttpRequest(url, null, null, HttpRequest.GET); + try { + HttpResponse httpResponse = httpTransport.doRequest(httpRequest); + if (httpResponse.getStatusCode() == HttpStatus.SC_NOT_MODIFIED + || httpResponse.getStatusCode() == HttpStatus.SC_OK) { + addressManager.recoverIsolatedAddress(address); + } + } catch (IOException e) { + LOGGER.error("check kie config isolation address {} available error!", address, e); + } + } + private String buildUrl(ConfigurationsRequest request, String currentAddress) { StringBuilder sb = new StringBuilder(); sb.append(currentAddress); diff --git a/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigManager.java b/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigManager.java index 2a8b4cd1c44..bd5ff875c7f 100644 --- a/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigManager.java +++ b/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigManager.java @@ -125,6 +125,7 @@ protected void initTaskPool(String taskName) { public void startConfigKieManager() { this.configurationsRequests.forEach((t) -> this.startTask(new PollConfigurationTask(0, t))); + startTask(new CheckKieConfigAddressTask(configurationsRequests.get(0))); } class PollConfigurationTask implements Task { @@ -159,4 +160,25 @@ public void execute() { } } } + + class CheckKieConfigAddressTask implements Task { + ConfigurationsRequest configurationsRequest; + + public CheckKieConfigAddressTask(ConfigurationsRequest configurationsRequest) { + this.configurationsRequest = configurationsRequest; + } + + @Override + public void execute() { + List isolationAddresses = kieAddressManager.getIsolationAddresses(); + if (isolationAddresses.isEmpty()) { + return; + } + for (String address : isolationAddresses) { + configKieClient.checkAddressAvailable(this.configurationsRequest, address); + } + startTask(new BackOffSleepTask(kieConfiguration.getRefreshIntervalInMillis(), + new CheckKieConfigAddressTask(this.configurationsRequest))); + } + } } diff --git a/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigOperation.java b/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigOperation.java index 229439867cf..44f66137fef 100644 --- a/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigOperation.java +++ b/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigOperation.java @@ -31,4 +31,12 @@ public interface KieConfigOperation { * @throws OperationException If some problems happened to contact service center or non http 200 returned. */ ConfigurationsResponse queryConfigurations(ConfigurationsRequest request, String address); + + /** + * Check kie isolation address available + * + * @param configurationsRequest configurationsRequest + * @param address isolation address + */ + void checkAddressAvailable(ConfigurationsRequest configurationsRequest, String address); } diff --git a/clients/http-client-common/src/main/java/org/apache/servicecomb/http/client/common/AbstractAddressManager.java b/clients/http-client-common/src/main/java/org/apache/servicecomb/http/client/common/AbstractAddressManager.java index 261ecc36bda..59cb3e5be8c 100644 --- a/clients/http-client-common/src/main/java/org/apache/servicecomb/http/client/common/AbstractAddressManager.java +++ b/clients/http-client-common/src/main/java/org/apache/servicecomb/http/client/common/AbstractAddressManager.java @@ -17,21 +17,11 @@ package org.apache.servicecomb.http.client.common; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.net.URI; -import java.net.URISyntaxException; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Random; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.commons.lang.StringUtils; @@ -42,7 +32,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.eventbus.EventBus; -import com.google.common.util.concurrent.ThreadFactoryBuilder; public class AbstractAddressManager { private static final Logger LOGGER = LoggerFactory.getLogger(AbstractAddressManager.class); @@ -53,8 +42,6 @@ public class AbstractAddressManager { private static final String V3_PREFIX = "/v3/"; - private static final int DEFAULT_ADDRESS_CHECK_TIME = 30; - private static final int ISOLATION_THRESHOLD = 3; private volatile List addresses = new ArrayList<>(); @@ -68,9 +55,6 @@ public class AbstractAddressManager { private String projectName; - // all address list. - private final Set addressCategory = new HashSet<>(); - // recording continuous times of failure of an address. private final Map addressFailureStatus = new ConcurrentHashMap<>(); @@ -90,27 +74,22 @@ public class AbstractAddressManager { private EventBus eventBus; - private final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1, - new ThreadFactoryBuilder() - .setNameFormat("check-available-address-%d") - .build()); - public AbstractAddressManager(List addresses) { this.projectName = DEFAULT_PROJECT; this.addresses.addAll(addresses); this.defaultAddress.addAll(addresses); - this.addressCategory.addAll(addresses); - this.index = !addresses.isEmpty() ? random.nextInt(addresses.size()) : 0; - startCheck(); + this.index = !addresses.isEmpty() ? getRandomIndex() : 0; } public AbstractAddressManager(String projectName, List addresses) { this.projectName = StringUtils.isEmpty(projectName) ? DEFAULT_PROJECT : projectName; this.addresses = this.transformAddress(addresses); this.defaultAddress.addAll(addresses); - this.addressCategory.addAll(this.addresses); - this.index = !addresses.isEmpty() ? random.nextInt(addresses.size()) : 0; - startCheck(); + this.index = !addresses.isEmpty() ? getRandomIndex() : 0; + } + + private int getRandomIndex() { + return random.nextInt(addresses.size()); } public void refreshEndpoint(RefreshEndpointEvent event, String key) { @@ -120,8 +99,6 @@ public void refreshEndpoint(RefreshEndpointEvent event, String key) { availableZone = event.getSameZone().stream().map(this::normalizeUri).collect(Collectors.toList()); availableRegion = event.getSameRegion().stream().map(this::normalizeUri).collect(Collectors.toList()); - addressCategory.addAll(availableZone); - addressCategory.addAll(availableRegion); addressAutoRefreshed = true; } @@ -146,10 +123,6 @@ public List getAvailableRegion() { return availableRegion; } - private void startCheck() { - executorService.scheduleAtFixedRate(this::checkHistory, 0, DEFAULT_ADDRESS_CHECK_TIME, TimeUnit.SECONDS); - } - public String formatUrl(String url, boolean absoluteUrl, String address) { return absoluteUrl ? address + url : formatAddress(address) + url; } @@ -188,7 +161,7 @@ private String getDefaultAddress() { } LOGGER.warn("all addresses are isolation, please check server status."); // when all addresses are isolation, it will use all default address for polling. - return getCurrentAddress(new ArrayList<>(defaultAddress)); + return getCurrentAddress(defaultAddress); } private String getAvailableZoneAddress() { @@ -221,42 +194,7 @@ private List getZoneOrRegionAddress() { return results; } - @VisibleForTesting - protected void checkHistory() { - addressCategory.forEach(address -> { - if (telnetTest(address)) { - // isolation addresses find address and restore it - findAndRestoreAddress(address); - } else { - recordFailState(address); - } - }); - } - - protected boolean telnetTest(String address) { - URI uri = parseIpPortFromURI(address); - if (uri == null) { - return false; - } - try (Socket s = new Socket()) { - s.connect(new InetSocketAddress(uri.getHost(), uri.getPort()), 3000); - return true; - } catch (IOException e) { - LOGGER.warn("ping endpoint {} failed, It will be quarantined again.", address); - return false; - } - } - - private URI parseIpPortFromURI(String address) { - try { - return new URI(address); - } catch (URISyntaxException e) { - LOGGER.error("parse address [{}] failed.", address, e); - return null; - } - } - - protected void findAndRestoreAddress(String address) { + public void recoverIsolatedAddress(String address) { recordSuccessState(address); if (addressAutoRefreshed) { if (isolationZoneAddress.remove(address)) { @@ -326,4 +264,11 @@ void removeAddress(String address) { public void setEventBus(EventBus eventBus) { this.eventBus = eventBus; } + + public List getIsolationAddresses() { + List isolationAddresses = new ArrayList<>(defaultIsolationAddress); + isolationAddresses.addAll(isolationZoneAddress); + isolationAddresses.addAll(isolationRegionAddress); + return isolationAddresses; + } } diff --git a/clients/http-client-common/src/test/java/org/apache/servicecomb/http/client/common/AbstractAddressManagerTest.java b/clients/http-client-common/src/test/java/org/apache/servicecomb/http/client/common/AbstractAddressManagerTest.java index 9229baa7a0b..83406ec747e 100644 --- a/clients/http-client-common/src/test/java/org/apache/servicecomb/http/client/common/AbstractAddressManagerTest.java +++ b/clients/http-client-common/src/test/java/org/apache/servicecomb/http/client/common/AbstractAddressManagerTest.java @@ -42,8 +42,6 @@ public class AbstractAddressManagerTest { private static AbstractAddressManager addressManager3; - private static int index; - @BeforeEach public void setUp() throws NoSuchFieldException, IllegalAccessException { addresses.add("http://127.0.0.1:30103"); @@ -90,12 +88,7 @@ public void recordStateTest() throws ExecutionException { zoneAndRegion.put("sameZone", addressAZ); zoneAndRegion.put("sameRegion", addressRG); RefreshEndpointEvent event = new RefreshEndpointEvent(zoneAndRegion, "TEST"); - AbstractAddressManager addressManager = new AbstractAddressManager(addresses) { - @Override - protected boolean telnetTest(String address) { - return true; - } - }; + AbstractAddressManager addressManager = new AbstractAddressManager(addresses) {}; addressManager.refreshEndpoint(event, "TEST"); @@ -122,8 +115,7 @@ protected boolean telnetTest(String address) { Assertions.assertEquals("http://127.0.0.4:30100", addressManager.address()); // test restore isolation - addressManager.checkHistory(); - addressManager.findAndRestoreAddress("http://127.0.0.3:30100"); + addressManager.recoverIsolatedAddress("http://127.0.0.3:30100"); Assertions.assertEquals("http://127.0.0.3:30100", addressManager.address()); Assertions.assertEquals("http://127.0.0.3:30100", addressManager.address()); } diff --git a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterClient.java b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterClient.java index f5cec283e65..c9948615124 100755 --- a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterClient.java +++ b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterClient.java @@ -553,4 +553,16 @@ public boolean updateMicroserviceProperties(String serviceId, Map isolationAddresses = addressManager.getIsolationAddresses(); + if (isolationAddresses.isEmpty()) { + return; + } + for (String address : isolationAddresses) { + httpClient.checkServiceCenterAddressAvailable("/registry/microservices/" + serviceId + "/instances/" + instanceId + + "/heartbeat", null, null, address); + } + } } diff --git a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterOperation.java b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterOperation.java index 5521a732244..cd91b240b8a 100644 --- a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterOperation.java +++ b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterOperation.java @@ -193,4 +193,12 @@ boolean updateMicroserviceInstanceStatus(String serviceId, String instanceId, * @throws OperationException If some problems happened to contact service center or non http 200 returned. */ boolean updateMicroserviceProperties(String microserviceId, Map serviceProperties); + + /** + * Check serviceCenter isolation address available + * + * @param serviceId serviceId + * @param instanceId instanceId + */ + void checkIsolationAddressAvailable(String serviceId, String instanceId); } diff --git a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRawClient.java b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRawClient.java index 6c293004cc0..c58e5dc8bac 100755 --- a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRawClient.java +++ b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRawClient.java @@ -71,11 +71,7 @@ private HttpResponse doHttpRequest(String url, boolean absoluteUrl, Map(); - } - headers.put(HEADER_TENANT_NAME, tenantName); - HttpRequest httpRequest = new HttpRequest(formatUrl, headers, content, method); + HttpRequest httpRequest = buildHttpRequest(formatUrl, headers, content, method); try { HttpResponse httpResponse = httpTransport.doRequest(httpRequest); @@ -97,6 +93,26 @@ private HttpResponse doHttpRequest(String url, boolean absoluteUrl, Map headers, String content, + String address) { + String formatUrl = addressManager.formatUrl(url, false, address); + HttpRequest httpRequest = buildHttpRequest(formatUrl, headers, content, HttpRequest.PUT); + try { + httpTransport.doRequest(httpRequest); + addressManager.recoverIsolatedAddress(address); + } catch (IOException e) { + LOGGER.error("check service center isolation address {} available error!", address, e); + } + } + + private HttpRequest buildHttpRequest(String url, Map headers, String content, String method) { + if (headers == null) { + headers = new HashMap<>(); + } + headers.put(HEADER_TENANT_NAME, tenantName); + return new HttpRequest(url, headers, content, method); + } + public static class Builder { private String tenantName; diff --git a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRegistration.java b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRegistration.java index c9340e4d380..2a1296d3728 100644 --- a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRegistration.java +++ b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRegistration.java @@ -248,6 +248,7 @@ public void execute() { microserviceInstance.getInstanceId()); eventBus.post(new MicroserviceInstanceRegistrationEvent(true, microservice, microserviceInstance)); startTask(new SendHeartBeatTask(0)); + startTask(new CheckServiceCenterAddressTask()); } } catch (Exception e) { LOGGER.error("register microservice instance failed, and will try again.", e); @@ -292,4 +293,14 @@ public void execute() { } } } + + class CheckServiceCenterAddressTask implements Task { + @Override + public void execute() { + serviceCenterClient.checkIsolationAddressAvailable(microservice.getServiceId(), + microserviceInstance.getInstanceId()); + startTask(new BackOffSleepTask(Math.max(heartBeatInterval, heartBeatRequestTimeout), + new CheckServiceCenterAddressTask())); + } + } } From 0a2376735487a05156ab9cd8bb09a5c2ba35e476 Mon Sep 17 00:00:00 2001 From: chengyouling Date: Thu, 26 Sep 2024 20:31:18 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E5=88=A0=E9=99=A4telnet=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ServiceRegistryAddressManager.java | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/service-registry/registry-service-center/src/main/java/org/apache/servicecomb/serviceregistry/refresh/ServiceRegistryAddressManager.java b/service-registry/registry-service-center/src/main/java/org/apache/servicecomb/serviceregistry/refresh/ServiceRegistryAddressManager.java index 3ced076229c..875b8b643a0 100644 --- a/service-registry/registry-service-center/src/main/java/org/apache/servicecomb/serviceregistry/refresh/ServiceRegistryAddressManager.java +++ b/service-registry/registry-service-center/src/main/java/org/apache/servicecomb/serviceregistry/refresh/ServiceRegistryAddressManager.java @@ -17,9 +17,6 @@ package org.apache.servicecomb.serviceregistry.refresh; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.Socket; import java.net.URI; import java.util.List; @@ -27,15 +24,11 @@ import org.apache.servicecomb.foundation.common.net.URIEndpointObject; import org.apache.servicecomb.http.client.common.AbstractAddressManager; import org.apache.servicecomb.http.client.event.RefreshEndpointEvent; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.google.common.eventbus.EventBus; import com.google.common.eventbus.Subscribe; public class ServiceRegistryAddressManager extends AbstractAddressManager { - private static final Logger LOGGER = LoggerFactory.getLogger(ServiceRegistryAddressManager.class); - private static final String URI_PREFIX = "rest://"; public ServiceRegistryAddressManager(List addresses, EventBus eventBus) { @@ -52,18 +45,6 @@ protected String normalizeUri(String endpoint) { return new URIEndpointObject(endpoint).toString(); } - @Override - protected boolean telnetTest(String address) { - IpPort ipPort = transformIpPort(address); - try (Socket s = new Socket()) { - s.connect(new InetSocketAddress(ipPort.getHostOrIp(), ipPort.getPort()), 3000); - return true; - } catch (IOException e) { - LOGGER.warn("ping endpoint {} failed, It will be quarantined again.", address); - } - return false; - } - private IpPort transformIpPort(String address) { URI uri = URI.create(URI_PREFIX + address); return new IpPort(uri.getHost(), uri.getPort());