diff --git a/pom.xml b/pom.xml index 663398f01..957568d51 100644 --- a/pom.xml +++ b/pom.xml @@ -63,6 +63,7 @@ org.hpccsystems.commons.annotations.BaseTests 1.0.0 false + 2.4.0-alpha @@ -99,7 +100,54 @@ + + + + io.opentelemetry + opentelemetry-bom + 1.38.0 + pom + import + + + + + io.opentelemetry + opentelemetry-api + + + io.opentelemetry + opentelemetry-sdk + + + io.opentelemetry + opentelemetry-exporter-logging + + + io.opentelemetry + opentelemetry-sdk-extension-autoconfigure + + + io.opentelemetry + opentelemetry-sdk-extension-autoconfigure-spi + + + + io.opentelemetry.semconv + opentelemetry-semconv + 1.30.1-alpha + + + io.opentelemetry + opentelemetry-exporter-otlp + + + + io.opentelemetry.semconv + opentelemetry-semconv + 1.25.0-alpha + junit junit diff --git a/wsclient/src/main/java/org/hpccsystems/ws/client/BaseHPCCWsClient.java b/wsclient/src/main/java/org/hpccsystems/ws/client/BaseHPCCWsClient.java index 2a856a5a2..835052083 100644 --- a/wsclient/src/main/java/org/hpccsystems/ws/client/BaseHPCCWsClient.java +++ b/wsclient/src/main/java/org/hpccsystems/ws/client/BaseHPCCWsClient.java @@ -4,6 +4,9 @@ import java.net.MalformedURLException; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.text.Format; +import java.util.HashMap; +import java.util.Map; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.ParserConfigurationException; @@ -40,6 +43,30 @@ import org.w3c.dom.Node; import org.w3c.dom.NodeList; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanBuilder; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import io.opentelemetry.context.propagation.TextMapPropagator; +import io.opentelemetry.context.propagation.TextMapSetter; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import io.opentelemetry.semconv.HttpAttributes; +import io.opentelemetry.semconv.ResourceAttributes; +import io.opentelemetry.semconv.ServerAttributes; + + /** * Defines functionality common to all HPCC Systmes web service clients. * @@ -47,6 +74,7 @@ */ public abstract class BaseHPCCWsClient extends DataSingleton { + private static OpenTelemetry openTelemetry = null; // = AutoConfiguredOpenTelemetrySdk.initialize().getOpenTelemetrySdk(); /** Constant log */ protected static final Logger log = LogManager.getLogger(BaseHPCCWsClient.class); /** Constant DEAFULTECLWATCHPORT="8010" */ @@ -164,6 +192,46 @@ private String getTargetHPCCBuildVersionString() throws Exception } + public SpanBuilder getSpanBuilder(String spanName) + { + Tracer tracer = GlobalOpenTelemetry.getTracer(""); + SpanBuilder spanBuilder = tracer.spanBuilder(spanName) + .setAttribute(ServerAttributes.SERVER_ADDRESS, wsconn.getHost()) + //.setAttribute(ServerAttributes.SERVER_PORT, wsconn.getPort()) + .setAttribute(HttpAttributes.HTTP_REQUEST_METHOD, HttpAttributes.HttpRequestMethodValues.GET) + .setSpanKind(SpanKind.CLIENT); + //.startSpan(); + + return spanBuilder; + } + + static public void injectTraceParentHeader(Options options) + { + if (options != null) + { + Span currentSpan = Span.current(); + if (currentSpan != null && currentSpan.getSpanContext().isValid()) + W3CTraceContextPropagator.getInstance().inject(Context.current(), options, Options::setProperty); + } + } + + static public String getTraceParentHeader() + { + String traceparent = null; + Span currentSpan = Span.current(); + if (currentSpan != null && currentSpan.getSpanContext().isValid()) + { + Map carrier = new HashMap<>(); + TextMapSetter> setter = Map::put; + W3CTraceContextPropagator.getInstance().inject(Context.current(), carrier, setter); + + traceparent = carrier.getOrDefault("traceparent", "00-" + currentSpan.getSpanContext().getTraceId() + "-" + currentSpan.getSpanContext().getSpanId() + "-00"); + carrier.clear(); + } + + return traceparent; + } + /** * All instances of HPCCWsXYZClient should utilize this init function * Attempts to establish the target HPCC build version and its container mode @@ -175,6 +243,32 @@ private String getTargetHPCCBuildVersionString() throws Exception */ protected boolean initBaseWsClient(Connection connection, boolean fetchVersionAndContainerMode) { + //we should do this higher in the stack if possible and only if enabled + if (openTelemetry == null) + { + //Create an instance of the OpenTelemetry interface using the OpenTelemetrySdk. + try + { + openTelemetry = GlobalOpenTelemetry.get(); + if (openTelemetry == null) + { + openTelemetry = AutoConfiguredOpenTelemetrySdk.initialize().getOpenTelemetrySdk(); + /*if we want to overwrite auto configuration: + openTelemetry = AutoConfiguredOpenTelemetrySdk.builder() + .addTracerProviderCustomizer( + (sdkTracerProviderBuilder, configProperties) -> + sdkTracerProviderBuilder.addSpanProcessor( + new SpanProcessor() { // implementation omitted for brevity + })).build(); + */ + } + } + catch (Exception e) //ConfigurationException + { + System.out.println("OTEL autoconfiguration failure, please revise OTEL dependencies: " + e.getLocalizedMessage()); + } + } + boolean success = true; initErrMessage = ""; setActiveConnectionInfo(connection); @@ -582,6 +676,10 @@ static public Stub setStubOptions(Stub thestub, Connection connection) throws Ax opt.setProperty(HTTPConstants.CHUNKED, Boolean.FALSE); + //only do this if tracing enabled? + injectTraceParentHeader(opt); + //opt.setProperty("traceparent", getTraceParentHeader()); + if (connection.getPreemptiveHTTPAuthenticate()) { //Axis2 now forces connection authenticate, even if target is not secure diff --git a/wsclient/src/main/java/org/hpccsystems/ws/client/HPCCWsWorkUnitsClient.java b/wsclient/src/main/java/org/hpccsystems/ws/client/HPCCWsWorkUnitsClient.java index 3c7b74424..ce7ce02ee 100644 --- a/wsclient/src/main/java/org/hpccsystems/ws/client/HPCCWsWorkUnitsClient.java +++ b/wsclient/src/main/java/org/hpccsystems/ws/client/HPCCWsWorkUnitsClient.java @@ -100,6 +100,10 @@ import org.hpccsystems.ws.client.wrappers.wsworkunits.WorkunitWrapper; import org.hpccsystems.ws.client.wrappers.wsworkunits.WsWorkunitsClientStubWrapper; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanBuilder; +import io.opentelemetry.api.trace.StatusCode; + /** * Facilitates ECL WorkUnit related actions. * @@ -295,6 +299,8 @@ public void fastWURefresh(WorkunitWrapper wu) throws Exception, ArrayOfEspExcept { verifyStub(); // Throws exception if stub failed + SpanBuilder spanBuilder = getSpanBuilder("FastWURefresh"); + WUQuery request = new WUQuery(); WUState previousState = getStateID(wu); @@ -304,17 +310,20 @@ public void fastWURefresh(WorkunitWrapper wu) throws Exception, ArrayOfEspExcept WUQueryResponse response = null; + Span span = spanBuilder.startSpan(); try { response = ((WsWorkunits) stub).wUQuery(request); } catch (RemoteException e) { + span.setStatus(StatusCode.ERROR, e.getLocalizedMessage()); throw new Exception("WsWorkunits.fastWURefresh(...) encountered RemoteException.", e); } catch (EspSoapFault e) { - handleEspSoapFaults(new EspSoapFaultWrapper(e), "Could Not perform fastWURefresh"); + span.setStatus(StatusCode.ERROR, e.getLocalizedMessage()); + handleEspSoapFaults(new EspSoapFaultWrapper(e), "Could Not perform fastWURefresh",); } if (response != null) @@ -322,11 +331,13 @@ public void fastWURefresh(WorkunitWrapper wu) throws Exception, ArrayOfEspExcept if (response.getExceptions() != null) handleEspExceptions(new ArrayOfEspExceptionWrapper(response.getExceptions()), "Could Not perform fastWURefresh"); + span.setStatus(StatusCode.OK); if (response.getWorkunits() != null) { ECLWorkunit[] eclWorkunit = response.getWorkunits().getECLWorkunit(); - if (eclWorkunit != null && eclWorkunit.length == 1) wu.update(eclWorkunit[0]); + if (eclWorkunit != null && eclWorkunit.length == 1) + wu.update(eclWorkunit[0]); } if (previousState != getStateID(wu)) diff --git a/wsclient/src/test/java/org/hpccsystems/ws/client/BaseRemoteTest.java b/wsclient/src/test/java/org/hpccsystems/ws/client/BaseRemoteTest.java index 402e2782d..79454de31 100644 --- a/wsclient/src/test/java/org/hpccsystems/ws/client/BaseRemoteTest.java +++ b/wsclient/src/test/java/org/hpccsystems/ws/client/BaseRemoteTest.java @@ -43,6 +43,27 @@ HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®. import org.junit.Assert; import org.junit.experimental.categories.Category; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.exporter.logging.LoggingSpanExporter; +import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.OpenTelemetrySdkBuilder; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import io.opentelemetry.semconv.ResourceAttributes; +import io.opentelemetry.semconv.SemanticAttributes; + import java.net.URL; import java.nio.file.Paths; @@ -86,133 +107,183 @@ public abstract class BaseRemoteTest static { - // This allows testing against locally created self signed certs to work. - // In production certs will need to be created valid hostnames - javax.net.ssl.HttpsURLConnection.setDefaultHostnameVerifier( - new javax.net.ssl.HostnameVerifier() - { - public boolean verify(String hostname,javax.net.ssl.SSLSession sslSession) + String instrumentedServiceName = System.getenv("OTEL_SERVICE_NAME"); + if (instrumentedServiceName == null || instrumentedServiceName.isEmpty()) + instrumentedServiceName = "HPCC4J-JUNIT-TESTS"; + + // set service name on all OTel signals + Resource resource = Resource.getDefault().merge(Resource.create( + Attributes.of(io.opentelemetry.semconv.ResourceAttributes.SERVICE_NAME, instrumentedServiceName, + ResourceAttributes.SERVICE_VERSION,"1.0", + ResourceAttributes.DEPLOYMENT_ENVIRONMENT,"test"))); + + // init OTel trace provider https://opentelemetry.io/docs/languages/java/exporters/ + SdkTracerProvider sdkTracerProvider = SdkTracerProvider.builder() + .setResource(resource) + .setSampler(Sampler.alwaysOn()) + .addSpanProcessor( + //BatchSpanProcessor.builder( + SimpleSpanProcessor.create(LoggingSpanExporter.create()) + //OtlpGrpcSpanExporter.builder() + //.setEndpoint(System.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")) + //.addHeader("Authorization", "Bearer " + System.getenv("ELASTIC_APM_SECRET_TOKEN")).build()).build()).build(); + ).build(); + + // init OTel meter provider with export to OTLP + //SdkMeterProvider sdkMeterProvider = SdkMeterProvider.builder().setResource(resource).registerMetricReader(PeriodicMetricReader.builder(OtlpGrpcMetricExporter.builder().setEndpoint(System.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")).addHeader("Authorization", "Bearer " + System.getenv("ELASTIC_APM_SECRET_TOKEN")).build()).build()).build(); + + // create sdk object and set it as global + OpenTelemetrySdk sdk = OpenTelemetrySdk.builder().setTracerProvider(sdkTracerProvider) + //.setLoggerProvider(sdkLoggerProvider) + //.setMeterProvider(sdkMeterProvider) + .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance())).buildAndRegisterGlobal(); + +// GlobalOpenTelemetry.set(sdk); + Runtime.getRuntime().addShutdownHook(new Thread(sdk::close)); + + Tracer tracer = GlobalOpenTelemetry.getTracer(instrumentedServiceName); + Span span = tracer.spanBuilder("my span").startSpan(); + // put the span into the current Context + try (Scope scope = span.makeCurrent()) + { + + // This allows testing against locally created self signed certs to work. + // In production certs will need to be created valid hostnames + javax.net.ssl.HttpsURLConnection.setDefaultHostnameVerifier( + new javax.net.ssl.HostnameVerifier() { - if (hostname.equals("localhost")) + public boolean verify(String hostname,javax.net.ssl.SSLSession sslSession) { - return true; - } - - return false; - } - }); - - String legacythorcluster = System.getProperty("thorcluster"); - if (legacythorcluster != null && !legacythorcluster.isEmpty()) - System.out.println("WARNING! 'thorcluster' has been deprecated - Use 'thorclustername' and/or 'thorgroupname' instead"); + if (hostname.equals("localhost")) + { + return true; + } - if (System.getProperty("thorclustername") == null) - System.out.println("thorclustername not provided - defaulting to '" + thorclustername + "'"); - - if (System.getProperty("roxieclustername") == null) - System.out.println("roxieclustername not provided - defaulting to '" + roxieclustername + "'"); - - InetAddress ip; - String hostname; - try - { - ip = InetAddress.getLocalHost(); - hostname = ip.getHostName(); - System.out.println("RemoteTest executing on: " + hostname + "(" + ip + ")"); - } - catch (UnknownHostException e) - { - e.printStackTrace(); - } + return false; + } + }); - if (System.getProperty("hpccconn") == null) - System.out.println("RemoteTest: No 'hpccconn' provided, defaulting to http://localhost:8010"); - else - System.out.println("RemoteTest: 'hpccconn' set to: '" + connString + "'"); + String legacythorcluster = System.getProperty("thorcluster"); + if (legacythorcluster != null && !legacythorcluster.isEmpty()) + System.out.println("WARNING! 'thorcluster' has been deprecated - Use 'thorclustername' and/or 'thorgroupname' instead"); - if (System.getProperty("hpccuser") == null) - System.out.println("RemoteTest: No 'hpccuser' provided, defaulting to '" + defaultUserName + "'"); - else - System.out.println("RemoteTest: 'hpccuser' set to: '" + hpccUser + "'"); + if (System.getProperty("thorclustername") == null) + System.out.println("thorclustername not provided - defaulting to '" + thorclustername + "'"); - if (System.getProperty("hpccpass") == null) - System.out.println("RemoteTest: No 'hpccpass' provided."); + if (System.getProperty("roxieclustername") == null) + System.out.println("roxieclustername not provided - defaulting to '" + roxieclustername + "'"); - if (platform == null) - { + InetAddress ip; + String hostname; try { - connection = new Connection(connString); + ip = InetAddress.getLocalHost(); + hostname = ip.getHostName(); + System.out.println("RemoteTest executing on: " + hostname + "(" + ip + ")"); } - catch (MalformedURLException e) + catch (UnknownHostException e) { - fail("Could not acquire connection object based on: '" + connString + "' - " + e.getLocalizedMessage()); + e.printStackTrace(); } - Assert.assertNotNull("Could not acquire connection object", connection); - connection.setCredentials(hpccUser, hpccPass); + if (System.getProperty("hpccconn") == null) + System.out.println("RemoteTest: No 'hpccconn' provided, defaulting to http://localhost:8010"); + else + System.out.println("RemoteTest: 'hpccconn' set to: '" + connString + "'"); - if (connTO != null) - connection.setConnectTimeoutMilli(connTO); + if (System.getProperty("hpccuser") == null) + System.out.println("RemoteTest: No 'hpccuser' provided, defaulting to '" + defaultUserName + "'"); + else + System.out.println("RemoteTest: 'hpccuser' set to: '" + hpccUser + "'"); - if (sockTO != null) - connection.setSocketTimeoutMilli(Integer.valueOf(sockTO)); + if (System.getProperty("hpccpass") == null) + System.out.println("RemoteTest: No 'hpccpass' provided."); - platform = Platform.get(connection); + if (platform == null) + { + try + { + connection = new Connection(connString); + } + catch (MalformedURLException e) + { + fail("Could not acquire connection object based on: '" + connString + "' - " + e.getLocalizedMessage()); + } - Assert.assertNotNull("Could not acquire platform object", platform); - } + Assert.assertNotNull("Could not acquire connection object", connection); + connection.setCredentials(hpccUser, hpccPass); - try - { - wsclient = platform.checkOutHPCCWsClient(); - if (thorClusterFileGroup == null || thorClusterFileGroup.isEmpty()) + if (connTO != null) + connection.setConnectTimeoutMilli(connTO); + + if (sockTO != null) + connection.setSocketTimeoutMilli(Integer.valueOf(sockTO)); + + platform = Platform.get(connection); + + Assert.assertNotNull("Could not acquire platform object", platform); + } + + try { - List grouplist = wsclient.getTopologyGroups(wsclient.isContainerized() ? TopologyGroupQueryKind.PLANE : TopologyGroupQueryKind.THOR); - for (TpGroupWrapper tpGroupWrapper : grouplist) + wsclient = platform.checkOutHPCCWsClient(); + if (thorClusterFileGroup == null || thorClusterFileGroup.isEmpty()) + { + List grouplist = wsclient.getTopologyGroups(wsclient.isContainerized() ? TopologyGroupQueryKind.PLANE : TopologyGroupQueryKind.THOR); + for (TpGroupWrapper tpGroupWrapper : grouplist) + { + thorClusterFileGroup = tpGroupWrapper.getName(); + if (thorClusterFileGroup != null) + break; + } + System.out.println("RemoteTest: No 'thorClusterFileGroup' provided, using '" + thorClusterFileGroup + "'"); + } + else + { + System.out.println("RemoteTest: 'thorClusterFileGroup': '" + thorClusterFileGroup + "'"); + } + + if (roxieClusterGroup == null || roxieClusterGroup.isEmpty()) + { + List grouplist = wsclient.getTopologyGroups(wsclient.isContainerized() ? TopologyGroupQueryKind.PLANE : TopologyGroupQueryKind.ROXIE); + for (TpGroupWrapper tpGroupWrapper : grouplist) + { + roxieClusterGroup = tpGroupWrapper.getName(); + if (roxieClusterGroup != null) + break; + } + System.out.println("RemoteTest: No 'roxiegroupname' provided, using '" + roxieClusterGroup + "'"); + } + else { - thorClusterFileGroup = tpGroupWrapper.getName(); - if (thorClusterFileGroup != null) - break; + System.out.println("RemoteTest: 'roxiegroupname': '" + roxieclustername + "'"); } - System.out.println("RemoteTest: No 'thorClusterFileGroup' provided, using '" + thorClusterFileGroup + "'"); } - else + catch (Exception e) { - System.out.println("RemoteTest: 'thorClusterFileGroup': '" + thorClusterFileGroup + "'"); + fail("Could not acquire wsclient object: " + e.getMessage() ); } - if (roxieClusterGroup == null || roxieClusterGroup.isEmpty()) + Assert.assertNotNull("Could not acquire wsclient object", wsclient); + + // Run the generate-datasets.ecl script if present in the project resources + try { - List grouplist = wsclient.getTopologyGroups(wsclient.isContainerized() ? TopologyGroupQueryKind.PLANE : TopologyGroupQueryKind.ROXIE); - for (TpGroupWrapper tpGroupWrapper : grouplist) - { - roxieClusterGroup = tpGroupWrapper.getName(); - if (roxieClusterGroup != null) - break; - } - System.out.println("RemoteTest: No 'roxiegroupname' provided, using '" + roxieClusterGroup + "'"); + executeECLScript("generate-datasets.ecl"); } - else + catch (Exception e) { - System.out.println("RemoteTest: 'roxiegroupname': '" + roxieclustername + "'"); + Assert.fail("Error executing test data generation scripts with error: " + e.getMessage()); } - } - catch (Exception e) - { - fail("Could not acquire wsclient object: " + e.getMessage() ); - } - - Assert.assertNotNull("Could not acquire wsclient object", wsclient); - // Run the generate-datasets.ecl script if present in the project resources - try + } + catch (Throwable t) { - executeECLScript("generate-datasets.ecl"); + span.setStatus(StatusCode.ERROR, "Change it to your error message"); } - catch (Exception e) + finally { - Assert.fail("Error executing test data generation scripts with error: " + e.getMessage()); + span.end(); // closing the scope does not end the span, this has to be done manually } }