Allow listeners to be disabled at runtime

Closes #2381

CHANGES.txt

@@ -1,4 +1,5 @@
 Current (7.10.0)
+Fixed: GITHUB-2381: Controlling the inclusion of the listener at runtime (Krishnan Mahadevan)
 Fixed: GITHUB-3082: IInvokedMethodListener Iinvoked method does not return correct instance during @BeforeMethod, @AfterMethod and @AfterClass (Krishnan Mahadevan)
 Fixed: GITHUB-3084: Document project's PGP artifact signing keys (Krishnan Mahadevan)
 Fixed: GITHUB-3079: Associate a unique id with every test class object instantiated by TestNG (Krishnan Mahadevan)

testng-core-api/src/main/java/org/testng/ITestNGListener.java

@@ -5,4 +5,10 @@
  *
  * @author cbeust
  */
-public interface ITestNGListener {}
+public interface ITestNGListener {
+
+  /** @return - <code>true</code> if the current listener can be considered for execution. */
+  default boolean isEnabled() {
+    return true;
+  }
+}

testng-core/src/main/java/org/testng/TestNG.java

@@ -731,6 +731,9 @@ public void addListener(ITestNGListener listener) {
     if (listener == null) {
       return;
     }
+    if (!listener.isEnabled()) {
+      return;
+    }
     if (listener instanceof IExecutionVisualiser) {
       IExecutionVisualiser visualiser = (IExecutionVisualiser) listener;
       maybeAddListener(m_executionVisualisers, visualiser);

testng-core/src/main/java/org/testng/internal/annotations/JDK15AnnotationFinder.java

@@ -159,6 +159,9 @@ private void transform(
       Constructor<?> testConstructor,
       Method testMethod,
       Class<?> whichClass) {
+    if (!m_transformer.isEnabled()) {
+      return;
+    }
     //
     // Transform @Test
     //

testng-core/src/test/java/test/listeners/ListenersTest.java

@@ -19,6 +19,9 @@
 import org.testng.internal.ExitCode;
 import org.testng.xml.XmlSuite;
 import test.SimpleBaseTest;
+import test.listeners.issue2381.FactoryTestClassSample;
+import test.listeners.issue2381.SampleGlobalListener;
+import test.listeners.issue2381.SampleTransformer;
 import test.listeners.issue2638.DummyInvokedMethodListener;
 import test.listeners.issue2638.TestClassASample;
 import test.listeners.issue2638.TestClassBSample;
@@ -524,6 +527,47 @@ public void testSkipStatusInBeforeAndAfterConfigurationMethod(
     assertThat(listener.getLogs()).containsExactlyElementsOf(expected);
   }
 
+  @Test(description = "GITHUB-2381")
+  public void ensureListenersCanBeDisabled() {
+    TestNG testng =
+        create(test.listeners.issue2381.TestClassSample.class, FactoryTestClassSample.class);
+    SampleGlobalListener listener = new SampleGlobalListener();
+    SampleTransformer transformer = new SampleTransformer();
+    testng.addListener(listener);
+    testng.addListener(transformer);
+    testng.run();
+    assertThat(SampleGlobalListener.getLogs()).isEmpty();
+    assertThat(SampleTransformer.getLogs()).isEmpty();
+  }
+
+  @Test(description = "GITHUB-2381")
+  public void ensureListenersCanBeDisabledViaCLI() {
+    SampleGlobalListener.clearLogs();
+    SampleTransformer.clearLogs();
+    String listeners =
+        String.join(",", SampleGlobalListener.class.getName(), SampleTransformer.class.getName());
+    String testClasses =
+        String.join(
+            ",",
+            test.listeners.issue2381.TestClassSample.class.getName(),
+            FactoryTestClassSample.class.getName());
+    List<String> args = List.of("-listener", listeners, "-testclass", testClasses);
+    TestNG.privateMain(args.toArray(String[]::new), null);
+    assertThat(SampleGlobalListener.getLogs()).isEmpty();
+    assertThat(SampleTransformer.getLogs()).isEmpty();
+  }
+
+  @Test(description = "GITHUB-2381")
+  public void ensureListenersCanBeDisabledViaSuiteFile() {
+    SampleGlobalListener.clearLogs();
+    SampleTransformer.clearLogs();
+    TestNG testng = new TestNG();
+    testng.setTestSuites(List.of("src/test/resources/2381.xml"));
+    testng.run();
+    assertThat(SampleGlobalListener.getLogs()).isEmpty();
+    assertThat(SampleTransformer.getLogs()).isEmpty();
+  }
+
   @Test(description = "GITHUB-3082")
   public void ensureListenerWorksWithCorrectTestClassInstance() {
     TestNG tng = create(test.listeners.issue3082.TestClassSample.class);

testng-core/src/test/java/test/listeners/issue2381/FactoryTestClassSample.java

@@ -0,0 +1,27 @@
+package test.listeners.issue2381;
+
+import org.testng.annotations.AfterSuite;
+import org.testng.annotations.BeforeSuite;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Factory;
+import org.testng.annotations.Test;
+
+public class FactoryTestClassSample {
+
+  @Factory(dataProvider = "dp")
+  public FactoryTestClassSample(int ignored) {}
+
+  @DataProvider
+  public static Object[][] dp() {
+    return new Object[][] {{1}};
+  }
+
+  @BeforeSuite
+  public void beforeSuite() {}
+
+  @Test
+  public void testMethod() {}
+
+  @AfterSuite
+  public void afterSuite() {}
+}

testng-core/src/test/java/test/listeners/issue2381/SampleGlobalListener.java

@@ -0,0 +1,141 @@
+package test.listeners.issue2381;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.testng.IAlterSuiteListener;
+import org.testng.IClassListener;
+import org.testng.IExecutionListener;
+import org.testng.IExecutionVisualiser;
+import org.testng.IInvokedMethod;
+import org.testng.IInvokedMethodListener;
+import org.testng.IReporter;
+import org.testng.ISuite;
+import org.testng.ISuiteListener;
+import org.testng.ITestClass;
+import org.testng.ITestContext;
+import org.testng.ITestListener;
+import org.testng.ITestResult;
+import org.testng.xml.XmlSuite;
+
+public class SampleGlobalListener
+    implements IExecutionListener,
+        IAlterSuiteListener,
+        ISuiteListener,
+        ITestListener,
+        IInvokedMethodListener,
+        IClassListener,
+        IExecutionVisualiser,
+        IReporter {
+
+  private static final List<String> logs = new ArrayList<>();
+
+  public static List<String> getLogs() {
+    return Collections.unmodifiableList(logs);
+  }
+
+  public static void clearLogs() {
+    logs.clear();
+  }
+
+  @Override
+  public void onExecutionStart() {
+    logs.add("onExecutionStart");
+  }
+
+  @Override
+  public void alter(List<XmlSuite> suites) {
+    logs.add("alter");
+  }
+
+  @Override
+  public void onStart(ISuite suite) {
+    logs.add("onStart(ISuite)");
+  }
+
+  @Override
+  public void onStart(ITestContext context) {
+    logs.add("onStart(ITestContext)");
+  }
+
+  @Override
+  public void onBeforeClass(ITestClass testClass) {
+    logs.add("onBeforeClass");
+  }
+
+  @Override
+  public void consumeDotDefinition(String dotDefinition) {
+    logs.add("consumeDotDefinition");
+  }
+
+  @Override
+  public void onTestStart(ITestResult result) {
+    logs.add("onTestStart(ITestResult)");
+  }
+
+  @Override
+  public void beforeInvocation(IInvokedMethod method, ITestResult testResult) {
+    logs.add("beforeInvocation");
+  }
+
+  @Override
+  public void afterInvocation(IInvokedMethod method, ITestResult testResult) {
+    logs.add("afterInvocation");
+  }
+
+  @Override
+  public void onTestSuccess(ITestResult result) {
+    logs.add("onTestSuccess(ITestResult)");
+  }
+
+  @Override
+  public void onTestFailure(ITestResult result) {
+    logs.add("onTestFailure(ITestResult)");
+  }
+
+  @Override
+  public void onTestSkipped(ITestResult result) {
+    logs.add("onTestSkipped(ITestResult)");
+  }
+
+  @Override
+  public void onTestFailedButWithinSuccessPercentage(ITestResult result) {
+    logs.add("onTestFailedButWithinSuccessPercentage(ITestResult)");
+  }
+
+  @Override
+  public void onTestFailedWithTimeout(ITestResult result) {
+    logs.add("onTestFailedWithTimeout(ITestResult)");
+  }
+
+  @Override
+  public void onAfterClass(ITestClass testClass) {
+    logs.add("onAfterClass");
+  }
+
+  @Override
+  public void onFinish(ITestContext context) {
+    logs.add("onFinish(ITestContext)");
+  }
+
+  @Override
+  public void onFinish(ISuite suite) {
+    logs.add("onFinish(ISuite)");
+  }
+
+  @Override
+  public void generateReport(
+      List<XmlSuite> xmlSuites, List<ISuite> suites, String outputDirectory) {
+    logs.add("generateReport");
+  }
+
+  @Override
+  public void onExecutionFinish() {
+    logs.add("onExecutionFinish");
+  }
+
+  @Override
+  public boolean isEnabled() {
+    return false;
+  }
+}

testng-core/src/test/java/test/listeners/issue2381/SampleTransformer.java

@@ -0,0 +1,61 @@
+package test.listeners.issue2381;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.testng.IAnnotationTransformer;
+import org.testng.annotations.IConfigurationAnnotation;
+import org.testng.annotations.IDataProviderAnnotation;
+import org.testng.annotations.IFactoryAnnotation;
+import org.testng.annotations.IListenersAnnotation;
+import org.testng.annotations.ITestAnnotation;
+
+public class SampleTransformer implements IAnnotationTransformer {
+
+  private static final List<String> logs = new ArrayList<>();
+
+  public static List<String> getLogs() {
+    return Collections.unmodifiableList(logs);
+  }
+
+  public static void clearLogs() {
+    logs.clear();
+  }
+
+  @Override
+  public boolean isEnabled() {
+    return false;
+  }
+
+  @Override
+  public void transform(IListenersAnnotation annotation, Class<?> testClass) {
+    logs.add("transform_listener");
+  }
+
+  @Override
+  public void transform(IDataProviderAnnotation annotation, Method method) {
+    logs.add("transform_data_provider");
+  }
+
+  @Override
+  public void transform(IFactoryAnnotation annotation, Method method) {
+    logs.add("transform_factory");
+  }
+
+  @Override
+  public void transform(
+      IConfigurationAnnotation annotation,
+      Class testClass,
+      Constructor testConstructor,
+      Method testMethod) {
+    logs.add("transform_configuration");
+  }
+
+  @Override
+  public void transform(
+      ITestAnnotation annotation, Class testClass, Constructor testConstructor, Method testMethod) {
+    logs.add("transform_test");
+  }
+}

testng-core/src/test/java/test/listeners/issue2381/TestClassSample.java

@@ -0,0 +1,18 @@
+package test.listeners.issue2381;
+
+import org.testng.IExecutionListener;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Listeners;
+import org.testng.annotations.Test;
+
+@Listeners(TestClassSample.MyListener.class)
+public class TestClassSample {
+
+  @BeforeClass
+  public void beforeClass() {}
+
+  @Test
+  public void testMethod() {}
+
+  public static class MyListener implements IExecutionListener {}
+}

testng-core/src/test/resources/2381.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE suite SYSTEM "https://testng.org/testng-1.0.dtd">
+<suite name="2381_suite"  verbose="2">
+  <listeners>
+    <listener class-name="test.listeners.issue2381.SampleTransformer"/>
+    <listener class-name="test.listeners.issue2381.SampleGlobalListener"/>
+  </listeners>
+
+  <test name="2381_test" verbose="2">
+    <classes>
+      <class name="test.listeners.issue2381.TestClassSample"/>
+      <class name="test.listeners.issue2381.FactoryTestClassSample"/>
+    </classes>
+  </test>
+
+</suite>
+