Release XLT 5.4.0

NOTICE.md

@@ -231,6 +231,12 @@ pegdown
   * Homepage: https://github.com/sirthias/pegdown
   * More information in folder: doc/3rd-party-licenses/pegdown
 
+progressbar
+
+  * License: MIT License
+  * Homepage: https://github.com/ctongfei/progressbar
+  * More information in folder: doc/3rd-party-licenses/progressbar
+
 selenium-java
 
   * License: Apache License 2.0

bin/ec2_admin.cmd

@@ -19,5 +19,11 @@ set JAVA_OPTIONS=%JAVA_OPTIONS% -Dlog4j.configuration="file:%XLT_CONFIG_DIR%\ec2
 rem set JAVA_OPTIONS=%JAVA_OPTIONS% -agentlib:jdwp=transport=dt_socket,address=localhost:6666,server=y,suspend=n
 set JAVA_OPTIONS=%JAVA_OPTIONS% -cp "%CLASSPATH%"
 
+:: append options to suppress illegal access warnings for Java 9+
+set PACKAGES=java.xml/com.sun.org.apache.xpath.internal
+for %%p in (%PACKAGES%) do set JAVA_OPTIONS=!JAVA_OPTIONS! --add-opens=%%p=ALL-UNNAMED
+set JAVA_OPTIONS=%JAVA_OPTIONS% -XX:+IgnoreUnrecognizedVMOptions
+rem set JAVA_OPTIONS=%JAVA_OPTIONS% --illegal-access=debug
+
 :: run Java
 java %JAVA_OPTIONS% com.xceptance.xlt.ec2.Main %*

bin/ec2_admin.sh

@@ -22,6 +22,12 @@ JAVA_OPTIONS="$JAVA_OPTIONS -Dlog4j.configuration=\"file:$XLT_CONFIG_DIR/ec2_adm
 #JAVA_OPTIONS="$JAVA_OPTIONS -agentlib:jdwp=transport=dt_socket,address=localhost:6666,server=y,suspend=n"
 JAVA_OPTIONS="$JAVA_OPTIONS -cp \"$CLASSPATH\""
 
+# append options to suppress illegal access warnings for Java 9+
+PACKAGES="java.xml/com.sun.org.apache.xpath.internal"
+for p in $PACKAGES; do JAVA_OPTIONS="$JAVA_OPTIONS --add-opens=$p=ALL-UNNAMED"; done
+JAVA_OPTIONS="$JAVA_OPTIONS -XX:+IgnoreUnrecognizedVMOptions" 
+#JAVA_OPTIONS="$JAVA_OPTIONS --illegal-access=debug"
+
 # run Java
 CMD="java $JAVA_OPTIONS com.xceptance.xlt.ec2.Main"
 ARGS=""

config/mastercontroller.properties

@@ -63,6 +63,16 @@ com.xceptance.xlt.mastercontroller.password = xceptance
 #com.xceptance.xlt.mastercontroller.maxParallelUploads = -1
 #com.xceptance.xlt.mastercontroller.maxParallelDownloads = -1
 
+# ==================
+#  Result Storage
+# ==================
+
+## Do we want to keep the timer files as .gz files after downloading? This
+## will save a lot of disk space and also improve report creation speed when
+## the disk is slow, such as on cloud machines.
+## - true (default): timer files are stored as GZIPed files
+## - false: timer files are expanded, behavior of XLT prior to version 5.4.0
+#com.xceptance.xlt.mastercontroller.compressedTimerFiles = true
 
 
 ###############################################################################

doc/3rd-party-licenses/progressbar/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015--2020 Tongfei Chen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

doc/3rd-party-licenses/progressbar/NOTICE

@@ -0,0 +1,6 @@
+Terminal-based progress bar for Java / JVM 
+http://tongfei.me/progressbar/
+https://github.com/ctongfei/progressbar/
+
+Copyright (c) 2015--2020 Tongfei Chen
+MIT License

doc/internal-doc/licenses/licenses.csv

@@ -33,6 +33,7 @@ junit,Eclipse Public License 1.0,https://junit.org/junit4/,
 log4j,Apache License 2.0,http://logging.apache.org/log4j/1.2/,
 neko-htmlunit,Apache License 2.0,https://github.com/HtmlUnit/htmlunit-neko,
 pegdown,Apache License 2.0,https://github.com/sirthias/pegdown,
+progressbar,MIT License,https://github.com/ctongfei/progressbar,
 selenium-java,Apache License 2.0,https://selenium.dev/,
 slf4j-log4j12,MIT License,http://www.slf4j.org/,
 trove,GNU Lesser General Public License 2.1,https://bitbucket.org/trove4j/trove,

doc/xltdoc/release-notes/5.4.x.textile

@@ -0,0 +1,48 @@
+---
+layout: manual
+title: 5.4.x
+position: 980
+sorted: true
+---
+
+h2. XLT 5.4.0
+
+h3. Load Testing
+
+h4. Progress indication when generating reports
+
+The report generation hasn't indicated the progress in detail which made it hard to judge the remaining runtime. This release adds simple progress bars which will indicate a progress as well as an estimated remaining runtime. Big thanks to the "ProgressBar project":https://github.com/ctongfei/progressbar/!
+
+h4. Timer files stored in compressed form
+
+XLT has been keeping all data in clear text files for easy post processing and open data handling. However, the @timers.csv@ files occupy a large share of the required space on disk after a test run. While the download already runs compressed, the final on-disk storage was plain. This release changes this behavior and stores the @timers.csv@ in a compressed GZIP format by default. The report generator learned to deal with compressed as well as uncompressed formats automatically.
+
+For example, a result set with previously 9.5 GB of storage space now only requires 1.1 GB. As an additional advantage, report generation might run faster on systems with slower disks such as regular cloud machines. Furthermore, the download of results from the agents will also run faster due to less write operations.
+
+If you find it necessary to revert to the old behavior of plain uncompressed storage, you can set the property @com.xceptance.xlt.mastercontroller.compressedTimerFiles@ in file @mastercontroller.properties@ to false.
+
+h4. Illegal access warning in ec2_admin
+
+The @ec2_admin@ raised an illegal reflective access warning when it's reporting problems such as trying to delete something that does not exist:
+
+bc(plain). 
+Terminating the selected instances in region 'eu-central-1' ... WARNING: An illegal reflective access operation has occurred
+WARNING: Illegal reflective access by com.amazonaws.util.XpathUtils (file:/home/anyone/projects/loadtest/xlt-5.3.0/lib/aws-java-sdk-core-1.11.762.jar) to method com.sun.org.apache.xpath.internal.XPathContext.getDTMManager()
+WARNING: Please consider reporting this to the maintainers of com.amazonaws.util.XpathUtils
+WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
+WARNING: All illegal access operations will be denied in a future release
+Failed: No instances specified (Service: AmazonEC2; Status Code: 400; Error Code: InvalidParameterCombination; Request ID: 23234-3bd3-4157-b365-4bf7b4432221ec7)
+
+While we cannot fix the illegal access (it is made in a foreign library), we can at least suppress these warnings by passing the appropriate command line options to java in @ec2_admin.sh@.
+
+
+h3. Test Framework
+
+h4. Socket instrumentation won't work with Java 16
+
+When running a test scenario with Java 16, the socket instrumentation code to gather network timing data (connect time, time to first bytes, etc.) could not be installed and the test scenario failed with this error message:
+
+bc(plain). 
+java.lang.NoClassDefFoundError: Could not initialize class com.xceptance.xlt.engine.socket.InstrumentedSocketImpl
+
+This issue is fixed now.

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.xceptance</groupId>
     <artifactId>xlt</artifactId>
-    <version>5.3.0</version>
+    <version>5.4.0</version>
     <packaging>jar</packaging>
 
     <name>XLT</name>
@@ -392,6 +392,16 @@
             <version>2.3.2</version>
         </dependency>
 
+        <!--  
+            https://github.com/ctongfei/progressbar 
+            MIT license
+        -->
+        <dependency>
+            <groupId>me.tongfei</groupId>
+            <artifactId>progressbar</artifactId>
+            <version>0.9.1</version>
+        </dependency>
+
         <!-- ================= -->
         <!-- Test dependencies -->
         <!-- ================= -->

src/main/java/com/xceptance/common/util/zip/ZipUtils.java

@@ -22,6 +22,8 @@
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.OutputStream;
+import java.util.zip.GZIPOutputStream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 import java.util.zip.ZipOutputStream;
@@ -32,6 +34,7 @@
 import org.apache.commons.logging.LogFactory;
 
 import com.xceptance.common.util.ParameterCheckUtils;
+import com.xceptance.xlt.common.XltConstants;
 
 /**
  * The ZipUtils class provides convenience methods for creating and unpacking ZIP archives.
@@ -296,6 +299,24 @@ public static void addRegularFile(final ZipOutputStream out, final File file, fi
      *             if an I/O error occurs
      */
     public static void unzipFile(final File zipFile, final File directory) throws IOException
+    {
+        unzipFile(zipFile, directory, false);
+    }
+
+    /**
+     * Unzips the given ZIP file to the specified directory. If the directory does not exist yet, it will be created.
+     * Depending on the parameter, timers files are stored to disk either in plain or in compressed form (gzipped).
+     * 
+     * @param zipFile
+     *            the zip file
+     * @param directory
+     *            the target directory
+     * @param compressedTimerFiles
+     *            do we want to keep the timers in a compressed form
+     * @throws java.io.IOException
+     *             if an I/O error occurs
+     */
+    public static void unzipFile(final File zipFile, final File directory, final boolean compressedTimerFiles) throws IOException
     {
         ParameterCheckUtils.isReadableFile(zipFile, "zipFile");
         ParameterCheckUtils.isNotNull(directory, "directory");
@@ -320,11 +341,34 @@ public static void unzipFile(final File zipFile, final File directory) throws IO
                 }
                 else
                 {
-                    // cannot use this as it DOES close the input stream
-                    // FileUtils.copyToFile(in, file);
+                    // do we want to store the timers compressed
+                    File compressedFile = null;
+                    boolean compressIt = false;
 
-                    try (final FileOutputStream out = new FileOutputStream(file))
+                    // shall we compress timers?
+                    if (compressedTimerFiles)
                     {
+                        // we need the name of the file, without any path element
+                        final String fileName = file.getName();
+
+                        boolean b1 = XltConstants.TIMER_FILENAME_PATTERNS.stream().anyMatch(p -> p.asPredicate().test(fileName));
+                        boolean b2 = XltConstants.CPT_TIMER_FILENAME_PATTERNS.stream().anyMatch(p -> p.asPredicate().test(fileName));
+
+                        // one pattern matched
+                        if (b1 || b2)
+                        {
+                            // determine the new name
+                            compressedFile = new File(directory, entry.getName() + ".gz");
+                            compressIt = true; // indicate the need for compression
+                        }
+                    }
+
+                    try (final OutputStream out = compressIt ? new GZIPOutputStream(new FileOutputStream(compressedFile))
+                                                             : new FileOutputStream(file))
+                    {
+                        // cannot use this as it DOES close the input stream
+                        // FileUtils.copyToFile(in, file);
+
                         IOUtils.copy(in, out);
                     }
                 }

src/main/java/com/xceptance/xlt/common/XltConstants.java

@@ -15,6 +15,11 @@
  */
 package com.xceptance.xlt.common;
 
+import java.util.List;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
 /**
  * Collection of global constants for directories, default values and so on. Some of the values will be overwritten with
  * custom values later on. Named XltConstants, because there are too many other Constants classes in other packages.
@@ -133,6 +138,28 @@ private XltConstants()
      * The name of the timer files.
      */
     public static final String TIMER_FILENAME = "timers.csv";
+
+    /**
+     * The possible name of the timer files.
+     */
+    public static final List<Pattern> TIMER_FILENAME_PATTERNS = 
+        Stream.of(
+                  "^timers\\.csv$", 
+                  "^timers\\.csv\\.gz$", 
+                  "^timers\\.csv\\.[0-9]{4}-[0-9]{2}-[0-9]{2}$", 
+                  "^timers\\.csv\\.[0-9]{4}-[0-9]{2}-[0-9]{2}\\.gz$")
+        .map(Pattern::compile).collect(Collectors.toList());
+
+    /**
+     * The possible name of the CPT timer files.
+     * <p>
+     * Note: Needed for backward compatibility. Separate CPT timers files have been removed in XLT 4.8.
+     */
+    public static final List<Pattern> CPT_TIMER_FILENAME_PATTERNS = 
+        Stream.of(
+                  "^timer-wd-.+\\.csv$", 
+                  "^timer-wd-.+\\.csv\\.gz$")
+        .map(Pattern::compile).collect(Collectors.toList());
 
     /**
      * The option name of the <em>from</em> option on the command line.

src/main/java/com/xceptance/xlt/engine/socket/InstrumentedSocketImpl.java

@@ -30,7 +30,6 @@
 import org.apache.commons.lang3.JavaVersion;
 import org.apache.commons.lang3.SystemUtils;
 
-import com.xceptance.common.lang.ReflectionUtils;
 import com.xceptance.xlt.engine.RequestExecutionContext;
 
 /**
@@ -43,6 +42,10 @@
  */
 class InstrumentedSocketImpl extends SocketImpl
 {
+    private static final Constructor<?> CONSTRUCTOR;
+
+    private static final Method FACTORY_METHOD;
+
     private static final Method ACCEPT_METHOD;
 
     private static final Method AVAILABLE_METHOD;
@@ -57,8 +60,6 @@ class InstrumentedSocketImpl extends SocketImpl
 
     private static final Method CONNECT_SOCKETADDRESS_METHOD;
 
-    private static final Constructor<?> CONSTRUCTOR;
-
     private static final Method CREATE_METHOD;
 
     private static final Method GETFILEDESCRIPTOR_METHOD;
@@ -89,22 +90,22 @@ class InstrumentedSocketImpl extends SocketImpl
     {
         try
         {
-            // get the constructor of the chosen SocketImpl subclass and make it accessible
+            // get the constructor/the factory method to create a platform SocketImpl and make it accessible
             if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_13))
             {
-                final boolean usePlainSocketImpl = ReflectionUtils.readStaticField(SocketImpl.class, "USE_PLAINSOCKETIMPL");
-                final String socketImplClassName = usePlainSocketImpl ? "java.net.PlainSocketImpl" : "sun.nio.ch.NioSocketImpl";
-
-                CONSTRUCTOR = Class.forName(socketImplClassName).getDeclaredConstructor(boolean.class);
+                // use SocketImpl.createPlatformSocketImpl() instead of messing around with internal SocketImpl classes
+                FACTORY_METHOD = SocketImpl.class.getDeclaredMethod("createPlatformSocketImpl", boolean.class);
+                FACTORY_METHOD.setAccessible(true);
+                CONSTRUCTOR = null;
             }
             else
             {
                 CONSTRUCTOR = Class.forName("java.net.PlainSocketImpl").getDeclaredConstructor();
+                CONSTRUCTOR.setAccessible(true);
+                FACTORY_METHOD = null;
             }
 
-            CONSTRUCTOR.setAccessible(true);
-
-            // get the methods of SocketImpl and make them callable even though they are abstract
+            // get the protected methods of SocketImpl and make them callable
             final Class<?> ABSTRACT_CLASS = Class.forName("java.net.SocketImpl");
 
             ACCEPT_METHOD = ABSTRACT_CLASS.getDeclaredMethod("accept", SocketImpl.class);
@@ -181,7 +182,7 @@ class InstrumentedSocketImpl extends SocketImpl
     }
 
     /**
-     * The actual socket implementation, i.e. a PlainSocketImpl.
+     * The actual socket implementation.
      */
     private final SocketImpl socketImpl;
 
@@ -195,7 +196,7 @@ public InstrumentedSocketImpl()
             // create a SocketImpl instance
             if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_13))
             {
-                socketImpl = (SocketImpl) CONSTRUCTOR.newInstance(false);
+                socketImpl = (SocketImpl) FACTORY_METHOD.invoke(null, false);
             }
             else
             {

src/main/java/com/xceptance/xlt/mastercontroller/MasterController.java

@@ -178,6 +178,11 @@ public class MasterController
      */
     private boolean stoppedByUser = false;
 
+    /**
+     * Keep timer files compressed after download
+     */
+    private boolean compressedTimerFiles = false;
+
     /**
      * Creates a new MasterController object.
      * 
@@ -219,6 +224,7 @@ public MasterController(final Map<String, AgentController> agentControllerMap, f
         resultOutputDirectory = config.getResultOutputDirectory();
 
         isEmbedded = config.isEmbedded();
+        compressedTimerFiles = config.isCompressedTimerFiles();
 
         checkTestPropertiesFileName();
     }
@@ -308,7 +314,7 @@ public boolean downloadTestResults(final TestResultAmount testResultAmount)
         // download results
         final ResultDownloader resultDownloader = new ResultDownloader(downloadExecutor, currentTestResultsDir, tempDirectory,
                                                                        agentControllers, progress);
-        final boolean downloadSuccess = resultDownloader.download(testResultAmount);
+        final boolean downloadSuccess = resultDownloader.download(testResultAmount, compressedTimerFiles);
 
         // inform user
         final FailedAgentControllerCollection failedAgentControllers = resultDownloader.getFailedAgentControllerCollection();