diff --git a/NOTICE.md b/NOTICE.md index 80e594fdf..2a3d70417 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -231,6 +231,12 @@ pegdown * Homepage: https://github.com/sirthias/pegdown * More information in folder: doc/3rd-party-licenses/pegdown +progressbar + + * License: MIT License + * Homepage: https://github.com/ctongfei/progressbar + * More information in folder: doc/3rd-party-licenses/progressbar + selenium-java * License: Apache License 2.0 diff --git a/bin/ec2_admin.cmd b/bin/ec2_admin.cmd index e51e1b9f1..1666315c2 100644 --- a/bin/ec2_admin.cmd +++ b/bin/ec2_admin.cmd @@ -19,5 +19,11 @@ set JAVA_OPTIONS=%JAVA_OPTIONS% -Dlog4j.configuration="file:%XLT_CONFIG_DIR%\ec2 rem set JAVA_OPTIONS=%JAVA_OPTIONS% -agentlib:jdwp=transport=dt_socket,address=localhost:6666,server=y,suspend=n set JAVA_OPTIONS=%JAVA_OPTIONS% -cp "%CLASSPATH%" +:: append options to suppress illegal access warnings for Java 9+ +set PACKAGES=java.xml/com.sun.org.apache.xpath.internal +for %%p in (%PACKAGES%) do set JAVA_OPTIONS=!JAVA_OPTIONS! --add-opens=%%p=ALL-UNNAMED +set JAVA_OPTIONS=%JAVA_OPTIONS% -XX:+IgnoreUnrecognizedVMOptions +rem set JAVA_OPTIONS=%JAVA_OPTIONS% --illegal-access=debug + :: run Java java %JAVA_OPTIONS% com.xceptance.xlt.ec2.Main %* diff --git a/bin/ec2_admin.sh b/bin/ec2_admin.sh index cb13729c6..6f1d811d6 100644 --- a/bin/ec2_admin.sh +++ b/bin/ec2_admin.sh @@ -22,6 +22,12 @@ JAVA_OPTIONS="$JAVA_OPTIONS -Dlog4j.configuration=\"file:$XLT_CONFIG_DIR/ec2_adm #JAVA_OPTIONS="$JAVA_OPTIONS -agentlib:jdwp=transport=dt_socket,address=localhost:6666,server=y,suspend=n" JAVA_OPTIONS="$JAVA_OPTIONS -cp \"$CLASSPATH\"" +# append options to suppress illegal access warnings for Java 9+ +PACKAGES="java.xml/com.sun.org.apache.xpath.internal" +for p in $PACKAGES; do JAVA_OPTIONS="$JAVA_OPTIONS --add-opens=$p=ALL-UNNAMED"; done +JAVA_OPTIONS="$JAVA_OPTIONS -XX:+IgnoreUnrecognizedVMOptions" +#JAVA_OPTIONS="$JAVA_OPTIONS --illegal-access=debug" + # run Java CMD="java $JAVA_OPTIONS com.xceptance.xlt.ec2.Main" ARGS="" diff --git a/config/mastercontroller.properties b/config/mastercontroller.properties index 436bb2b35..71de6d5ee 100644 --- a/config/mastercontroller.properties +++ b/config/mastercontroller.properties @@ -63,6 +63,16 @@ com.xceptance.xlt.mastercontroller.password = xceptance #com.xceptance.xlt.mastercontroller.maxParallelUploads = -1 #com.xceptance.xlt.mastercontroller.maxParallelDownloads = -1 +# ================== +# Result Storage +# ================== + +## Do we want to keep the timer files as .gz files after downloading? This +## will save a lot of disk space and also improve report creation speed when +## the disk is slow, such as on cloud machines. +## - true (default): timer files are stored as GZIPed files +## - false: timer files are expanded, behavior of XLT prior to version 5.4.0 +#com.xceptance.xlt.mastercontroller.compressedTimerFiles = true ############################################################################### diff --git a/doc/3rd-party-licenses/progressbar/LICENSE b/doc/3rd-party-licenses/progressbar/LICENSE new file mode 100644 index 000000000..59a97018c --- /dev/null +++ b/doc/3rd-party-licenses/progressbar/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015--2020 Tongfei Chen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/doc/3rd-party-licenses/progressbar/NOTICE b/doc/3rd-party-licenses/progressbar/NOTICE new file mode 100644 index 000000000..229485f42 --- /dev/null +++ b/doc/3rd-party-licenses/progressbar/NOTICE @@ -0,0 +1,6 @@ +Terminal-based progress bar for Java / JVM +http://tongfei.me/progressbar/ +https://github.com/ctongfei/progressbar/ + +Copyright (c) 2015--2020 Tongfei Chen +MIT License diff --git a/doc/internal-doc/licenses/licenses.csv b/doc/internal-doc/licenses/licenses.csv index 1fd30cc52..850d03450 100644 --- a/doc/internal-doc/licenses/licenses.csv +++ b/doc/internal-doc/licenses/licenses.csv @@ -33,6 +33,7 @@ junit,Eclipse Public License 1.0,https://junit.org/junit4/, log4j,Apache License 2.0,http://logging.apache.org/log4j/1.2/, neko-htmlunit,Apache License 2.0,https://github.com/HtmlUnit/htmlunit-neko, pegdown,Apache License 2.0,https://github.com/sirthias/pegdown, +progressbar,MIT License,https://github.com/ctongfei/progressbar, selenium-java,Apache License 2.0,https://selenium.dev/, slf4j-log4j12,MIT License,http://www.slf4j.org/, trove,GNU Lesser General Public License 2.1,https://bitbucket.org/trove4j/trove, diff --git a/doc/xltdoc/release-notes/5.4.x.textile b/doc/xltdoc/release-notes/5.4.x.textile new file mode 100644 index 000000000..24ec41196 --- /dev/null +++ b/doc/xltdoc/release-notes/5.4.x.textile @@ -0,0 +1,48 @@ +--- +layout: manual +title: 5.4.x +position: 980 +sorted: true +--- + +h2. XLT 5.4.0 + +h3. Load Testing + +h4. Progress indication when generating reports + +The report generation hasn't indicated the progress in detail which made it hard to judge the remaining runtime. This release adds simple progress bars which will indicate a progress as well as an estimated remaining runtime. Big thanks to the "ProgressBar project":https://github.com/ctongfei/progressbar/! + +h4. Timer files stored in compressed form + +XLT has been keeping all data in clear text files for easy post processing and open data handling. However, the @timers.csv@ files occupy a large share of the required space on disk after a test run. While the download already runs compressed, the final on-disk storage was plain. This release changes this behavior and stores the @timers.csv@ in a compressed GZIP format by default. The report generator learned to deal with compressed as well as uncompressed formats automatically. + +For example, a result set with previously 9.5 GB of storage space now only requires 1.1 GB. As an additional advantage, report generation might run faster on systems with slower disks such as regular cloud machines. Furthermore, the download of results from the agents will also run faster due to less write operations. + +If you find it necessary to revert to the old behavior of plain uncompressed storage, you can set the property @com.xceptance.xlt.mastercontroller.compressedTimerFiles@ in file @mastercontroller.properties@ to false. + +h4. Illegal access warning in ec2_admin + +The @ec2_admin@ raised an illegal reflective access warning when it's reporting problems such as trying to delete something that does not exist: + +bc(plain). +Terminating the selected instances in region 'eu-central-1' ... WARNING: An illegal reflective access operation has occurred +WARNING: Illegal reflective access by com.amazonaws.util.XpathUtils (file:/home/anyone/projects/loadtest/xlt-5.3.0/lib/aws-java-sdk-core-1.11.762.jar) to method com.sun.org.apache.xpath.internal.XPathContext.getDTMManager() +WARNING: Please consider reporting this to the maintainers of com.amazonaws.util.XpathUtils +WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations +WARNING: All illegal access operations will be denied in a future release +Failed: No instances specified (Service: AmazonEC2; Status Code: 400; Error Code: InvalidParameterCombination; Request ID: 23234-3bd3-4157-b365-4bf7b4432221ec7) + +While we cannot fix the illegal access (it is made in a foreign library), we can at least suppress these warnings by passing the appropriate command line options to java in @ec2_admin.sh@. + + +h3. Test Framework + +h4. Socket instrumentation won't work with Java 16 + +When running a test scenario with Java 16, the socket instrumentation code to gather network timing data (connect time, time to first bytes, etc.) could not be installed and the test scenario failed with this error message: + +bc(plain). +java.lang.NoClassDefFoundError: Could not initialize class com.xceptance.xlt.engine.socket.InstrumentedSocketImpl + +This issue is fixed now. \ No newline at end of file diff --git a/pom.xml b/pom.xml index 4dd75386a..c1c0055de 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.xceptance xlt - 5.3.0 + 5.4.0 jar XLT @@ -392,6 +392,16 @@ 2.3.2 + + + me.tongfei + progressbar + 0.9.1 + + diff --git a/src/main/java/com/xceptance/common/util/zip/ZipUtils.java b/src/main/java/com/xceptance/common/util/zip/ZipUtils.java index 071d95613..7147d71ba 100644 --- a/src/main/java/com/xceptance/common/util/zip/ZipUtils.java +++ b/src/main/java/com/xceptance/common/util/zip/ZipUtils.java @@ -22,6 +22,8 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; +import java.util.zip.GZIPOutputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; @@ -32,6 +34,7 @@ import org.apache.commons.logging.LogFactory; import com.xceptance.common.util.ParameterCheckUtils; +import com.xceptance.xlt.common.XltConstants; /** * The ZipUtils class provides convenience methods for creating and unpacking ZIP archives. @@ -296,6 +299,24 @@ public static void addRegularFile(final ZipOutputStream out, final File file, fi * if an I/O error occurs */ public static void unzipFile(final File zipFile, final File directory) throws IOException + { + unzipFile(zipFile, directory, false); + } + + /** + * Unzips the given ZIP file to the specified directory. If the directory does not exist yet, it will be created. + * Depending on the parameter, timers files are stored to disk either in plain or in compressed form (gzipped). + * + * @param zipFile + * the zip file + * @param directory + * the target directory + * @param compressedTimerFiles + * do we want to keep the timers in a compressed form + * @throws java.io.IOException + * if an I/O error occurs + */ + public static void unzipFile(final File zipFile, final File directory, final boolean compressedTimerFiles) throws IOException { ParameterCheckUtils.isReadableFile(zipFile, "zipFile"); ParameterCheckUtils.isNotNull(directory, "directory"); @@ -320,11 +341,34 @@ public static void unzipFile(final File zipFile, final File directory) throws IO } else { - // cannot use this as it DOES close the input stream - // FileUtils.copyToFile(in, file); + // do we want to store the timers compressed + File compressedFile = null; + boolean compressIt = false; - try (final FileOutputStream out = new FileOutputStream(file)) + // shall we compress timers? + if (compressedTimerFiles) { + // we need the name of the file, without any path element + final String fileName = file.getName(); + + boolean b1 = XltConstants.TIMER_FILENAME_PATTERNS.stream().anyMatch(p -> p.asPredicate().test(fileName)); + boolean b2 = XltConstants.CPT_TIMER_FILENAME_PATTERNS.stream().anyMatch(p -> p.asPredicate().test(fileName)); + + // one pattern matched + if (b1 || b2) + { + // determine the new name + compressedFile = new File(directory, entry.getName() + ".gz"); + compressIt = true; // indicate the need for compression + } + } + + try (final OutputStream out = compressIt ? new GZIPOutputStream(new FileOutputStream(compressedFile)) + : new FileOutputStream(file)) + { + // cannot use this as it DOES close the input stream + // FileUtils.copyToFile(in, file); + IOUtils.copy(in, out); } } diff --git a/src/main/java/com/xceptance/xlt/common/XltConstants.java b/src/main/java/com/xceptance/xlt/common/XltConstants.java index 4cec828ef..4a9e45eb6 100644 --- a/src/main/java/com/xceptance/xlt/common/XltConstants.java +++ b/src/main/java/com/xceptance/xlt/common/XltConstants.java @@ -15,6 +15,11 @@ */ package com.xceptance.xlt.common; +import java.util.List; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + /** * Collection of global constants for directories, default values and so on. Some of the values will be overwritten with * custom values later on. Named XltConstants, because there are too many other Constants classes in other packages. @@ -133,6 +138,28 @@ private XltConstants() * The name of the timer files. */ public static final String TIMER_FILENAME = "timers.csv"; + + /** + * The possible name of the timer files. + */ + public static final List TIMER_FILENAME_PATTERNS = + Stream.of( + "^timers\\.csv$", + "^timers\\.csv\\.gz$", + "^timers\\.csv\\.[0-9]{4}-[0-9]{2}-[0-9]{2}$", + "^timers\\.csv\\.[0-9]{4}-[0-9]{2}-[0-9]{2}\\.gz$") + .map(Pattern::compile).collect(Collectors.toList()); + + /** + * The possible name of the CPT timer files. + *

+ * Note: Needed for backward compatibility. Separate CPT timers files have been removed in XLT 4.8. + */ + public static final List CPT_TIMER_FILENAME_PATTERNS = + Stream.of( + "^timer-wd-.+\\.csv$", + "^timer-wd-.+\\.csv\\.gz$") + .map(Pattern::compile).collect(Collectors.toList()); /** * The option name of the from option on the command line. diff --git a/src/main/java/com/xceptance/xlt/engine/socket/InstrumentedSocketImpl.java b/src/main/java/com/xceptance/xlt/engine/socket/InstrumentedSocketImpl.java index 9be4dd6af..a71a0cd91 100644 --- a/src/main/java/com/xceptance/xlt/engine/socket/InstrumentedSocketImpl.java +++ b/src/main/java/com/xceptance/xlt/engine/socket/InstrumentedSocketImpl.java @@ -30,7 +30,6 @@ import org.apache.commons.lang3.JavaVersion; import org.apache.commons.lang3.SystemUtils; -import com.xceptance.common.lang.ReflectionUtils; import com.xceptance.xlt.engine.RequestExecutionContext; /** @@ -43,6 +42,10 @@ */ class InstrumentedSocketImpl extends SocketImpl { + private static final Constructor CONSTRUCTOR; + + private static final Method FACTORY_METHOD; + private static final Method ACCEPT_METHOD; private static final Method AVAILABLE_METHOD; @@ -57,8 +60,6 @@ class InstrumentedSocketImpl extends SocketImpl private static final Method CONNECT_SOCKETADDRESS_METHOD; - private static final Constructor CONSTRUCTOR; - private static final Method CREATE_METHOD; private static final Method GETFILEDESCRIPTOR_METHOD; @@ -89,22 +90,22 @@ class InstrumentedSocketImpl extends SocketImpl { try { - // get the constructor of the chosen SocketImpl subclass and make it accessible + // get the constructor/the factory method to create a platform SocketImpl and make it accessible if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_13)) { - final boolean usePlainSocketImpl = ReflectionUtils.readStaticField(SocketImpl.class, "USE_PLAINSOCKETIMPL"); - final String socketImplClassName = usePlainSocketImpl ? "java.net.PlainSocketImpl" : "sun.nio.ch.NioSocketImpl"; - - CONSTRUCTOR = Class.forName(socketImplClassName).getDeclaredConstructor(boolean.class); + // use SocketImpl.createPlatformSocketImpl() instead of messing around with internal SocketImpl classes + FACTORY_METHOD = SocketImpl.class.getDeclaredMethod("createPlatformSocketImpl", boolean.class); + FACTORY_METHOD.setAccessible(true); + CONSTRUCTOR = null; } else { CONSTRUCTOR = Class.forName("java.net.PlainSocketImpl").getDeclaredConstructor(); + CONSTRUCTOR.setAccessible(true); + FACTORY_METHOD = null; } - CONSTRUCTOR.setAccessible(true); - - // get the methods of SocketImpl and make them callable even though they are abstract + // get the protected methods of SocketImpl and make them callable final Class ABSTRACT_CLASS = Class.forName("java.net.SocketImpl"); ACCEPT_METHOD = ABSTRACT_CLASS.getDeclaredMethod("accept", SocketImpl.class); @@ -181,7 +182,7 @@ class InstrumentedSocketImpl extends SocketImpl } /** - * The actual socket implementation, i.e. a PlainSocketImpl. + * The actual socket implementation. */ private final SocketImpl socketImpl; @@ -195,7 +196,7 @@ public InstrumentedSocketImpl() // create a SocketImpl instance if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_13)) { - socketImpl = (SocketImpl) CONSTRUCTOR.newInstance(false); + socketImpl = (SocketImpl) FACTORY_METHOD.invoke(null, false); } else { diff --git a/src/main/java/com/xceptance/xlt/mastercontroller/MasterController.java b/src/main/java/com/xceptance/xlt/mastercontroller/MasterController.java index be876fda6..de57b6e16 100644 --- a/src/main/java/com/xceptance/xlt/mastercontroller/MasterController.java +++ b/src/main/java/com/xceptance/xlt/mastercontroller/MasterController.java @@ -178,6 +178,11 @@ public class MasterController */ private boolean stoppedByUser = false; + /** + * Keep timer files compressed after download + */ + private boolean compressedTimerFiles = false; + /** * Creates a new MasterController object. * @@ -219,6 +224,7 @@ public MasterController(final Map agentControllerMap, f resultOutputDirectory = config.getResultOutputDirectory(); isEmbedded = config.isEmbedded(); + compressedTimerFiles = config.isCompressedTimerFiles(); checkTestPropertiesFileName(); } @@ -308,7 +314,7 @@ public boolean downloadTestResults(final TestResultAmount testResultAmount) // download results final ResultDownloader resultDownloader = new ResultDownloader(downloadExecutor, currentTestResultsDir, tempDirectory, agentControllers, progress); - final boolean downloadSuccess = resultDownloader.download(testResultAmount); + final boolean downloadSuccess = resultDownloader.download(testResultAmount, compressedTimerFiles); // inform user final FailedAgentControllerCollection failedAgentControllers = resultDownloader.getFailedAgentControllerCollection(); diff --git a/src/main/java/com/xceptance/xlt/mastercontroller/MasterControllerConfiguration.java b/src/main/java/com/xceptance/xlt/mastercontroller/MasterControllerConfiguration.java index 41096d7e7..1edc1050d 100644 --- a/src/main/java/com/xceptance/xlt/mastercontroller/MasterControllerConfiguration.java +++ b/src/main/java/com/xceptance/xlt/mastercontroller/MasterControllerConfiguration.java @@ -101,6 +101,8 @@ public class MasterControllerConfiguration extends AbstractConfiguration private static final String PROP_PASSWORD = PROP_PREFIX + "password"; + private static final String PROP_COMPRESSED_TIMER_FILES = PROP_PREFIX + "compressedTimerFiles"; + private final List agentControllerConnectionInfos; private File agentFilesDirectory; @@ -153,6 +155,8 @@ public class MasterControllerConfiguration extends AbstractConfiguration private final boolean isEmbedded; + private final boolean compressedTimerFiles; + /** * Creates a new MasterControllerConfiguration object. * @@ -268,6 +272,9 @@ else if (!(tempDirectory.isDirectory() && tempDirectory.canWrite())) // user name/password userName = XltConstants.USER_NAME; password = getStringProperty(PROP_PASSWORD, null); + + // do we want to keep the timer files compressed for efficency + compressedTimerFiles = getBooleanProperty(PROP_COMPRESSED_TIMER_FILES, true); } /** @@ -631,5 +638,14 @@ public boolean isEmbedded() { return isEmbedded; } - + + /** + * How shall we handle timer files after the download + * + * @return true, keep them compressed, false, classic expanded storage + */ + public boolean isCompressedTimerFiles() + { + return compressedTimerFiles; + } } diff --git a/src/main/java/com/xceptance/xlt/mastercontroller/ResultDownloader.java b/src/main/java/com/xceptance/xlt/mastercontroller/ResultDownloader.java index 767a82ca7..b41bcc551 100644 --- a/src/main/java/com/xceptance/xlt/mastercontroller/ResultDownloader.java +++ b/src/main/java/com/xceptance/xlt/mastercontroller/ResultDownloader.java @@ -74,7 +74,7 @@ public ResultDownloader(final ThreadPoolExecutor downloadExecutor, final File te /** * @progresscount 7 ac + 4 */ - public boolean download(final TestResultAmount testResultAmount) + public boolean download(final TestResultAmount testResultAmount, final boolean compressedTimerFiles) { // download test configuration final boolean testConfigDownloaded = getRemoteTestConfig(); @@ -94,7 +94,7 @@ public boolean download(final TestResultAmount testResultAmount) archiveResults(testResultAmount); // download and unzip archives - final boolean resultsDownloaded = downloadResults(testResultAmount); + final boolean resultsDownloaded = downloadResults(testResultAmount, compressedTimerFiles); // We have downloaded results from at least 1 agent controller. // AND @@ -386,7 +386,7 @@ public void run() /** * @progresscount 5 * ac */ - private boolean downloadResults(final TestResultAmount testResultAmount) + private boolean downloadResults(final TestResultAmount testResultAmount, final boolean compressedTimerFiles) { LOG.debug("Download results"); try @@ -400,7 +400,7 @@ public boolean call(final AgentController agentController) throws Exception { // download the archive LOG.debug("Downloading results from " + agentController); - downloadTestResults(agentController, testResultAmount); + downloadTestResults(agentController, testResultAmount, compressedTimerFiles); LOG.debug("Downloading results from " + agentController + " OK"); return true; } @@ -485,7 +485,7 @@ private boolean updateTimeData(final File testPropFile) * if an I/O error occurs * @progresscount 4 */ - private void downloadTestResults(final AgentController agentController, final TestResultAmount testResultAmount) throws IOException + private void downloadTestResults(final AgentController agentController, final TestResultAmount testResultAmount, final boolean compressedTimerFiles) throws IOException { /** agentID, downloadedZipFile */ final Map downloadedZipFiles = new HashMap(); @@ -528,7 +528,7 @@ private void downloadTestResults(final AgentController agentController, final Te final File agentResultsDir = new File(testResultsDir, agentID); LOG.debug("Unzipping '" + zipFile + "' to '" + agentResultsDir + "' ..."); - ZipUtils.unzipFile(zipFile, agentResultsDir); + ZipUtils.unzipFile(zipFile, agentResultsDir, compressedTimerFiles); } progress.increaseCount(); } diff --git a/src/main/java/com/xceptance/xlt/report/DataRecordReader.java b/src/main/java/com/xceptance/xlt/report/DataRecordReader.java index 40d3a8f1e..19b44c653 100644 --- a/src/main/java/com/xceptance/xlt/report/DataRecordReader.java +++ b/src/main/java/com/xceptance/xlt/report/DataRecordReader.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.atomic.AtomicInteger; +import java.util.zip.GZIPInputStream; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -149,12 +150,15 @@ private void readLogsFromTestUserDir() throws Exception if (file.getType() == FileType.FILE && file.isReadable()) { final String fileName = file.getName().getBaseName(); - if (fileName.startsWith(XltConstants.TIMER_FILENAME)) + + // timers.csv and timers.csv.gz + if (XltConstants.TIMER_FILENAME_PATTERNS.stream().anyMatch(r -> r.asPredicate().test(fileName))) { // remember regular timer files for later processing regularTimerFiles.add(file); } - else if (fileName.startsWith("timer-wd-") && fileName.endsWith(".csv")) + // timer-wd-.csv[.gz] (for backward compatibility with XLT < 4.8) + else if (XltConstants.CPT_TIMER_FILENAME_PATTERNS.stream().anyMatch(r -> r.asPredicate().test(fileName))) { // remember client performance timer files for later processing clientPerformanceTimerFiles.add(file); @@ -195,10 +199,15 @@ else if (fileName.startsWith("timer-wd-") && fileName.endsWith(".csv")) private void readTimerLog(final FileObject file, final boolean collectActionNames, final boolean adjustTimerName) { // that costs a lot of time, no idea why... real async logger might be an option, LOG.info did not help - // System.out.printf("Reading file '%s' ...", file)); - // LOG.info(String.format("Reading file '%s' ...", file)); +// System.out.printf("Reading file '%s' ...%n", file); +// LOG.info(String.format("Reading file '%s' ...", file)); + + // if we have an gz extension, we will try to decompress it while reading + final boolean isCompressed = "gz".equalsIgnoreCase(file.getName().getExtension()); - try (final BufferedReader reader = new BufferedReader(new InputStreamReader(file.getContent().getInputStream(), + try (final BufferedReader reader = new BufferedReader(new InputStreamReader(isCompressed ? new GZIPInputStream(file.getContent() + .getInputStream()) + : file.getContent().getInputStream(), XltConstants.UTF8_ENCODING))) { List lines = new ArrayList(CHUNK_SIZE); diff --git a/src/main/java/com/xceptance/xlt/report/Dispatcher.java b/src/main/java/com/xceptance/xlt/report/Dispatcher.java index da54c0d7b..010682767 100644 --- a/src/main/java/com/xceptance/xlt/report/Dispatcher.java +++ b/src/main/java/com/xceptance/xlt/report/Dispatcher.java @@ -19,10 +19,15 @@ import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicLong; import com.xceptance.common.util.SynchronizingCounter; import com.xceptance.xlt.api.engine.Data; +import me.tongfei.progressbar.ProgressBar; +import me.tongfei.progressbar.ProgressBarBuilder; +import me.tongfei.progressbar.ProgressBarStyle; + /** * The {@link Dispatcher} is responsible to coordinate the various reader/parser/processor threads involved when * processing test results. It does not only pass the results from one thread to another, but makes sure as well that no @@ -34,10 +39,15 @@ */ public class Dispatcher { + /** + * The total number of directories to be processed. + */ + private final AtomicLong totalDirectoryCount = new AtomicLong(); + /** * The number of directories that still need to be processed. */ - private final SynchronizingCounter directoriesToBeProcessed; + private final SynchronizingCounter directoriesToBeProcessed = new SynchronizingCounter(); /** * The number of chunks that still need to be processed. @@ -59,6 +69,11 @@ public class Dispatcher */ private final BlockingQueue> dataRecordChunkQueue; + /** + * Our progress bar + */ + private final ProgressBar progressBar = new ProgressBarBuilder().setTaskName("Reading").setStyle(ProgressBarStyle.ASCII).build(); + /** * Creates a new {@link Dispatcher} object with the given thread limit. * @@ -67,16 +82,23 @@ public class Dispatcher * @param maxActiveThreads * the maximum number of active threads */ - public Dispatcher(final SynchronizingCounter directoriesToBeProcessed, final int maxActiveThreads) + public Dispatcher(final int maxActiveThreads) { - this.directoriesToBeProcessed = directoriesToBeProcessed; - permits = new Semaphore(maxActiveThreads); chunksToBeProcessed = new SynchronizingCounter(); lineChunkQueue = new ArrayBlockingQueue(10); dataRecordChunkQueue = new ArrayBlockingQueue>(10); } + /** + * Indicates that another test user directory has been scheduled for processing. Called from the main thread. + */ + public void addDirectory() + { + progressBar.maxHint(totalDirectoryCount.incrementAndGet()); + directoriesToBeProcessed.increment(); + } + /** * Indicates that a reader thread is about to begin reading. Called by a reader thread. */ @@ -105,8 +127,8 @@ public void addNewLineChunk(final LineChunk lineChunk) throws InterruptedExcepti */ public void finishedReading() { + progressBar.step(); directoriesToBeProcessed.decrement(); - permits.release(); } @@ -170,5 +192,7 @@ public void waitForDataRecordProcessingToComplete() throws InterruptedException // wait for the data processor thread to finish data record chunks chunksToBeProcessed.awaitZero(); + + progressBar.close(); } } diff --git a/src/main/java/com/xceptance/xlt/report/LogReader.java b/src/main/java/com/xceptance/xlt/report/LogReader.java index 849159207..e47eb040d 100644 --- a/src/main/java/com/xceptance/xlt/report/LogReader.java +++ b/src/main/java/com/xceptance/xlt/report/LogReader.java @@ -28,7 +28,6 @@ import org.apache.commons.vfs2.FileType; import com.xceptance.common.util.StringMatcher; -import com.xceptance.common.util.SynchronizingCounter; import com.xceptance.common.util.concurrent.DaemonThreadFactory; import com.xceptance.xlt.agent.CustomSamplersRunner; import com.xceptance.xlt.agent.JvmResourceUsageDataGenerator; @@ -70,11 +69,6 @@ public class LogReader */ private final ExecutorService dataRecordReaderExecutor; - /** - * The number of directories that still needs to be processed. - */ - private final SynchronizingCounter directoriesToBeProcessed; - /** * The dispatcher that coordinates all the reader/parser/processor threads. */ @@ -137,7 +131,6 @@ public LogReader(final FileObject inputDir, final DataRecordFactory dataRecordFa this.inputDir = inputDir; totalLinesCounter = new AtomicInteger(); - directoriesToBeProcessed = new SynchronizingCounter(); testCaseFilter = new StringMatcher(testCaseIncludePatternList, testCaseExcludePatternList, true); agentFilter = new StringMatcher(agentIncludePatternList, agentExcludePatternList, true); @@ -167,7 +160,7 @@ public LogReader(final FileObject inputDir, final DataRecordFactory dataRecordFa } // create the dispatcher - dispatcher = new Dispatcher(directoriesToBeProcessed, maxActiveThreadCount); + dispatcher = new Dispatcher(maxActiveThreadCount); // create the reader executor dataRecordReaderExecutor = Executors.newFixedThreadPool(readerThreadCount, new DaemonThreadFactory("DataRecordReader-")); @@ -213,8 +206,6 @@ public final long getMinimumTime() */ public void readDataRecords() { - System.out.printf("Reading files from input directory '%s' ...\n", inputDir); - try { final long start = TimerUtils.getTime(); @@ -313,7 +304,7 @@ private void readDataRecordsFromTestCaseDir(final FileObject testCaseDir, final private void readDataRecordsFromTestUserDir(final FileObject testUserDir, final String agentName, final String testCaseName) throws Exception { - directoriesToBeProcessed.increment(); + dispatcher.addDirectory(); // create a new reader for each user directory and enqueue it for execution final String userNumber = testUserDir.getName().getBaseName(); diff --git a/src/main/java/com/xceptance/xlt/report/ReportGenerator.java b/src/main/java/com/xceptance/xlt/report/ReportGenerator.java index 69ab98264..dc0c5210a 100644 --- a/src/main/java/com/xceptance/xlt/report/ReportGenerator.java +++ b/src/main/java/com/xceptance/xlt/report/ReportGenerator.java @@ -246,8 +246,13 @@ public void generateReport(final long fromTime, final long toTime, final long du // clean output directory first -> Improvement #3243 FileUtils.cleanDirectory(outputDir); + System.out.printf("Reading files from input directory '%s' ...%n", inputDir); readLogs(fromTime, toTime, duration, noRampUp, fromTimeRel, toTimeRel); + + System.out.printf("%nCreating report artifacts ...%n"); final File xmlReport = createReport(outputDir); + + System.out.printf("Transforming XML data file '%s' ...%n", xmlReport); transformReport(xmlReport, outputDir); // output the path to the report either as file path (Win) or as clickable file URL @@ -378,9 +383,10 @@ private void read(final long fromTime, final long toTime) } // read the logs - final LogReader logReader = new LogReader(inputDir, statsFactory, fromTime, toTime, reportProviders, config.getRequestProcessingRules(), - config.getThreadCount(), testCaseIncludePatternList, testCaseExcludePatternList, - agentIncludePatternList, agentExcludePatternList, config.getRemoveIndexesFromRequestNames()); + final LogReader logReader = new LogReader(inputDir, statsFactory, fromTime, toTime, reportProviders, + config.getRequestProcessingRules(), config.getThreadCount(), testCaseIncludePatternList, + testCaseExcludePatternList, agentIncludePatternList, agentExcludePatternList, + config.getRemoveIndexesFromRequestNames()); logReader.readDataRecords(); final long minTime = logReader.getMinimumTime(); @@ -504,7 +510,7 @@ public File createReport(final File outputDir) throws Exception } // create the report - System.out.println("\nCreating report artifacts ..."); + TaskManager.getInstance().startProgress("Creating"); final long start = TimerUtils.getTime(); @@ -513,6 +519,7 @@ public File createReport(final File outputDir) throws Exception // wait for any asynchronous task to complete (e.g. chart generation) TaskManager.getInstance().waitForAllTasksToComplete(); + TaskManager.getInstance().stopProgress(); System.out.printf("Report artifacts created successfully (%,d ms)\n\n", TimerUtils.getTime() - start); @@ -635,12 +642,14 @@ public void transformReport(final File inputXmlFile, final File outputDir) throw // transform the report final ReportTransformer reportTransformer = new ReportTransformer(outputFiles, styleSheetFiles, parameters); + TaskManager.getInstance().startProgress("Transforming"); final long start = TimerUtils.getTime(); reportTransformer.run(inputXmlFile, outputDir); // wait for any asynchronous task to complete TaskManager.getInstance().waitForAllTasksToComplete(); + TaskManager.getInstance().stopProgress(); System.out.printf("Transformation completed successfully (%,d ms)\n", TimerUtils.getTime() - start); } diff --git a/src/main/java/com/xceptance/xlt/report/ReportTransformer.java b/src/main/java/com/xceptance/xlt/report/ReportTransformer.java index 8a8d697c7..72db3c6c1 100644 --- a/src/main/java/com/xceptance/xlt/report/ReportTransformer.java +++ b/src/main/java/com/xceptance/xlt/report/ReportTransformer.java @@ -54,8 +54,6 @@ public ReportTransformer(final List outputFiles, final List styleShe */ public void run(final File inputXmlFile, final File outputDir) { - System.out.printf("Transforming XML data file '%s' ...\n", inputXmlFile); - for (int i = 0; i < outputFiles.size(); i++) { final File outputFile = outputFiles.get(i); diff --git a/src/main/java/com/xceptance/xlt/report/util/TaskManager.java b/src/main/java/com/xceptance/xlt/report/util/TaskManager.java index 994d2d859..ca14b5dfa 100644 --- a/src/main/java/com/xceptance/xlt/report/util/TaskManager.java +++ b/src/main/java/com/xceptance/xlt/report/util/TaskManager.java @@ -15,14 +15,18 @@ */ package com.xceptance.xlt.report.util; -import java.lang.management.ManagementFactory; import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import com.xceptance.xlt.util.ConcurrencyUtils; +import me.tongfei.progressbar.ProgressBar; +import me.tongfei.progressbar.ProgressBarBuilder; +import me.tongfei.progressbar.ProgressBarStyle; + /** * A simple manager for asynchronous tasks. */ @@ -46,6 +50,17 @@ private static class SingletonHolder */ private static final long INTERVAL = 500; + /** + * The current progress bar. Never null. Note that the initial progress bar is inactive. To get active + * visual progress, call {@link #startProgress(String)}. + */ + private volatile ProgressBar progressBar = new ProgressBarBuilder().setStyle(ProgressBarStyle.ASCII).build().pause(); + + /** + * Total tasks for the progress bar. + */ + private final AtomicInteger totalTasks = new AtomicInteger(0); + /** * Returns the {@link TaskManager} singleton. * @@ -64,7 +79,7 @@ public static TaskManager getInstance() /** * The default maximum count of threads, which is equal to the number of available CPUs on the current machine. */ - public static final int DEFAULT_THREAD_COUNT = ManagementFactory.getOperatingSystemMXBean().getAvailableProcessors(); + public static final int DEFAULT_THREAD_COUNT = Runtime.getRuntime().availableProcessors(); /** * Constructor. @@ -74,6 +89,24 @@ private TaskManager() init(); } + /** + * Starts the progress meter. + */ + public void startProgress(final String msg) + { + totalTasks.set(0); + progressBar = new ProgressBarBuilder().setTaskName(msg).setStyle(ProgressBarStyle.ASCII).build(); + } + + /** + * Stops the progress meter. + */ + public void stopProgress() + { + progressBar.close(); + totalTasks.set(0); + } + /** * Adds the given task to the to-do list. * @@ -82,6 +115,8 @@ private TaskManager() */ public void addTask(final Runnable task) { + progressBar.maxHint(totalTasks.incrementAndGet()); + // wrap the task to allow for exception logging getExecutor().execute(new Runnable() { @@ -96,6 +131,10 @@ public void run() { log.error("Failed to execute task", e); } + finally + { + progressBar.step(); + } } }); } diff --git a/src/test/java/com/xceptance/xlt/report/DataRecordReaderTest.java b/src/test/java/com/xceptance/xlt/report/DataRecordReaderTest.java new file mode 100644 index 000000000..69a5c9007 --- /dev/null +++ b/src/test/java/com/xceptance/xlt/report/DataRecordReaderTest.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2005-2021 Xceptance Software Technologies GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.xceptance.xlt.report; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.function.Predicate; + +import org.junit.Test; + +import com.xceptance.xlt.common.XltConstants; + +public class DataRecordReaderTest +{ + /** + * Ensures that the filter catches the valid timer file names only. + */ + @Test + public void timerFilter() + { + final Predicate p = s -> XltConstants.TIMER_FILENAME_PATTERNS.stream().anyMatch(r -> r.asPredicate().test(s)); + + assertTrue(p.test("timers.csv")); + assertTrue(p.test("timers.csv.gz")); + assertTrue(p.test("timers.csv.2012-01-01")); + assertTrue(p.test("timers.csv.2012-01-01.gz")); + + assertFalse(p.test("timer-wd-57362576329865634278.csv")); + assertFalse(p.test("timer-wd-57362576329865634278.csv.gz")); + + assertFalse(p.test("atimers.csv")); + assertFalse(p.test("timers.gz")); + assertFalse(p.test("timers.2020.csv")); + assertFalse(p.test("Timers.csv")); + } + + /** + * Ensures that the filter catches the valid CPT timer file names only. + */ + @Test + public void timerWDFilter() + { + final Predicate p = s -> XltConstants.CPT_TIMER_FILENAME_PATTERNS.stream().anyMatch(r -> r.asPredicate().test(s)); + + assertTrue(p.test("timer-wd-57362576329865634278.csv")); + assertTrue(p.test("timer-wd-57362576329865634278.csv.gz")); + + assertFalse(p.test("timers.csv")); + assertFalse(p.test("timers.csv.gz")); + assertFalse(p.test("timers.csv.2012-01-01")); + assertFalse(p.test("timers.csv.2012-01-01.gz")); + + assertFalse(p.test("timer-wd.csv")); + assertFalse(p.test("timer-wd.csv.gz")); + } +}