From 8f567aa76433f9424872a6a82bb3d5117810f989 Mon Sep 17 00:00:00 2001 From: h-arlt <4641065+h-arlt@users.noreply.github.com> Date: Thu, 11 Jul 2024 16:58:54 +0200 Subject: [PATCH 1/2] 493: Partially automated test evaluation (#494) - dropped Xalan in favor of SaxonHE that supports XPath up to v3.1 - added JSON schema and class model for evaluation configuration - class model for evaluation result - added XSL stylesheets for transformation of evaluation result XML - integrate evaluation into report generator - dropped DOCTYPE declarations from existing XSL stylesheets as they became obsolete w/ SaxonHE - Include JSON files when downloading test-suite config from agent - added shell script to update evaluation/scorecard of an existing report for Unix and Windows - highlight rules that were effective for a given group in report - assemble message(s) for a group from its effective rules and display it in report - Mark rules and groups that cause test to fail explicitly to ease lookup in XSL stylesheet - group, rule and rule-check status is displayed as 'Result' and stored as 'result' element in XML - introduce property 'failsOn' for groups and rules to allow configuration of test-fail trigger; allowed values: 'PASSED', 'NOTPASSED'; default: 'NOTPASSED' --------- Co-authored-by: Hartmut Arlt Co-authored-by: Joerg Werner <4639399+jowerner@users.noreply.github.com> --- NOTICE.md | 18 +- bin/update_scorecard.cmd | 31 + bin/update_scorecard.sh | 47 + config/xsl/common/sections/header.xsl | 60 +- config/xsl/diffreport/actions.xsl | 1 - config/xsl/diffreport/custom-timers.xsl | 1 - config/xsl/diffreport/custom-values.xsl | 1 - config/xsl/diffreport/index.xsl | 1 - config/xsl/diffreport/page-load-timings.xsl | 1 - config/xsl/diffreport/requests.xsl | 1 - config/xsl/diffreport/transactions.xsl | 1 - config/xsl/diffreport/web-vitals.xsl | 1 - config/xsl/loadreport/actions.xsl | 8 +- config/xsl/loadreport/agents.xsl | 8 +- config/xsl/loadreport/configuration.xsl | 8 +- config/xsl/loadreport/custom-timers.xsl | 8 +- config/xsl/loadreport/custom-values.xsl | 8 +- config/xsl/loadreport/errors.xsl | 8 +- config/xsl/loadreport/events.xsl | 8 +- config/xsl/loadreport/external.xsl | 6 +- config/xsl/loadreport/index.xsl | 8 +- config/xsl/loadreport/network.xsl | 8 +- config/xsl/loadreport/page-load-timings.xsl | 8 +- config/xsl/loadreport/requests.xsl | 8 +- config/xsl/loadreport/sections/navigation.xsl | 120 +- config/xsl/loadreport/transactions.xsl | 8 +- config/xsl/loadreport/web-vitals.xsl | 8 +- config/xsl/scorecard/index.xsl | 134 ++ config/xsl/scorecard/sections/groups.xsl | 121 ++ config/xsl/scorecard/sections/ratings.xsl | 79 ++ config/xsl/scorecard/sections/rule-checks.xsl | 128 ++ config/xsl/scorecard/sections/rules.xsl | 157 +++ config/xsl/scorecard/sections/scorecard.xsl | 117 ++ config/xsl/scorecard/sections/util.xsl | 45 + config/xsl/scorecard/text/descriptions.xsl | 105 ++ config/xsl/trendreport/actions.xsl | 1 - config/xsl/trendreport/custom-timers.xsl | 1 - config/xsl/trendreport/index.xsl | 1 - config/xsl/trendreport/page-load-timings.xsl | 1 - config/xsl/trendreport/requests.xsl | 1 - config/xsl/trendreport/transactions.xsl | 1 - doc/3rd-party-licenses/saxon-he/LICENSE | 373 +++++ doc/3rd-party-licenses/xalan/LICENSE | 202 --- doc/3rd-party-licenses/xalan/NOTICE | 15 - doc/3rd-party-licenses/xercesImpl/LICENSE | 202 --- doc/3rd-party-licenses/xercesImpl/NOTICE | 17 - doc/internal-doc/licenses/licenses.csv | 3 +- pom.xml | 47 +- .../config/project.properties | 7 + .../config/scorecard-config.json | 1215 +++++++++++++++++ .../agentcontroller/AgentControllerImpl.java | 3 +- .../xceptance/xlt/common/XltConstants.java | 34 + .../engine/resultbrowser/ErrorCounter.java | 4 +- .../xceptance/xlt/report/ReportGenerator.java | 283 +++- .../xlt/report/XmlReportGenerator.java | 42 +- .../xlt/report/scorecard/Configuration.java | 308 +++++ .../xlt/report/scorecard/Evaluator.java | 583 ++++++++ .../xlt/report/scorecard/GroupDefinition.java | 227 +++ .../report/scorecard/RatingDefinition.java | 88 ++ .../xlt/report/scorecard/RuleDefinition.java | 298 ++++ .../xlt/report/scorecard/Scorecard.java | 431 ++++++ .../report/scorecard/SelectorDefinition.java | 41 + .../xlt/report/scorecard/Status.java | 71 + .../xlt/report/scorecard/TestFailTrigger.java | 23 + .../xlt/report/scorecard/UpdateMain.java | 150 ++ .../report/scorecard/ValidationException.java | 21 + .../util/xstream/SanitizingDomDriver.java | 18 + .../report/util/xstream/SanitizingWriter.java | 30 + .../scorecard/configuration-schema.json | 259 ++++ 69 files changed, 5573 insertions(+), 708 deletions(-) create mode 100644 bin/update_scorecard.cmd create mode 100755 bin/update_scorecard.sh create mode 100644 config/xsl/scorecard/index.xsl create mode 100644 config/xsl/scorecard/sections/groups.xsl create mode 100644 config/xsl/scorecard/sections/ratings.xsl create mode 100644 config/xsl/scorecard/sections/rule-checks.xsl create mode 100644 config/xsl/scorecard/sections/rules.xsl create mode 100644 config/xsl/scorecard/sections/scorecard.xsl create mode 100644 config/xsl/scorecard/sections/util.xsl create mode 100644 config/xsl/scorecard/text/descriptions.xsl create mode 100644 doc/3rd-party-licenses/saxon-he/LICENSE delete mode 100644 doc/3rd-party-licenses/xalan/LICENSE delete mode 100644 doc/3rd-party-licenses/xalan/NOTICE delete mode 100644 doc/3rd-party-licenses/xercesImpl/LICENSE delete mode 100644 doc/3rd-party-licenses/xercesImpl/NOTICE create mode 100644 samples/testsuite-posters/config/scorecard-config.json create mode 100644 src/main/java/com/xceptance/xlt/report/scorecard/Configuration.java create mode 100644 src/main/java/com/xceptance/xlt/report/scorecard/Evaluator.java create mode 100644 src/main/java/com/xceptance/xlt/report/scorecard/GroupDefinition.java create mode 100644 src/main/java/com/xceptance/xlt/report/scorecard/RatingDefinition.java create mode 100644 src/main/java/com/xceptance/xlt/report/scorecard/RuleDefinition.java create mode 100644 src/main/java/com/xceptance/xlt/report/scorecard/Scorecard.java create mode 100644 src/main/java/com/xceptance/xlt/report/scorecard/SelectorDefinition.java create mode 100644 src/main/java/com/xceptance/xlt/report/scorecard/Status.java create mode 100644 src/main/java/com/xceptance/xlt/report/scorecard/TestFailTrigger.java create mode 100644 src/main/java/com/xceptance/xlt/report/scorecard/UpdateMain.java create mode 100644 src/main/java/com/xceptance/xlt/report/scorecard/ValidationException.java create mode 100644 src/main/java/com/xceptance/xlt/report/util/xstream/SanitizingDomDriver.java create mode 100644 src/main/java/com/xceptance/xlt/report/util/xstream/SanitizingWriter.java create mode 100644 src/main/resources/com/xceptance/xlt/report/scorecard/configuration-schema.json diff --git a/NOTICE.md b/NOTICE.md index 40f7f3859..4dcaf9174 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -250,6 +250,12 @@ progressbar * Homepage: https://github.com/ctongfei/progressbar * More information in folder: doc/3rd-party-licenses/progressbar +saxon-he + + * License: Mozilla Public License 2.0 + * Homepage: https://www.saxonica.com/ + * More information in folder: doc/3rd-party-licenses/saxon-he + selenium-java * License: Apache License 2.0 @@ -298,18 +304,6 @@ websocket-client * Homepage: https://www.eclipse.org/jetty/ * More information in folder: doc/3rd-party-licenses/websocket-client -xalan - - * License: Apache License 2.0 - * Homepage: https://xml.apache.org/xalan-j/ - * More information in folder: doc/3rd-party-licenses/xalan - -xerces - - * License: Apache License 2.0 - * Homepage: http://xerces.apache.org/xerces2-j/ - * More information in folder: doc/3rd-party-licenses/xerces - xstream * License: BSD 3-Clause License diff --git a/bin/update_scorecard.cmd b/bin/update_scorecard.cmd new file mode 100644 index 000000000..ee5456ce5 --- /dev/null +++ b/bin/update_scorecard.cmd @@ -0,0 +1,31 @@ +@echo off +setlocal enabledelayedexpansion + +:: setup basic paths +set CWD=%CD% +cd /d %~dp0\.. +set XLT_HOME=%CD% +cd /d %CWD% + +if not defined XLT_CONFIG_DIR set XLT_CONFIG_DIR=%XLT_HOME%\config + +:: setup Java class path +set CLASSPATH=%XLT_HOME%\target\classes;%XLT_HOME%\lib\* + +:: setup other Java options +set JAVA_OPTIONS= +set JAVA_OPTIONS=%JAVA_OPTIONS% -Xmx4g +set JAVA_OPTIONS=%JAVA_OPTIONS% -XX:+UseStringDeduplication +set JAVA_OPTIONS=%JAVA_OPTIONS% -Dcom.xceptance.xlt.home="%XLT_HOME%" +set JAVA_OPTIONS=%JAVA_OPTIONS% -Dlog4j2.configurationFile="%XLT_CONFIG_DIR%\reportgenerator.properties" +rem set JAVA_OPTIONS=%JAVA_OPTIONS% -agentlib:jdwp=transport=dt_socket,address=localhost:6666,server=y,suspend=n +set JAVA_OPTIONS=%JAVA_OPTIONS% -cp "%CLASSPATH%" + +:: append options to suppress illegal access warnings for Java 9+ +set PACKAGES=java.base/java.lang.reflect java.base/java.text java.base/java.util java.desktop/java.awt.font +for %%p in (%PACKAGES%) do set JAVA_OPTIONS=!JAVA_OPTIONS! --add-opens=%%p=ALL-UNNAMED +set JAVA_OPTIONS=%JAVA_OPTIONS% -XX:+IgnoreUnrecognizedVMOptions +rem set JAVA_OPTIONS=%JAVA_OPTIONS% --illegal-access=debug + +:: run Java +java %JAVA_OPTIONS% com.xceptance.xlt.report.scorecard.UpdateMain %* diff --git a/bin/update_scorecard.sh b/bin/update_scorecard.sh new file mode 100755 index 000000000..3ba2d7976 --- /dev/null +++ b/bin/update_scorecard.sh @@ -0,0 +1,47 @@ +#!/bin/sh + +# setup basic paths +CWD=`pwd` +cd "`dirname "$0"`/.." +XLT_HOME=`pwd` +cd "$CWD" +export XLT_HOME + +if [ -z "$XLT_CONFIG_DIR" ]; then + XLT_CONFIG_DIR=$XLT_HOME/config + export XLT_CONFIG_DIR +fi + +# setup Java class path +CLASSPATH="$XLT_HOME"/target/classes:"$XLT_HOME"/lib/* + +# setup other Java options +JAVA_OPTIONS= +JAVA_OPTIONS="$JAVA_OPTIONS -Xmx4g" +JAVA_OPTIONS="$JAVA_OPTIONS -XX:+UseStringDeduplication" +JAVA_OPTIONS="$JAVA_OPTIONS -Dcom.xceptance.xlt.home=\"$XLT_HOME\"" +JAVA_OPTIONS="$JAVA_OPTIONS -Dlog4j2.configurationFile=\"$XLT_CONFIG_DIR/reportgenerator.properties\"" +JAVA_OPTIONS="$JAVA_OPTIONS -Djava.awt.headless=true" +#JAVA_OPTIONS="$JAVA_OPTIONS -agentlib:jdwp=transport=dt_socket,address=localhost:6666,server=y,suspend=n" +#JAVA_OPTIONS="$JAVA_OPTIONS -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:+UnlockDiagnosticVMOptions -XX:+DebugNonSafepoints -XX:FlightRecorderOptions=stackdepth=1024" +#JAVA_OPTIONS="$JAVA_OPTIONS -XX:+UnlockDiagnosticVMOptions -XX:+PrintCompilation -XX:+PrintInlining" +#JAVA_OPTIONS="$JAVA_OPTIONS -XX:MaxInlineSize=128 -XX:FreqInlineSize=1024" + +JAVA_OPTIONS="$JAVA_OPTIONS -cp \"$CLASSPATH\"" + +# append options to suppress illegal access warnings for Java 9+ +PACKAGES="java.base/java.lang.reflect java.base/java.text java.base/java.util java.desktop/java.awt.font" +for p in $PACKAGES; do JAVA_OPTIONS="$JAVA_OPTIONS --add-opens=$p=ALL-UNNAMED"; done +JAVA_OPTIONS="$JAVA_OPTIONS -XX:+IgnoreUnrecognizedVMOptions" +#JAVA_OPTIONS="$JAVA_OPTIONS --illegal-access=debug" + +# run Java +CMD="java $JAVA_OPTIONS com.xceptance.xlt.report.scorecard.UpdateMain" +ARGS="" +I=1 +while [ $I -le $# ]; do + eval x=\${$I} + ARGS="$ARGS \"$x\"" + I=$((I+1)) +done +eval $CMD "$ARGS" diff --git a/config/xsl/common/sections/header.xsl b/config/xsl/common/sections/header.xsl index 62edaf42e..672eb528d 100644 --- a/config/xsl/common/sections/header.xsl +++ b/config/xsl/common/sections/header.xsl @@ -8,11 +8,19 @@ + + + + + + + +