diff --git a/config/xsl/loadreport/agents.xsl b/config/xsl/loadreport/agents.xsl
index 0c5a371f1..c368763bb 100644
--- a/config/xsl/loadreport/agents.xsl
+++ b/config/xsl/loadreport/agents.xsl
@@ -36,11 +36,11 @@
-<! DOCTYPE html >
-
+
-
+
diff --git a/config/xsl/loadreport/configuration.xsl b/config/xsl/loadreport/configuration.xsl
index 75ea3efbb..40d35a157 100644
--- a/config/xsl/loadreport/configuration.xsl
+++ b/config/xsl/loadreport/configuration.xsl
@@ -35,11 +35,11 @@
-<! DOCTYPE html >
-
+
-
+
diff --git a/config/xsl/loadreport/custom-timers.xsl b/config/xsl/loadreport/custom-timers.xsl
index 44a5a0aa5..7c870bcc3 100644
--- a/config/xsl/loadreport/custom-timers.xsl
+++ b/config/xsl/loadreport/custom-timers.xsl
@@ -41,11 +41,11 @@
-<! DOCTYPE html >
-
+
-
+
diff --git a/config/xsl/loadreport/custom-values.xsl b/config/xsl/loadreport/custom-values.xsl
index 4a46b785d..a3714a1f2 100644
--- a/config/xsl/loadreport/custom-values.xsl
+++ b/config/xsl/loadreport/custom-values.xsl
@@ -41,11 +41,11 @@
-<! DOCTYPE html >
-
+
-
+
diff --git a/config/xsl/loadreport/errors.xsl b/config/xsl/loadreport/errors.xsl
index ee2c2dad8..35ec9f6c5 100644
--- a/config/xsl/loadreport/errors.xsl
+++ b/config/xsl/loadreport/errors.xsl
@@ -33,11 +33,11 @@
-<! DOCTYPE html >
-
+
-
+
diff --git a/config/xsl/loadreport/events.xsl b/config/xsl/loadreport/events.xsl
index e9d130e32..14016ca75 100644
--- a/config/xsl/loadreport/events.xsl
+++ b/config/xsl/loadreport/events.xsl
@@ -33,11 +33,11 @@
-<! DOCTYPE html >
-
+
-
+
diff --git a/config/xsl/loadreport/external.xsl b/config/xsl/loadreport/external.xsl
index fd49ef88b..e8e293cab 100644
--- a/config/xsl/loadreport/external.xsl
+++ b/config/xsl/loadreport/external.xsl
@@ -28,10 +28,10 @@
-<! DOCTYPE html >
@@ -42,7 +42,9 @@
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/config/xsl/scorecard/sections/groups.xsl b/config/xsl/scorecard/sections/groups.xsl
new file mode 100644
index 000000000..e4d6ad19f
--- /dev/null
+++ b/config/xsl/scorecard/sections/groups.xsl
@@ -0,0 +1,121 @@
+
+
+
+
+
+
+
diff --git a/config/xsl/scorecard/sections/ratings.xsl b/config/xsl/scorecard/sections/ratings.xsl
new file mode 100644
index 000000000..40d897e39
--- /dev/null
+++ b/config/xsl/scorecard/sections/ratings.xsl
@@ -0,0 +1,79 @@
+
+
+
+
+
+
+
diff --git a/config/xsl/scorecard/sections/rule-checks.xsl b/config/xsl/scorecard/sections/rule-checks.xsl
new file mode 100644
index 000000000..5c2068278
--- /dev/null
+++ b/config/xsl/scorecard/sections/rule-checks.xsl
@@ -0,0 +1,128 @@
+
+
+
+
+
+
+
diff --git a/config/xsl/scorecard/sections/rules.xsl b/config/xsl/scorecard/sections/rules.xsl
new file mode 100644
index 000000000..e0503e4bd
--- /dev/null
+++ b/config/xsl/scorecard/sections/rules.xsl
@@ -0,0 +1,157 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/config/xsl/scorecard/sections/scorecard.xsl b/config/xsl/scorecard/sections/scorecard.xsl
new file mode 100644
index 000000000..e368671d0
--- /dev/null
+++ b/config/xsl/scorecard/sections/scorecard.xsl
@@ -0,0 +1,117 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/config/xsl/scorecard/sections/util.xsl b/config/xsl/scorecard/sections/util.xsl
new file mode 100644
index 000000000..f7f95ab18
--- /dev/null
+++ b/config/xsl/scorecard/sections/util.xsl
@@ -0,0 +1,45 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/config/xsl/scorecard/text/descriptions.xsl b/config/xsl/scorecard/text/descriptions.xsl
new file mode 100644
index 000000000..390680a2a
--- /dev/null
+++ b/config/xsl/scorecard/text/descriptions.xsl
@@ -0,0 +1,105 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/config/xsl/trendreport/actions.xsl b/config/xsl/trendreport/actions.xsl
index d3e3b4373..14c789105 100644
--- a/config/xsl/trendreport/actions.xsl
+++ b/config/xsl/trendreport/actions.xsl
@@ -41,7 +41,6 @@
-<! DOCTYPE html >
diff --git a/config/xsl/trendreport/custom-timers.xsl b/config/xsl/trendreport/custom-timers.xsl
index 46392a1eb..b1fa06eac 100644
--- a/config/xsl/trendreport/custom-timers.xsl
+++ b/config/xsl/trendreport/custom-timers.xsl
@@ -41,7 +41,6 @@
-<! DOCTYPE html >
diff --git a/config/xsl/trendreport/index.xsl b/config/xsl/trendreport/index.xsl
index 6b5255a24..2f0973673 100644
--- a/config/xsl/trendreport/index.xsl
+++ b/config/xsl/trendreport/index.xsl
@@ -41,7 +41,6 @@
-<! DOCTYPE html >
diff --git a/config/xsl/trendreport/page-load-timings.xsl b/config/xsl/trendreport/page-load-timings.xsl
index 5a4b05e95..09da62c85 100644
--- a/config/xsl/trendreport/page-load-timings.xsl
+++ b/config/xsl/trendreport/page-load-timings.xsl
@@ -41,7 +41,6 @@
-<! DOCTYPE html >
diff --git a/config/xsl/trendreport/requests.xsl b/config/xsl/trendreport/requests.xsl
index df8675b03..83b8f307c 100644
--- a/config/xsl/trendreport/requests.xsl
+++ b/config/xsl/trendreport/requests.xsl
@@ -43,7 +43,6 @@
-<! DOCTYPE html >
diff --git a/config/xsl/trendreport/transactions.xsl b/config/xsl/trendreport/transactions.xsl
index 1e099ac2f..7f2681f91 100644
--- a/config/xsl/trendreport/transactions.xsl
+++ b/config/xsl/trendreport/transactions.xsl
@@ -43,7 +43,6 @@
-<! DOCTYPE html >
diff --git a/doc/3rd-party-licenses/saxon-he/LICENSE b/doc/3rd-party-licenses/saxon-he/LICENSE
new file mode 100644
index 000000000..d0a1fa148
--- /dev/null
+++ b/doc/3rd-party-licenses/saxon-he/LICENSE
@@ -0,0 +1,373 @@
+Mozilla Public License Version 2.0
+==================================
+
+1. Definitions
+--------------
+
+1.1. "Contributor"
+ means each individual or legal entity that creates, contributes to
+ the creation of, or owns Covered Software.
+
+1.2. "Contributor Version"
+ means the combination of the Contributions of others (if any) used
+ by a Contributor and that particular Contributor's Contribution.
+
+1.3. "Contribution"
+ means Covered Software of a particular Contributor.
+
+1.4. "Covered Software"
+ means Source Code Form to which the initial Contributor has attached
+ the notice in Exhibit A, the Executable Form of such Source Code
+ Form, and Modifications of such Source Code Form, in each case
+ including portions thereof.
+
+1.5. "Incompatible With Secondary Licenses"
+ means
+
+ (a) that the initial Contributor has attached the notice described
+ in Exhibit B to the Covered Software; or
+
+ (b) that the Covered Software was made available under the terms of
+ version 1.1 or earlier of the License, but not also under the
+ terms of a Secondary License.
+
+1.6. "Executable Form"
+ means any form of the work other than Source Code Form.
+
+1.7. "Larger Work"
+ means a work that combines Covered Software with other material, in
+ a separate file or files, that is not Covered Software.
+
+1.8. "License"
+ means this document.
+
+1.9. "Licensable"
+ means having the right to grant, to the maximum extent possible,
+ whether at the time of the initial grant or subsequently, any and
+ all of the rights conveyed by this License.
+
+1.10. "Modifications"
+ means any of the following:
+
+ (a) any file in Source Code Form that results from an addition to,
+ deletion from, or modification of the contents of Covered
+ Software; or
+
+ (b) any new file in Source Code Form that contains any Covered
+ Software.
+
+1.11. "Patent Claims" of a Contributor
+ means any patent claim(s), including without limitation, method,
+ process, and apparatus claims, in any patent Licensable by such
+ Contributor that would be infringed, but for the grant of the
+ License, by the making, using, selling, offering for sale, having
+ made, import, or transfer of either its Contributions or its
+ Contributor Version.
+
+1.12. "Secondary License"
+ means either the GNU General Public License, Version 2.0, the GNU
+ Lesser General Public License, Version 2.1, the GNU Affero General
+ Public License, Version 3.0, or any later versions of those
+ licenses.
+
+1.13. "Source Code Form"
+ means the form of the work preferred for making modifications.
+
+1.14. "You" (or "Your")
+ means an individual or a legal entity exercising rights under this
+ License. For legal entities, "You" includes any entity that
+ controls, is controlled by, or is under common control with You. For
+ purposes of this definition, "control" means (a) the power, direct
+ or indirect, to cause the direction or management of such entity,
+ whether by contract or otherwise, or (b) ownership of more than
+ fifty percent (50%) of the outstanding shares or beneficial
+ ownership of such entity.
+
+2. License Grants and Conditions
+--------------------------------
+
+2.1. Grants
+
+Each Contributor hereby grants You a world-wide, royalty-free,
+non-exclusive license:
+
+(a) under intellectual property rights (other than patent or trademark)
+ Licensable by such Contributor to use, reproduce, make available,
+ modify, display, perform, distribute, and otherwise exploit its
+ Contributions, either on an unmodified basis, with Modifications, or
+ as part of a Larger Work; and
+
+(b) under Patent Claims of such Contributor to make, use, sell, offer
+ for sale, have made, import, and otherwise transfer either its
+ Contributions or its Contributor Version.
+
+2.2. Effective Date
+
+The licenses granted in Section 2.1 with respect to any Contribution
+become effective for each Contribution on the date the Contributor first
+distributes such Contribution.
+
+2.3. Limitations on Grant Scope
+
+The licenses granted in this Section 2 are the only rights granted under
+this License. No additional rights or licenses will be implied from the
+distribution or licensing of Covered Software under this License.
+Notwithstanding Section 2.1(b) above, no patent license is granted by a
+Contributor:
+
+(a) for any code that a Contributor has removed from Covered Software;
+ or
+
+(b) for infringements caused by: (i) Your and any other third party's
+ modifications of Covered Software, or (ii) the combination of its
+ Contributions with other software (except as part of its Contributor
+ Version); or
+
+(c) under Patent Claims infringed by Covered Software in the absence of
+ its Contributions.
+
+This License does not grant any rights in the trademarks, service marks,
+or logos of any Contributor (except as may be necessary to comply with
+the notice requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+No Contributor makes additional grants as a result of Your choice to
+distribute the Covered Software under a subsequent version of this
+License (see Section 10.2) or under the terms of a Secondary License (if
+permitted under the terms of Section 3.3).
+
+2.5. Representation
+
+Each Contributor represents that the Contributor believes its
+Contributions are its original creation(s) or it has sufficient rights
+to grant the rights to its Contributions conveyed by this License.
+
+2.6. Fair Use
+
+This License is not intended to limit any rights You have under
+applicable copyright doctrines of fair use, fair dealing, or other
+equivalents.
+
+2.7. Conditions
+
+Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
+in Section 2.1.
+
+3. Responsibilities
+-------------------
+
+3.1. Distribution of Source Form
+
+All distribution of Covered Software in Source Code Form, including any
+Modifications that You create or to which You contribute, must be under
+the terms of this License. You must inform recipients that the Source
+Code Form of the Covered Software is governed by the terms of this
+License, and how they can obtain a copy of this License. You may not
+attempt to alter or restrict the recipients' rights in the Source Code
+Form.
+
+3.2. Distribution of Executable Form
+
+If You distribute Covered Software in Executable Form then:
+
+(a) such Covered Software must also be made available in Source Code
+ Form, as described in Section 3.1, and You must inform recipients of
+ the Executable Form how they can obtain a copy of such Source Code
+ Form by reasonable means in a timely manner, at a charge no more
+ than the cost of distribution to the recipient; and
+
+(b) You may distribute such Executable Form under the terms of this
+ License, or sublicense it under different terms, provided that the
+ license for the Executable Form does not attempt to limit or alter
+ the recipients' rights in the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+You may create and distribute a Larger Work under terms of Your choice,
+provided that You also comply with the requirements of this License for
+the Covered Software. If the Larger Work is a combination of Covered
+Software with a work governed by one or more Secondary Licenses, and the
+Covered Software is not Incompatible With Secondary Licenses, this
+License permits You to additionally distribute such Covered Software
+under the terms of such Secondary License(s), so that the recipient of
+the Larger Work may, at their option, further distribute the Covered
+Software under the terms of either this License or such Secondary
+License(s).
+
+3.4. Notices
+
+You may not remove or alter the substance of any license notices
+(including copyright notices, patent notices, disclaimers of warranty,
+or limitations of liability) contained within the Source Code Form of
+the Covered Software, except that You may alter any license notices to
+the extent required to remedy known factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+You may choose to offer, and to charge a fee for, warranty, support,
+indemnity or liability obligations to one or more recipients of Covered
+Software. However, You may do so only on Your own behalf, and not on
+behalf of any Contributor. You must make it absolutely clear that any
+such warranty, support, indemnity, or liability obligation is offered by
+You alone, and You hereby agree to indemnify every Contributor for any
+liability incurred by such Contributor as a result of warranty, support,
+indemnity or liability terms You offer. You may include additional
+disclaimers of warranty and limitations of liability specific to any
+jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+---------------------------------------------------
+
+If it is impossible for You to comply with any of the terms of this
+License with respect to some or all of the Covered Software due to
+statute, judicial order, or regulation then You must: (a) comply with
+the terms of this License to the maximum extent possible; and (b)
+describe the limitations and the code they affect. Such description must
+be placed in a text file included with all distributions of the Covered
+Software under this License. Except to the extent prohibited by statute
+or regulation, such description must be sufficiently detailed for a
+recipient of ordinary skill to be able to understand it.
+
+5. Termination
+--------------
+
+5.1. The rights granted under this License will terminate automatically
+if You fail to comply with any of its terms. However, if You become
+compliant, then the rights granted under this License from a particular
+Contributor are reinstated (a) provisionally, unless and until such
+Contributor explicitly and finally terminates Your grants, and (b) on an
+ongoing basis, if such Contributor fails to notify You of the
+non-compliance by some reasonable means prior to 60 days after You have
+come back into compliance. Moreover, Your grants from a particular
+Contributor are reinstated on an ongoing basis if such Contributor
+notifies You of the non-compliance by some reasonable means, this is the
+first time You have received notice of non-compliance with this License
+from such Contributor, and You become compliant prior to 30 days after
+Your receipt of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent
+infringement claim (excluding declaratory judgment actions,
+counter-claims, and cross-claims) alleging that a Contributor Version
+directly or indirectly infringes any patent, then the rights granted to
+You by any and all Contributors for the Covered Software under Section
+2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all
+end user license agreements (excluding distributors and resellers) which
+have been validly granted by You or Your distributors under this License
+prior to termination shall survive termination.
+
+************************************************************************
+* *
+* 6. Disclaimer of Warranty *
+* ------------------------- *
+* *
+* Covered Software is provided under this License on an "as is" *
+* basis, without warranty of any kind, either expressed, implied, or *
+* statutory, including, without limitation, warranties that the *
+* Covered Software is free of defects, merchantable, fit for a *
+* particular purpose or non-infringing. The entire risk as to the *
+* quality and performance of the Covered Software is with You. *
+* Should any Covered Software prove defective in any respect, You *
+* (not any Contributor) assume the cost of any necessary servicing, *
+* repair, or correction. This disclaimer of warranty constitutes an *
+* essential part of this License. No use of any Covered Software is *
+* authorized under this License except under this disclaimer. *
+* *
+************************************************************************
+
+************************************************************************
+* *
+* 7. Limitation of Liability *
+* -------------------------- *
+* *
+* Under no circumstances and under no legal theory, whether tort *
+* (including negligence), contract, or otherwise, shall any *
+* Contributor, or anyone who distributes Covered Software as *
+* permitted above, be liable to You for any direct, indirect, *
+* special, incidental, or consequential damages of any character *
+* including, without limitation, damages for lost profits, loss of *
+* goodwill, work stoppage, computer failure or malfunction, or any *
+* and all other commercial damages or losses, even if such party *
+* shall have been informed of the possibility of such damages. This *
+* limitation of liability shall not apply to liability for death or *
+* personal injury resulting from such party's negligence to the *
+* extent applicable law prohibits such limitation. Some *
+* jurisdictions do not allow the exclusion or limitation of *
+* incidental or consequential damages, so this exclusion and *
+* limitation may not apply to You. *
+* *
+************************************************************************
+
+8. Litigation
+-------------
+
+Any litigation relating to this License may be brought only in the
+courts of a jurisdiction where the defendant maintains its principal
+place of business and such litigation shall be governed by laws of that
+jurisdiction, without reference to its conflict-of-law provisions.
+Nothing in this Section shall prevent a party's ability to bring
+cross-claims or counter-claims.
+
+9. Miscellaneous
+----------------
+
+This License represents the complete agreement concerning the subject
+matter hereof. If any provision of this License is held to be
+unenforceable, such provision shall be reformed only to the extent
+necessary to make it enforceable. Any law or regulation which provides
+that the language of a contract shall be construed against the drafter
+shall not be used to construe this License against a Contributor.
+
+10. Versions of the License
+---------------------------
+
+10.1. New Versions
+
+Mozilla Foundation is the license steward. Except as provided in Section
+10.3, no one other than the license steward has the right to modify or
+publish new versions of this License. Each version will be given a
+distinguishing version number.
+
+10.2. Effect of New Versions
+
+You may distribute the Covered Software under the terms of the version
+of the License under which You originally received the Covered Software,
+or under the terms of any subsequent version published by the license
+steward.
+
+10.3. Modified Versions
+
+If you create software not governed by this License, and you want to
+create a new license for such software, you may create and use a
+modified version of this License if you rename the license and remove
+any references to the name of the license steward (except to note that
+such modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary
+Licenses
+
+If You choose to distribute Source Code Form that is Incompatible With
+Secondary Licenses under the terms of this version of the License, the
+notice described in Exhibit B of this License must be attached.
+
+Exhibit A - Source Code Form License Notice
+-------------------------------------------
+
+ This Source Code Form is subject to the terms of the Mozilla Public
+ License, v. 2.0. If a copy of the MPL was not distributed with this
+ file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular
+file, then You may include the notice in a location (such as a LICENSE
+file in a relevant directory) where a recipient would be likely to look
+for such a notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - "Incompatible With Secondary Licenses" Notice
+---------------------------------------------------------
+
+ This Source Code Form is "Incompatible With Secondary Licenses", as
+ defined by the Mozilla Public License, v. 2.0.
diff --git a/doc/3rd-party-licenses/xalan/LICENSE b/doc/3rd-party-licenses/xalan/LICENSE
deleted file mode 100644
index d64569567..000000000
--- a/doc/3rd-party-licenses/xalan/LICENSE
+++ /dev/null
@@ -1,202 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/doc/3rd-party-licenses/xalan/NOTICE b/doc/3rd-party-licenses/xalan/NOTICE
deleted file mode 100644
index b462040a1..000000000
--- a/doc/3rd-party-licenses/xalan/NOTICE
+++ /dev/null
@@ -1,15 +0,0 @@
- ======================================================================================
- == NOTICE file corresponding to the section 4d of the Apache License, Version 2.0, ==
- == in this case for the Apache Xalan distribution. ==
- ======================================================================================
-
- This product includes software developed by
- The Apache Software Foundation (http://www.apache.org/).
-
- Portions of this software was originally based on the following:
-
- - software copyright (c) 1999-2002, Lotus Development Corporation., http://www.lotus.com.
- - software copyright (c) 2001-2002, Sun Microsystems., http://www.sun.com.
- - software copyright (c) 2003, IBM Corporation., http://www.ibm.com.
- - voluntary contributions made by Ovidiu Predescu (ovidiu@cup.hp.com) on behalf of the
- Apache Software Foundation and was originally developed at Hewlett Packard Company.
\ No newline at end of file
diff --git a/doc/3rd-party-licenses/xercesImpl/LICENSE b/doc/3rd-party-licenses/xercesImpl/LICENSE
deleted file mode 100644
index d64569567..000000000
--- a/doc/3rd-party-licenses/xercesImpl/LICENSE
+++ /dev/null
@@ -1,202 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/doc/3rd-party-licenses/xercesImpl/NOTICE b/doc/3rd-party-licenses/xercesImpl/NOTICE
deleted file mode 100644
index 19bebbd58..000000000
--- a/doc/3rd-party-licenses/xercesImpl/NOTICE
+++ /dev/null
@@ -1,17 +0,0 @@
- =========================================================================
- == NOTICE file corresponding to section 4(d) of the Apache License, ==
- == Version 2.0, in this case for the Apache Xerces Java distribution. ==
- =========================================================================
-
- Apache Xerces Java
- Copyright 1999-2018 The Apache Software Foundation
-
- This product includes software developed at
- The Apache Software Foundation (http://www.apache.org/).
-
- Portions of this software were originally based on the following:
- - software copyright (c) 1999, IBM Corporation., http://www.ibm.com.
- - software copyright (c) 1999, Sun Microsystems., http://www.sun.com.
- - voluntary contributions made by Paul Eng on behalf of the
- Apache Software Foundation that were originally developed at iClick, Inc.,
- software copyright (c) 1999.
\ No newline at end of file
diff --git a/doc/internal-doc/licenses/licenses.csv b/doc/internal-doc/licenses/licenses.csv
index adf230238..b7a6134e3 100644
--- a/doc/internal-doc/licenses/licenses.csv
+++ b/doc/internal-doc/licenses/licenses.csv
@@ -36,6 +36,7 @@ neko-htmlunit,Apache License 2.0,https://github.com/HtmlUnit/htmlunit-neko,
okhttp,Apache License 2.0,https://github.com/square/okhttp,
pegdown,Apache License 2.0,https://github.com/sirthias/pegdown,
progressbar,MIT License,https://github.com/ctongfei/progressbar,
+saxon-he,Mozilla Public License 2.0,https://www.saxonica.com/,
selenium-java,Apache License 2.0,https://selenium.dev/,
slf4j-log4j12,MIT License,http://www.slf4j.org/,
sparsebitset,Apache License 2.0,https://github.com/brettwooldridge/SparseBitSet,
@@ -44,8 +45,6 @@ tyrus-container-grizzly-server,Eclipse Public License 2.0,https://github.com/ecl
webdav-servlet,Apache License 2.0,https://sourceforge.net/projects/webdav-servlet/,
webp-imageio,Apache License 2.0,https://github.com/darkxanter/webp-imageio,
websocket-client,Apache License 2.0,https://www.eclipse.org/jetty/,
-xalan,Apache License 2.0,https://xml.apache.org/xalan-j/,
-xerces,Apache License 2.0,http://xerces.apache.org/xerces2-j/,
xstream,BSD 3-Clause License,https://x-stream.github.io/,
,,,
bootstrap,MIT License,https://getbootstrap.com/,
diff --git a/pom.xml b/pom.xml
index 3c336fb6e..a8c799f8b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
com.xceptance
xlt
- 8.2.1
+ 8.3.0
jar
XLT
@@ -364,27 +364,6 @@
webp-imageio
0.3.3
-
- xalan
- xalan
- 2.7.3
-
-
- xerces
- xercesImpl
-
-
- xml-apis
- xml-apis
-
-
-
-
-
- xalan
- serializer
- 2.7.3
-
com.thoughtworks.xstream
xstream
@@ -482,6 +461,16 @@
jline
3.23.0
+
+ dev.harrel
+ json-schema
+ 1.5.2
+
+
+ net.sf.saxon
+ Saxon-HE
+ 12.5
+
@@ -540,6 +529,16 @@
pmd-java
6.53.0
test
+
+
+ net.sourceforge.saxon
+ saxon
+
+
+ net.sourceforge.saxon
+ saxon-dom
+
+
@@ -597,6 +596,12 @@
checkstyle
10.6.0
test
+
+
+ net.sf.saxon
+ Saxon-HE
+
+
diff --git a/samples/testsuite-posters/config/project.properties b/samples/testsuite-posters/config/project.properties
index 052a86654..3b6f4a35b 100644
--- a/samples/testsuite-posters/config/project.properties
+++ b/samples/testsuite-posters/config/project.properties
@@ -131,3 +131,10 @@ com.xceptance.xlt.propertiesInclude.1 = misc/
#
################################################################################
com.xceptance.xlt.propertiesInclude.30 = testcase-mapping.properties
+
+################################################################################
+#
+# Scorecard Rules
+#
+################################################################################
+com.xceptance.xlt.scorecard.config = scorecard-config.json
diff --git a/samples/testsuite-posters/config/scorecard-config.json b/samples/testsuite-posters/config/scorecard-config.json
new file mode 100644
index 000000000..22455c438
--- /dev/null
+++ b/samples/testsuite-posters/config/scorecard-config.json
@@ -0,0 +1,1215 @@
+{
+ "comment": "A first demo for auto-rating its resulting scorecard",
+ "version": 2,
+
+ "selectors" : [
+ {
+ "id" : "homepageP95",
+ "expression" : "max(//requests/request[matches(name, '^Homepage')]/percentiles/p95)",
+ "comment" : "Get us the max of all P95 of homepage like requests"
+ },
+ {
+ "id" : "homepageP99",
+ "expression" : "max(//requests/request[matches(name, '^Homepage')]/percentiles/p99)",
+ "comment" : "Get us the max of all P99 of homepage like requests"
+ },
+
+ {
+ "id" : "catalogP95",
+ "expression" : "max(//requests/request[matches(name, '^(SelectTopCategory|SelectCategory|ProductDetailView)')]/percentiles/p95)",
+ "comment" : "Get us the max of all P95 of catalog like requests"
+ },
+ {
+ "id" : "catalogP99",
+ "expression" : "max(//requests/request[matches(name, '^(SelectTopCategory|SelectCategory|ProductDetailView)')]/percentiles/p99)",
+ "comment" : "Get us the max of all P99 of catalog like requests"
+ },
+
+
+ {
+ "id" : "accountP95",
+ "expression" : "max(//requests/request[matches(name, '^(Login|Logout|Register|GoToRegistrationForm|GoToSignIn)')]/percentiles/p95)",
+ "comment" : "Get us the max of all P95 of account like requests"
+ },
+ {
+ "id" : "accountP99",
+ "expression" : "max(//requests/request[matches(name, '^(Login|Logout|Register|GoToRegistrationForm|GoToSignIn)')]/percentiles/p99)",
+ "comment" : "Get us the max of all P99 of account like requests"
+ },
+
+
+ {
+ "id" : "cartP95",
+ "expression" : "max(//requests/request[matches(name, '^(AddToCart|ViewCart)')]/percentiles/p95)",
+ "comment" : "Get us the max of all P95 of cart like requests"
+ },
+ {
+ "id" : "cartP99",
+ "expression" : "max(//requests/request[matches(name, '^(AddToCart|ViewCart)')]/percentiles/p99)",
+ "comment" : "Get us the max of all P99 of cart like requests"
+ },
+
+
+ {
+ "id" : "checkoutP95",
+ "expression" : "max(//requests/request[matches(name, '^(StartCheckout|EnterShippingAddress|EnterBillingAddress|EnterPaymentMethod)') and count > 50]/percentiles/p95)",
+ "comment" : "Get us the max of all P95 of checkout like requests, ensure that we don't hit PlaceOrder and have enough samples to consider it."
+ },
+ {
+ "id" : "checkoutP99",
+ "expression" : "max(//requests/request[matches(name, '^(StartCheckout|EnterShippingAddress|EnterBillingAddress|EnterPaymentMethod)') and count > 50]/percentiles/p99)",
+ "comment" : "Get us the max of all P99 of checkout like requests, ensure that we don't hit PlaceOrder and have enough samples to consider it."
+ },
+
+
+ {
+ "id" : "orderP95",
+ "expression" : "max(//requests/request[matches(name, '^(PlaceOrder)')]/percentiles/p95)",
+ "comment" : "Get us the max of all P95 of order like requests"
+ },
+ {
+ "id" : "orderP99",
+ "expression" : "max(//requests/request[matches(name, '^(PlaceOrder)')]/percentiles/p99)",
+ "comment" : "Get us the max of all P99 of order like requests"
+ },
+
+
+ {
+ "id" : "orderCount",
+ "expression" : "max(//requests/request[matches(name, '^(PlaceOrder)')]/count)",
+ "comment" : "Get us the order count"
+ },
+
+
+ {
+ "id" : "orderErrors",
+ "expression" : "sum(//actions/action[matches(name, '^(PlaceOrder)')]/errorPercentage)",
+ "comment" : "Get us the order percentage errors"
+ },
+ {
+ "id" : "transactionErrors",
+ "expression" : "/testreport/summary/transactions/errorPercentage",
+ "comment" : "Get transaction error percentage"
+ },
+ {
+ "id" : "actionErrors",
+ "expression" : "/testreport/summary/actions/errorPercentage",
+ "comment" : "Get action errors"
+ },
+ {
+ "id" : "requestErrors",
+ "expression" : "/testreport/summary/requests/errorPercentage",
+ "comment" : "Get requests errors"
+ }
+ ],
+
+ "rules" : [
+ {
+ "id" : "baseCpuCheck",
+ "name" : "Agent CPU Usage",
+ "description" : "Verify that the CPU usage of agents is low enough",
+ "failsTest" : false,
+ "enabled" : true,
+ "checks" : [
+ {
+ "selector" : "count(//agents/agent/totalCpuUsage/mean[number() > 60])",
+ "condition" : "= 0",
+ "comment" : "Check the mean to avoid consuming too much"
+ }
+ ],
+ "messages" : {
+ "success" : "Agent CPU usage is within limits",
+ "fail" : "Agent CPU usage might have influenced the result, apply caution",
+ },
+ "points" : 0
+ },
+ {
+ "id" : "critialCpuCheck",
+ "name" : "Max Agent CPU Usage",
+ "description" : "Verify that the CPU usage was never too high",
+ "failsTest" : true,
+ "enabled" : true,
+ "checks" : [
+ {
+ "selector" : "max(//agents/agent/totalCpuUsage/max)",
+ "condition" : "< 95",
+ "comment" : "Check the max to avoid influencing the test result"
+ },
+ ],
+ "messages" : {
+ "success" : "Agent CPU usage is within limits",
+ "fail" : "Max CPU usage was too high for a moment",
+ },
+ "points" : 0
+ },
+ {
+ "id" : "wastingResources",
+ "name" : "Agent Wasting",
+ "description" : "Check that we use our machines wisely",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selector" : "count(//agents/agent/totalCpuUsage/mean[number() < 25])",
+ "condition" : "= 0",
+ "comment" : "Check the mean to avoid consuming too little"
+ },
+ ],
+ "messages" : {
+ "success" : "All agents seem to be utilized well enough",
+ "fail" : "You might have too many ",
+ },
+ "points" : 0
+ },
+
+ {
+ "id" : "fallThroughF",
+ "name" : "F",
+ "description" : "F",
+ "failsTest" : true,
+ "failsOn" : "PASSED",
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ ],
+ "messages" : {
+ "success" : "F",
+ "fail" : "",
+ },
+ "points" : 0
+ },
+
+
+ {
+ "id" : "homepageAplus",
+ "name" : "Homepage A+",
+ "description" : "Homepage A+ rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "homepageP95",
+ "condition" : "<= 100",
+ },
+ {
+ "selectorId" : "homepageP99",
+ "condition" : "<= 250",
+ },
+ ],
+ "messages" : {
+ "success" : "A+",
+ "fail" : "",
+ },
+ "points" : 12
+ },
+ {
+ "id" : "homepageA",
+ "name" : "Homepage A",
+ "description" : "Homepage A rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "homepageP95",
+ "condition" : "<= 250",
+ },
+ {
+ "selectorId" : "homepageP99",
+ "condition" : "<= 750",
+ },
+ ],
+ "messages" : {
+ "success" : "A",
+ "fail" : "",
+ },
+ "points" : 11
+ },
+ {
+ "id" : "homepageB",
+ "name" : "Homepage B",
+ "description" : "Homepage B rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "homepageP95",
+ "condition" : "<= 500",
+ },
+ {
+ "selectorId" : "homepageP99",
+ "condition" : "<= 1500",
+ },
+ ],
+ "messages" : {
+ "success" : "B",
+ "fail" : "",
+ },
+ "points" : 10
+ },
+ {
+ "id" : "homepageC",
+ "name" : "Homepage C",
+ "description" : "Homepage C rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "homepageP95",
+ "condition" : "<= 1000",
+ },
+ {
+ "selectorId" : "homepageP99",
+ "condition" : "<= 3000",
+ },
+ ],
+ "messages" : {
+ "success" : "C",
+ "fail" : "",
+ },
+ "points" : 6
+ },
+ {
+ "id" : "homepageD",
+ "name" : "Homepage D",
+ "description" : "Homepage D rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "homepageP95",
+ "condition" : "<= 2000",
+ },
+ {
+ "selectorId" : "homepageP99",
+ "condition" : "<= 7000",
+ },
+ ],
+ "messages" : {
+ "success" : "D",
+ "fail" : "",
+ },
+ "points" : 2
+ },
+ {
+ "id" : "homepageF",
+ "name" : "Homepage F",
+ "description" : "Homepage F rating criteria, this is for information only, we don't need that here, rather we have a generic F.",
+ "failsTest" : true,
+ "enabled" : false,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "homepageP95",
+ "condition" : "> 2000",
+ },
+ {
+ "selectorId" : "homepageP99",
+ "condition" : "> 7000",
+ },
+ ],
+ "messages" : {
+ "success" : "F",
+ "fail" : "",
+ },
+ "points" : 0
+ },
+
+
+ {
+ "id" : "catalogAplus",
+ "name" : "Catalog A+",
+ "description" : "Catalog A+ rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "catalogP95",
+ "condition" : "<= 100",
+ },
+ {
+ "selectorId" : "catalogP99",
+ "condition" : "<= 250",
+ },
+ ],
+ "messages" : {
+ "success" : "A+",
+ "fail" : "",
+ },
+ "points" : 12
+ },
+ {
+ "id" : "catalogA",
+ "name" : "Catalog A",
+ "description" : "Catalog A rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "catalogP95",
+ "condition" : "<= 250",
+ },
+ {
+ "selectorId" : "catalogP99",
+ "condition" : "<= 750",
+ },
+ ],
+ "messages" : {
+ "success" : "A",
+ "fail" : "",
+ },
+ "points" : 11
+ },
+ {
+ "id" : "catalogB",
+ "name" : "Catalog B",
+ "description" : "Catalog B rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "catalogP95",
+ "condition" : "<= 500",
+ },
+ {
+ "selectorId" : "catalogP99",
+ "condition" : "<= 1500",
+ },
+ ],
+ "messages" : {
+ "success" : "B",
+ "fail" : "",
+ },
+ "points" : 10
+ },
+ {
+ "id" : "catalogC",
+ "name" : "Catalog C",
+ "description" : "Catalog C rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "catalogP95",
+ "condition" : "<= 1000",
+ },
+ {
+ "selectorId" : "catalogP99",
+ "condition" : "<= 3000",
+ },
+ ],
+ "messages" : {
+ "success" : "C",
+ "fail" : "",
+ },
+ "points" : 6
+ },
+ {
+ "id" : "catalogD",
+ "name" : "Catalog D",
+ "description" : "Catalog D rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "catalogP95",
+ "condition" : "<= 2000",
+ },
+ {
+ "selectorId" : "catalogP99",
+ "condition" : "<= 7000",
+ },
+ ],
+ "messages" : {
+ "success" : "D",
+ "fail" : "",
+ },
+ "points" : 2
+ },
+ {
+ "id" : "catalogF",
+ "name" : "Catalog F",
+ "description" : "Catalog F rating criteria, just for information, a generic F rule wil catch it all.",
+ "failsTest" : true,
+ "enabled" : false,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "catalogP95",
+ "condition" : "> 2000",
+ },
+ {
+ "selectorId" : "catalogP99",
+ "condition" : "> 7000",
+ },
+ ],
+ "messages" : {
+ "success" : "F",
+ "fail" : "",
+ },
+ "points" : 0
+ },
+
+
+ {
+ "id" : "cartAplus",
+ "name" : "Cart A+",
+ "description" : "Cart A+ rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "cartP95",
+ "condition" : "<= 100",
+ },
+ {
+ "selectorId" : "cartP99",
+ "condition" : "<= 250",
+ },
+ ],
+ "messages" : {
+ "success" : "A+",
+ "fail" : "",
+ },
+ "points" : 12
+ },
+ {
+ "id" : "cartA",
+ "name" : "Cart A",
+ "description" : "Cart A rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "cartP95",
+ "condition" : "<= 250",
+ },
+ {
+ "selectorId" : "cartP99",
+ "condition" : "<= 750",
+ },
+ ],
+ "messages" : {
+ "success" : "A",
+ "fail" : "",
+ },
+ "points" : 11
+ },
+ {
+ "id" : "cartB",
+ "name" : "Cart B",
+ "description" : "Cart B rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "cartP95",
+ "condition" : "<= 500",
+ },
+ {
+ "selectorId" : "cartP99",
+ "condition" : "<= 1500",
+ },
+ ],
+ "messages" : {
+ "success" : "B",
+ "fail" : "",
+ },
+ "points" : 10
+ },
+ {
+ "id" : "cartC",
+ "name" : "Cart C",
+ "description" : "Cart C rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "cartP95",
+ "condition" : "<= 1000",
+ },
+ {
+ "selectorId" : "cartP99",
+ "condition" : "<= 3000",
+ },
+ ],
+ "messages" : {
+ "success" : "C",
+ "fail" : "",
+ },
+ "points" : 6
+ },
+ {
+ "id" : "cartD",
+ "name" : "Cart D",
+ "description" : "Cart D rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "cartP95",
+ "condition" : "<= 2000",
+ },
+ {
+ "selectorId" : "cartP99",
+ "condition" : "<= 7000",
+ },
+ ],
+ "messages" : {
+ "success" : "D",
+ "fail" : "",
+ },
+ "points" : 2
+ },
+ {
+ "id" : "cartF",
+ "name" : "Cart F",
+ "description" : "Cart F rating criteria, just for information, a generic F rule wil catch it all.",
+ "failsTest" : true,
+ "enabled" : false,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "cartP95",
+ "condition" : "> 2000",
+ },
+ {
+ "selectorId" : "cartP99",
+ "condition" : "> 7000",
+ },
+ ],
+ "messages" : {
+ "success" : "F",
+ "fail" : "",
+ },
+ "points" : 0
+ },
+
+ {
+ "id" : "accountAplus",
+ "name" : "Account A+",
+ "description" : "Account A+ rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "accountP95",
+ "condition" : "<= 100",
+ },
+ {
+ "selectorId" : "accountP99",
+ "condition" : "<= 250",
+ },
+ ],
+ "messages" : {
+ "success" : "A+",
+ "fail" : "",
+ },
+ "points" : 12
+ },
+ {
+ "id" : "accountA",
+ "name" : "Account A",
+ "description" : "Account A rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "accountP95",
+ "condition" : "<= 250",
+ },
+ {
+ "selectorId" : "accountP99",
+ "condition" : "<= 750",
+ },
+ ],
+ "messages" : {
+ "success" : "A",
+ "fail" : "",
+ },
+ "points" : 11
+ },
+ {
+ "id" : "accountB",
+ "name" : "Account B",
+ "description" : "Account B rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "accountP95",
+ "condition" : "<= 500",
+ },
+ {
+ "selectorId" : "accountP99",
+ "condition" : "<= 1500",
+ },
+ ],
+ "messages" : {
+ "success" : "B",
+ "fail" : "",
+ },
+ "points" : 10
+ },
+ {
+ "id" : "accountC",
+ "name" : "Account C",
+ "description" : "Account C rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "accountP95",
+ "condition" : "<= 1000",
+ },
+ {
+ "selectorId" : "accountP99",
+ "condition" : "<= 3000",
+ },
+ ],
+ "messages" : {
+ "success" : "C",
+ "fail" : "",
+ },
+ "points" : 6
+ },
+ {
+ "id" : "accountD",
+ "name" : "Account D",
+ "description" : "Account D rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "accountP95",
+ "condition" : "<= 2000",
+ },
+ {
+ "selectorId" : "accountP99",
+ "condition" : "<= 7000",
+ },
+ ],
+ "messages" : {
+ "success" : "D",
+ "fail" : "",
+ },
+ "points" : 2
+ },
+ {
+ "id" : "accountF",
+ "name" : "Account F",
+ "description" : "Account F rating criteria, just for information, a generic F rule wil catch it all.",
+ "failsTest" : true,
+ "enabled" : false,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "accountP95",
+ "condition" : "> 2000",
+ },
+ {
+ "selectorId" : "accountP99",
+ "condition" : "> 7000",
+ },
+ ],
+ "messages" : {
+ "success" : "F",
+ "fail" : "",
+ },
+ "points" : 0
+ },
+
+ {
+ "id" : "checkoutAplus",
+ "name" : "Checkout A+",
+ "description" : "Checkout A+ rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "checkoutP95",
+ "condition" : "<= 100",
+ },
+ {
+ "selectorId" : "checkoutP99",
+ "condition" : "<= 250",
+ },
+ ],
+ "messages" : {
+ "success" : "A+",
+ "fail" : "",
+ },
+ "points" : 12
+ },
+ {
+ "id" : "checkoutA",
+ "name" : "Checkout A",
+ "description" : "Checkout A rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "checkoutP95",
+ "condition" : "<= 250",
+ },
+ {
+ "selectorId" : "checkoutP99",
+ "condition" : "<= 750",
+ },
+ ],
+ "messages" : {
+ "success" : "A",
+ "fail" : "",
+ },
+ "points" : 11
+ },
+ {
+ "id" : "checkoutB",
+ "name" : "Checkout B",
+ "description" : "Checkout B rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "checkoutP95",
+ "condition" : "<= 500",
+ },
+ {
+ "selectorId" : "checkoutP99",
+ "condition" : "<= 1500",
+ },
+ ],
+ "messages" : {
+ "success" : "B",
+ "fail" : "",
+ },
+ "points" : 10
+ },
+ {
+ "id" : "checkoutC",
+ "name" : "Checkout C",
+ "description" : "Checkout C rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "checkoutP95",
+ "condition" : "<= 1000",
+ },
+ {
+ "selectorId" : "checkoutP99",
+ "condition" : "<= 3000",
+ },
+ ],
+ "messages" : {
+ "success" : "C",
+ "fail" : "",
+ },
+ "points" : 6
+ },
+ {
+ "id" : "checkoutD",
+ "name" : "Checkout D",
+ "description" : "Checkout D rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "checkoutP95",
+ "condition" : "<= 2000",
+ },
+ {
+ "selectorId" : "checkoutP99",
+ "condition" : "<= 7000",
+ },
+ ],
+ "messages" : {
+ "success" : "D",
+ "fail" : "",
+ },
+ "points" : 2
+ },
+ {
+ "id" : "checkoutF",
+ "name" : "Checkout F",
+ "description" : "Checkout F rating criteria, just for information, a generic F rule wil catch it all.",
+ "failsTest" : true,
+ "enabled" : false,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "checkoutP95",
+ "condition" : "> 2000",
+ },
+ {
+ "selectorId" : "checkoutP99",
+ "condition" : "> 7000",
+ },
+ ],
+ "messages" : {
+ "success" : "F",
+ "fail" : "",
+ },
+ "points" : 0
+ },
+
+
+ {
+ "id" : "orderAplus",
+ "name" : "Order A+",
+ "description" : "Order A+ rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "orderP95",
+ "condition" : "<= 100",
+ },
+ {
+ "selectorId" : "orderP99",
+ "condition" : "<= 250",
+ },
+ ],
+ "messages" : {
+ "success" : "A+",
+ "fail" : "",
+ },
+ "points" : 12
+ },
+ {
+ "id" : "orderA",
+ "name" : "Order A",
+ "description" : "Order A rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "orderP95",
+ "condition" : "<= 250",
+ },
+ {
+ "selectorId" : "orderP99",
+ "condition" : "<= 750",
+ },
+ ],
+ "messages" : {
+ "success" : "A",
+ "fail" : "",
+ },
+ "points" : 11
+ },
+ {
+ "id" : "orderB",
+ "name" : "Order B",
+ "description" : "Order B rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "orderP95",
+ "condition" : "<= 500",
+ },
+ {
+ "selectorId" : "orderP99",
+ "condition" : "<= 1500",
+ },
+ ],
+ "messages" : {
+ "success" : "B",
+ "fail" : "",
+ },
+ "points" : 10
+ },
+ {
+ "id" : "orderC",
+ "name" : "Order C",
+ "description" : "Order C rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "orderP95",
+ "condition" : "<= 1000",
+ },
+ {
+ "selectorId" : "orderP99",
+ "condition" : "<= 3000",
+ },
+ ],
+ "messages" : {
+ "success" : "C",
+ "fail" : "",
+ },
+ "points" : 6
+ },
+ {
+ "id" : "orderD",
+ "name" : "Order D",
+ "description" : "Order D rating criteria",
+ "failsTest" : false,
+ "enabled" : true,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "orderP95",
+ "condition" : "<= 2000",
+ },
+ {
+ "selectorId" : "orderP99",
+ "condition" : "<= 7000",
+ },
+ ],
+ "messages" : {
+ "success" : "D",
+ "fail" : "a",
+ },
+ "points" : 2
+ },
+ {
+ "id" : "orderF",
+ "name" : "Order F",
+ "description" : "Order F rating criteria, just for information, a generic F rule wil catch it all.",
+ "failsTest" : true,
+ "enabled" : false,
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "orderP95",
+ "condition" : "> 2000",
+ },
+ {
+ "selectorId" : "orderP99",
+ "condition" : "> 7000",
+ },
+ ],
+ "messages" : {
+ "success" : "F",
+ "fail" : "a",
+ },
+ "points" : 0
+ },
+
+ {
+ "id" : "orderErrorsRule",
+ "name" : "Order Errors",
+ "description" : "Ensure that only few orders failed",
+ "failsTest" : true,
+ "enabled" : true,
+ "failsOn" : "NOTPASSED",
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "orderErrors",
+ "condition" : "< 1.0 ",
+ },
+ ],
+ "messages" : {
+ "success" : "Order failure rate is ok",
+ "fail" : "Too many orders failed",
+ },
+ "points" : 5
+ },
+ {
+ "id" : "requestErrorsRule",
+ "name" : "Request Errors",
+ "description" : "Request error rate",
+ "failsTest" : true,
+ "enabled" : true,
+ "failsOn" : "NOTPASSED",
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "requestErrors",
+ "condition" : "< 0.5 ",
+ },
+ ],
+ "messages" : {
+ "success" : "Request failure rate is ok",
+ "fail" : "Too many requests failed",
+ },
+ "points" : 5
+ },
+ {
+ "id" : "actionErrorsRule",
+ "name" : "Action Errors",
+ "description" : "Action error rate",
+ "failsTest" : true,
+ "enabled" : true,
+ "failsOn" : "NOTPASSED",
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "actionErrors",
+ "condition" : "< 0.5 ",
+ },
+ ],
+ "messages" : {
+ "success" : "Action failure rate is ok",
+ "fail" : "Too many actions failed",
+ },
+ "points" : 5
+ },
+ {
+ "id" : "transactionErrorsRule",
+ "name" : "Transaction Errors",
+ "description" : "Transaction error rate",
+ "failsTest" : true,
+ "enabled" : true,
+ "failsOn" : "NOTPASSED",
+ "negateResult" : false,
+ "checks" : [
+ {
+ "selectorId" : "transactionErrors",
+ "condition" : "< 1.0 ",
+ },
+ ],
+ "messages" : {
+ "success" : "Transaction failure rate is ok",
+ "fail" : "Too many transactions failed",
+ },
+ "points" : 5
+ },
+ ],
+
+ "groups" : [
+ {
+ "id" : "CPUs",
+ "name" : "Agent CPU Usage",
+ "description" : "Verify that the CPU usage of agents is ok",
+ "failsTest" : true,
+ "mode" : "allPassed",
+ "enabled" : true,
+ "rules" : ["baseCpuCheck", "critialCpuCheck", "wastingResources"],
+ "messages" : {
+ "success" : "All CPU metrics are ok",
+ "fail" : "Verify the test setup, because some CPU metrics are off",
+ },
+ },
+ {
+ "id" : "Homepage",
+ "name" : "Homepage Rating",
+ "description" : "Rates the homepage",
+ "failsTest" : false,
+ "mode" : "firstPassed",
+ "enabled" : true,
+ "rules" : ["homepageAplus", "homepageA", "homepageB", "homepageC", "homepageD", "homepageF", "fallThroughF"],
+ "messages" : {
+ "success" : "Homepage Success",
+ "fail" : "Homepage fail",
+ },
+ },
+ {
+ "id" : "Catalog",
+ "enabled": true,
+ "name" : "Catalog Rating",
+ "description" : "Rates the catalog",
+ "failsTest" : false,
+ "mode" : "firstPassed",
+ "rules" : ["catalogAplus", "catalogA", "catalogB", "catalogC", "catalogD", "catalogF", "fallThroughF"],
+ },
+ {
+ "id" : "Cart",
+ "enabled": true,
+ "name" : "Cart Rating",
+ "description" : "Rates the cart",
+ "failsTest" : false,
+ "mode" : "firstPassed",
+ "rules" : ["cartAplus", "cartA", "cartB", "cartC", "cartD", "cartF", "fallThroughF"],
+ },
+ {
+ "id" : "Account",
+ "enabled": true,
+ "name" : "Account Rating",
+ "description" : "Rates the account",
+ "failsTest" : false,
+ "mode" : "firstPassed",
+ "rules" : ["accountAplus", "accountA", "accountB", "accountC", "accountD", "accountF", "fallThroughF"],
+ }, {
+ "id" : "Checkout",
+ "enabled": true,
+ "name" : "Checkout Rating",
+ "description" : "Rates the checkout",
+ "failsTest" : false,
+ "mode" : "firstPassed",
+ "rules" : ["checkoutAplus", "checkoutA", "checkoutB", "checkoutC", "checkoutD", "checkoutF", "fallThroughF"],
+ },
+ {
+ "id" : "Order",
+ "enabled": true,
+ "name" : "Order Rating",
+ "description" : "Rates the ordering",
+ "failsTest" : false,
+ "mode" : "firstPassed",
+ "rules" : ["orderAplus", "orderA", "orderB", "orderC", "orderD", "orderF", "fallThroughF"],
+ },
+ {
+ "id" : "Errors",
+ "enabled": true,
+ "name" : "Error Check",
+ "description" : "Error occurrences",
+ "failsTest" : true,
+ "mode" : "allPassed",
+ "rules" : ["requestErrorsRule", "actionErrorsRule", "transactionErrorsRule", "orderErrorsRule"],
+ "messages" : {
+ "success" : "Error rates seem low enough",
+ "fail" : "Too many errors in some sectors",
+ },
+ },
+ ],
+
+ "ratings" : [
+ {
+ "id": "poor",
+ "name": "Poor",
+ "enabled": true,
+ "description": "Load test performed poorly",
+ "value": 50.0,
+ "failsTest" : false
+ },
+ {
+ "id": "ok",
+ "name": "Ok",
+ "enabled": true,
+ "description": "Load test result likely valid",
+ "value": 80.0,
+ "failsTest" : false
+ },
+ {
+ "id": "success",
+ "name": "Success",
+ "enabled": true,
+ "description": "Load test performed nicely",
+ "value": 100.0,
+ "failsTest" : false
+ }
+ ]
+}
diff --git a/src/main/java/com/xceptance/xlt/agentcontroller/AgentControllerImpl.java b/src/main/java/com/xceptance/xlt/agentcontroller/AgentControllerImpl.java
index 33bf6b1a9..63c7dcbb8 100644
--- a/src/main/java/com/xceptance/xlt/agentcontroller/AgentControllerImpl.java
+++ b/src/main/java/com/xceptance/xlt/agentcontroller/AgentControllerImpl.java
@@ -1090,7 +1090,8 @@ public String archiveTestConfig()
final IOFileFilter propertiesFilesFilter = FileFilterUtils.suffixFileFilter(XltConstants.PROPERTY_FILE_EXTENSION);
final IOFileFilter cfgFilesFilter = FileFilterUtils.suffixFileFilter(XltConstants.CFG_FILE_EXTENSION);
final IOFileFilter xmlFilesFilter = FileFilterUtils.suffixFileFilter(XltConstants.XML_FILE_EXTENSION);
- final IOFileFilter extensionFilter = FileFilterUtils.or(propertiesFilesFilter, cfgFilesFilter, xmlFilesFilter);
+ final IOFileFilter jsonFilesFilter = FileFilterUtils.suffixFileFilter(XltConstants.JSON_FILE_EXTENSION);
+ final IOFileFilter extensionFilter = FileFilterUtils.or(propertiesFilesFilter, cfgFilesFilter, xmlFilesFilter, jsonFilesFilter);
final IOFileFilter filter = FileFilterUtils.and(FileFileFilter.INSTANCE, extensionFilter);
ZipOutputStream out = null;
diff --git a/src/main/java/com/xceptance/xlt/common/XltConstants.java b/src/main/java/com/xceptance/xlt/common/XltConstants.java
index 6edd8d835..becfa27cb 100644
--- a/src/main/java/com/xceptance/xlt/common/XltConstants.java
+++ b/src/main/java/com/xceptance/xlt/common/XltConstants.java
@@ -79,6 +79,11 @@ private XltConstants()
*/
public static final String XML_FILE_EXTENSION = ".xml";
+ /**
+ * The extension of JSON files, such as my-config.json.
+ */
+ public static final String JSON_FILE_EXTENSION = ".json";
+
/**
* The name of the system property which holds the agent configuration directory.
*/
@@ -460,4 +465,33 @@ private XltConstants()
* The name if we want to clean the user info
*/
public static final String PROP_REMOVE_USERINFO_FROM_REQUEST_URL = XltConstants.XLT_PACKAGE_PATH + ".results.data.request.removeUserInfoFromURL";
+
+ /*
+ * Scorecard configuration values
+ */
+
+ /**
+ * The property name for path to scorecard configuration file.
+ */
+ public static final String SCORECARD_CONFIG_FILE_PROPERTY = XLT_PACKAGE_PATH + ".scorecard.config";
+
+ /**
+ * The XML file name for scorecard.
+ */
+ public static final String SCORECARD_REPORT_XML_FILENAME = "scorecard.xml";
+
+ /**
+ * The path of the directory with all XSL files for scorecard rendering.
+ */
+ public static final String SCORECARD_REPORT_XSL_PATH = "xsl/scorecard";
+
+ /**
+ * The name of the XSL file for rendering the scorecard report page.
+ */
+ public static final String SCORECARD_REPORT_XSL_FILENAME = "index.xsl";
+
+ /**
+ * The name of the HTML output file for rendering the scorecard report page.
+ */
+ public static final String SCORECARD_REPORT_HTML_FILENAME = "scorecard.html";
}
diff --git a/src/main/java/com/xceptance/xlt/engine/resultbrowser/ErrorCounter.java b/src/main/java/com/xceptance/xlt/engine/resultbrowser/ErrorCounter.java
index a0683597a..f99a49183 100644
--- a/src/main/java/com/xceptance/xlt/engine/resultbrowser/ErrorCounter.java
+++ b/src/main/java/com/xceptance/xlt/engine/resultbrowser/ErrorCounter.java
@@ -141,8 +141,8 @@ private ErrorCounter(final XltPropertiesImpl properties)
}
catch (final Exception e)
{
- XltLogger.runTimeLogger.warn(String.format("The value '%s' of property '%s' cannot be resolved or parsed as time period. Disabling error limiter reset interval, keeping count limitation.",
- intervalString, COUNTER_RESET_INTERVAL_PROPERTY));
+ XltLogger.runTimeLogger.warn("The value '{}' of property '{}' cannot be resolved or parsed as time period. Disabling error limiter reset interval, keeping count limitation.",
+ intervalString, COUNTER_RESET_INTERVAL_PROPERTY);
}
// interval in milliseconds
diff --git a/src/main/java/com/xceptance/xlt/report/ReportGenerator.java b/src/main/java/com/xceptance/xlt/report/ReportGenerator.java
index 19457a480..eae7f749f 100644
--- a/src/main/java/com/xceptance/xlt/report/ReportGenerator.java
+++ b/src/main/java/com/xceptance/xlt/report/ReportGenerator.java
@@ -21,6 +21,7 @@
import java.util.Date;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import org.apache.commons.io.FileUtils;
@@ -30,6 +31,7 @@
import org.apache.commons.vfs2.FileName;
import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.NameScope;
import org.apache.commons.vfs2.Selectors;
import org.apache.commons.vfs2.VFS;
@@ -43,6 +45,8 @@
import com.xceptance.xlt.mastercontroller.TestCaseLoadProfileConfiguration;
import com.xceptance.xlt.mastercontroller.TestLoadProfileConfiguration;
import com.xceptance.xlt.report.external.ExternalReportGenerator;
+import com.xceptance.xlt.report.scorecard.Scorecard;
+import com.xceptance.xlt.report.scorecard.Evaluator;
import com.xceptance.xlt.report.util.ConcurrentUsersTable;
import com.xceptance.xlt.report.util.JFreeChartUtils;
import com.xceptance.xlt.report.util.ReportUtils;
@@ -101,7 +105,7 @@ public ReportGenerator(final FileObject inputDir, final File outputDir, final bo
final File overridePropertyFile, final Properties commandLineProperties, final String testCaseIncludePatternList,
final String testCaseExcludePatternList, final String agentIncludePatternList,
final String agentExcludePatternList)
- throws Exception
+ throws Exception
{
final FileObject configDir = inputDir.resolveFile(XltConstants.CONFIG_DIR_NAME);
@@ -156,8 +160,6 @@ public ReportGenerator(final FileObject inputDir, final File outputDir, final bo
this.outputDir = outputDir;
}
- // clean or create it
- ReportGenerator.ensureOutputDirAndClean(this.outputDir);
config.setReportDirectory(this.outputDir);
// configure the thread pool to be cpu count for now
@@ -169,30 +171,12 @@ public ReportGenerator(final FileObject inputDir, final File outputDir, final bo
// setup the report providers
reportProviders = new ArrayList();
- for (final Class c : config.getReportProviderClasses())
- {
- try
- {
- final ReportProvider processor = c.getDeclaredConstructor().newInstance();
- processor.setConfiguration(config);
-
- reportProviders.add(processor);
- }
- catch (final Throwable t)
- {
- final String message = String.format("Failed to instantiate and initialize report provider instance of class '%s': %s'",
- c.getCanonicalName(), t.getMessage());
- XltLogger.reportLogger.error(message, t);
- }
- }
-
repGen = new com.xceptance.xlt.report.external.ExternalReportGenerator();
}
/**
- * Ensure that output exists and is empty. We have that public here because we need it twice due to either
- * the dir coming in from external or is determining it when creating the report. Not really nice but
- * legacy.
+ * Ensure that output exists and is empty. We have that public here because we need it twice due to either the dir
+ * coming in from external or is determining it when creating the report. Not really nice but legacy.
*
* @throws IOException
*/
@@ -203,13 +187,13 @@ public static void ensureOutputDirAndClean(final File dir) throws IOException
*/
if (dir.exists() == false)
{
- XltLogger.reportLogger.info(String.format("Creating output directory: %s", dir));
+ XltLogger.reportLogger.info("Creating output directory: {}", dir);
FileUtils.forceMkdir(dir);
}
else
{
// clean output directory first -> Improvement #3243
- XltLogger.reportLogger.info(String.format("Cleaning output directory: %s", dir));
+ XltLogger.reportLogger.info("Cleaning output directory: {}", dir);
FileUtils.cleanDirectory(dir);
}
}
@@ -259,10 +243,29 @@ public void generateReport(final boolean noRampUp) throws Exception
*/
public void generateReport(final long fromTime, final long toTime, final long duration, final boolean noRampUp,
final boolean fromTimeRel, final boolean toTimeRel)
- throws Exception
+ throws Exception
{
try
{
+ // clean/create output directory first
+ ensureOutputDirAndClean(this.outputDir);
+
+ for (final Class c : config.getReportProviderClasses())
+ {
+ try
+ {
+ final ReportProvider processor = c.getDeclaredConstructor().newInstance();
+ processor.setConfiguration(config);
+
+ reportProviders.add(processor);
+ }
+ catch (final Throwable t)
+ {
+ XltLogger.reportLogger.error("Failed to instantiate and initialize report provider instance of class '{}'",
+ c.getCanonicalName(), t);
+ }
+ }
+
// read all log files and crunch the data
readLogs(fromTime, toTime, duration, noRampUp, fromTimeRel, toTimeRel);
@@ -272,14 +275,23 @@ public void generateReport(final long fromTime, final long toTime, final long du
// drop providers
reportProviders.clear();
+ // evaluate report if desired
+ final File scorecardXml = evaluateReport(xmlReport);
+
// create the html report
- transformReport(xmlReport, outputDir);
+ transformReport(xmlReport, outputDir, scorecardXml != null);
+
+ // create the report's Scorecard HTML (if evaluation took place)
+ if (scorecardXml != null)
+ {
+ transformScorecard(scorecardXml);
+ }
// output the path to the report either as file path (Win) or as clickable file URL
final File reportFile = new File(outputDir, "index.html");
final String reportPath = ReportUtils.toString(reportFile);
- XltLogger.reportLogger.info("Report: " + reportPath);
+ XltLogger.reportLogger.info("Report: {}", reportPath);
}
finally
{
@@ -389,9 +401,8 @@ private long excludeRampup(long fromTime)
}
else
{
- XltLogger.reportLogger.warn(
- String.format("PLEASE NOTE: Ramp-up could not be excluded since no value could be found for property '%s'.\n",
- XltConstants.LOAD_TEST_START_DATE));
+ XltLogger.reportLogger.warn("PLEASE NOTE: Ramp-up could not be excluded since no value could be found for property '{}'.",
+ XltConstants.LOAD_TEST_START_DATE);
}
return fromTime;
@@ -400,8 +411,10 @@ private long excludeRampup(long fromTime)
/**
* Processing of the log files within a defined time range
*
- * @param fromTime start time of the period to report
- * @param toTime end time of the period to report
+ * @param fromTime
+ * start time of the period to report
+ * @param toTime
+ * end time of the period to report
*/
private void read(final long fromTime, final long toTime)
{
@@ -409,13 +422,9 @@ private void read(final long fromTime, final long toTime)
final DataRecordFactory dataRecordFactory = new DataRecordFactory(config.getDataRecordClasses());
// read the logs
- final DataProcessor logReader = new DataProcessor(config,
- inputDir,
- dataRecordFactory,
- fromTime, toTime,
- reportProviders,
- testCaseIncludePatternList, testCaseExcludePatternList,
- agentIncludePatternList, agentExcludePatternList);
+ final DataProcessor logReader = new DataProcessor(config, inputDir, dataRecordFactory, fromTime, toTime, reportProviders,
+ testCaseIncludePatternList, testCaseExcludePatternList, agentIncludePatternList,
+ agentExcludePatternList);
logReader.readDataRecords();
XltLogger.reportLogger.info(Console.endSection());
@@ -466,9 +475,8 @@ private long recalculateOffsetTimeValue(final long offsetTimeValue, final long t
{
if (testStartDate == 0 || elapsedTime == 0)
{
- System.out.printf("PLEASE NOTE: The specified offset '" + offsetTimeValue +
- "' could not be used since no value could be found for properties '%1$s' and '%2$s'.\n",
- XltConstants.LOAD_TEST_START_DATE, XltConstants.LOAD_TEST_ELAPSED_TIME);
+ XltLogger.reportLogger.warn("PLEASE NOTE: The specified offset '{}' could not be used since no value could be found for properties '{}' and '{}'",
+ offsetTimeValue, XltConstants.LOAD_TEST_START_DATE, XltConstants.LOAD_TEST_ELAPSED_TIME);
}
else
{
@@ -480,9 +488,8 @@ private long recalculateOffsetTimeValue(final long offsetTimeValue, final long t
{
if (testStartDate == 0)
{
- System.out.printf("PLEASE NOTE: The specified offset '" + offsetTimeValue +
- "' could not be used since no value could be found for property '%s' .\n",
- XltConstants.LOAD_TEST_START_DATE);
+ XltLogger.reportLogger.warn("PLEASE NOTE: The specified offset '{}' could not be used since no value could be found for property '{}'",
+ offsetTimeValue, XltConstants.LOAD_TEST_START_DATE);
}
else
{
@@ -505,18 +512,18 @@ private void printStartAndEndTime(final long fromTime, final long toTime)
// only 'from' parameter is set
if (fromTime > 0 && toTime == Long.MAX_VALUE)
{
- XltLogger.reportLogger.info(String.format("Data start: %s", new Date(fromTime)));
+ XltLogger.reportLogger.info("Data start: {}", new Date(fromTime));
}
// only 'to' parameter is set
else if (fromTime == 0 && toTime != Long.MAX_VALUE)
{
- XltLogger.reportLogger.info(String.format("Data end: %s", new Date(toTime)));
+ XltLogger.reportLogger.info("Data end: {}", new Date(toTime));
}
// both parameter are set
else if (fromTime > 0 && toTime != Long.MAX_VALUE)
{
- XltLogger.reportLogger.info(String.format("Data start: %s", new Date(fromTime)));
- XltLogger.reportLogger.info(String.format("Data end : %s", new Date(toTime)));
+ XltLogger.reportLogger.info("Data start: {}", new Date(fromTime));
+ XltLogger.reportLogger.info("Data end : {}", new Date(toTime));
}
}
@@ -599,7 +606,7 @@ private void copyConfiguration(final File outputDir) throws FileSystemException
}
catch (FileSystemException e)
{
- XltLogger.reportLogger.error(String.format("Issue while copying original properties from %s", reportConfigDir.getPublicURIString()), e);
+ XltLogger.reportLogger.error("Issue while copying original properties from {}", reportConfigDir.getPublicURIString(), e);
}
}
else
@@ -621,7 +628,7 @@ private void copyConfiguration(final File outputDir) throws FileSystemException
* @throws Exception
* if anything goes wrong during transformation
*/
- public void transformReport(final File inputXmlFile, final File outputDir) throws Exception
+ public void transformReport(final File inputXmlFile, final File outputDir, final boolean scorecardPresent) throws Exception
{
XltLogger.reportLogger.info(Console.horizontalBar());
XltLogger.reportLogger.info(Console.startSection("Creating HTML Report..."));
@@ -656,6 +663,7 @@ public void transformReport(final File inputXmlFile, final File outputDir) throw
parameters.put("productName", ProductInformation.getProductInformation().getProductName());
parameters.put("productVersion", ProductInformation.getProductInformation().getVersion());
parameters.put("productUrl", ProductInformation.getProductInformation().getProductURL());
+ parameters.put("scorecardPresent", Boolean.valueOf(scorecardPresent));
// transform the report
final ReportTransformer reportTransformer = new ReportTransformer(outputFiles, styleSheetFiles, parameters);
@@ -664,7 +672,7 @@ public void transformReport(final File inputXmlFile, final File outputDir) throw
try
{
- XltLogger.reportLogger.info(String.format("XML data file: %s", inputXmlFile));
+ XltLogger.reportLogger.info("XML data file: {}", inputXmlFile);
// ok, we want to avoid high memory usage
TaskManager.getInstance().setMaximumThreadCount(1);
@@ -684,6 +692,51 @@ public void transformReport(final File inputXmlFile, final File outputDir) throw
}
}
+ /**
+ * Re-evaluates the load test report given as input and (re-)generates its Scorecard.
+ *
+ * @throws Exception
+ * thrown when evaluation of report or (re-)generation of its Scorecard failed
+ */
+ public void updateScorecard() throws Exception
+ {
+ try
+ {
+ inputDir.resolveFile(XltConstants.LOAD_REPORT_XML_FILENAME, NameScope.CHILD);
+ }
+ catch (FileSystemException fse)
+ {
+ XltLogger.reportLogger.error("Could not find '{}' in directory '{}'.", XltConstants.LOAD_REPORT_XML_FILENAME,
+ inputDir.getName().getPathDecoded());
+ return;
+ }
+
+ final FileObject targetDir = VFS.getManager().toFileObject(outputDir);
+ if (inputDir.getName() != targetDir.getName())
+ {
+ ensureOutputDirAndClean(outputDir);
+
+ targetDir.copyFrom(inputDir, Selectors.SELECT_ALL);
+ }
+
+ final File xmlReport = new File(outputDir, XltConstants.LOAD_REPORT_XML_FILENAME);
+
+ // evaluate test if desired
+ final File scorecardXml = evaluateReport(xmlReport);
+
+ // create the report's Scorecard HTML (if evaluation took place)
+ if (scorecardXml != null)
+ {
+ transformScorecard(scorecardXml);
+ }
+
+ // output the path to the report either as file path (Win) or as clickable file URL
+ final File reportFile = new File(outputDir, "index.html");
+ final String reportPath = ReportUtils.toString(reportFile);
+
+ XltLogger.reportLogger.info("Report: {}", reportPath);
+ }
+
/**
* Derives a directory name from the given input directory/archive file.
*
@@ -730,6 +783,130 @@ private String getResultsDirName(final FileObject input)
return inputDirName;
}
+ /**
+ * Evaluates the given test report XML file and writes its outcome to file named
+ * {@value XltConstants#SCORECARD_REPORT_XML_FILENAME}. N.B. No evaluation is done if there is no proper
+ * configuration file (see property {@value XltConstants#SCORECARD_CONFIG_FILE_PROPERTY}) configured.
+ *
+ * @param reportXMLFile
+ * the test report XML to evaluate
+ * @return the scorecard's XML file if evaluation took place, or {@code null} otherwise.
+ */
+ private File evaluateReport(final File reportXMLFile)
+ {
+ final String scorecardConfig = config.getStringProperty(XltConstants.SCORECARD_CONFIG_FILE_PROPERTY, null);
+ final File scorecardConfigFile = scorecardConfig != null ? new File(new File(outputDir, XltConstants.CONFIG_DIR_NAME),
+ scorecardConfig)
+ : null;
+ if (scorecardConfigFile != null)
+ {
+ XltLogger.reportLogger.debug("Evaluating test report using configuration file '{}'", scorecardConfigFile.getAbsolutePath());
+
+ final String errMessage = "Failed to evaluate test report";
+ try
+ {
+ final File scorecardXMLFile = new File(outputDir, XltConstants.SCORECARD_REPORT_XML_FILENAME);
+ final Evaluator evaluator = new Evaluator(scorecardConfigFile);
+ final Scorecard outcome = evaluator.evaluate(reportXMLFile);
+
+ final String error = outcome.result.getError();
+ if (StringUtils.isNotBlank(error))
+ {
+ XltLogger.reportLogger.error("{}: {}", errMessage, error);
+ }
+
+ evaluator.writeScorecardToFile(outcome, scorecardXMLFile);
+
+ return scorecardXMLFile;
+ }
+ catch (final Throwable t)
+ {
+ XltLogger.reportLogger.error(errMessage, t);
+ }
+ }
+
+ return null;
+ }
+
+ /**
+ * Transform the given scorecard XML file using the XSL stylesheet
+ * {@value XltConstants#SCORECARD_REPORT_XSL_FILENAME} that is expected to reside in report generator's
+ * configuration sub-directory {@value XltConstants#SCORECARD_REPORT_XSL_PATH}
+ *
+ * @param inputXmlFile
+ * the scorecard XMLfile to transform
+ * @throws Exception
+ * thrown when transformation failed for any reason
+ */
+ private void transformScorecard(final File inputXmlFile) throws Exception
+ {
+ XltLogger.reportLogger.info(Console.horizontalBar());
+ XltLogger.reportLogger.info(Console.startSection("Creating Scorecard..."));
+
+ final File styleSheetFile = new File(new File(config.getConfigDirectory(), XltConstants.SCORECARD_REPORT_XSL_PATH),
+ XltConstants.SCORECARD_REPORT_XSL_FILENAME);
+ final File outputFile = new File(outputDir, XltConstants.SCORECARD_REPORT_HTML_FILENAME);
+
+ // determine the name of the project from configuration
+ final String projectName;
+ {
+ final String projectNamePropValue = config.getStringProperty(XltConstants.PROJECT_NAME_PROPERTY, null);
+ projectName = StringUtils.trimToEmpty(projectNamePropValue);
+ }
+
+ // XTC specific parameters
+ final String organization, project, loadTestId, resultId, reportId;
+ {
+ final String organizationPropValue = config.getStringProperty("com.xceptance.xtc.organization", null);
+ organization = StringUtils.trimToEmpty(organizationPropValue);
+
+ final String projectPropValue = config.getStringProperty("com.xceptance.xtc.project", null);
+ project = StringUtils.trimToEmpty(projectPropValue);
+
+ final String loadTestIdPropValue = config.getStringProperty("com.xceptance.xtc.loadtest.run.id", null);
+ loadTestId = StringUtils.trimToEmpty(loadTestIdPropValue);
+
+ final String resultIdPropValue = config.getStringProperty("com.xceptance.xtc.loadtest.result.id", null);
+ resultId = StringUtils.trimToEmpty(resultIdPropValue);
+
+ final String reportIdPropValue = config.getStringProperty("com.xceptance.xtc.loadtest.report.id", null);
+ reportId = StringUtils.trimToEmpty(reportIdPropValue);
+ }
+
+ // create some dynamic parameters
+ final Map parameters = Map.of("productName", ProductInformation.getProductInformation().getProductName(),
+ "productVersion", ProductInformation.getProductInformation().getVersion(),
+ "productUrl", ProductInformation.getProductInformation().getProductURL(),
+ "projectName", projectName, "scorecardPresent", Boolean.TRUE, "xtcOrganization",
+ organization, "xtcProject", project, "xtcLoadTestId", loadTestId, "xtcResultId",
+ resultId, "xtcReportId", reportId);
+
+ // transform the report
+ final ReportTransformer reportTransformer = new ReportTransformer(List.of(outputFile), List.of(styleSheetFile), parameters);
+
+ final long start = TimerUtils.get().getStartTime();
+
+ try
+ {
+
+ // ok, we want to avoid high memory usage
+ TaskManager.getInstance().setMaximumThreadCount(1);
+
+ TaskManager.getInstance().startProgress("Creating");
+ reportTransformer.run(inputXmlFile, outputDir);
+
+ }
+ finally
+ {
+ // wait for any asynchronous task to complete
+ TaskManager.getInstance().waitForAllTasksToComplete();
+ TaskManager.getInstance().stopProgress();
+
+ XltLogger.reportLogger.info(String.format("...finished - %,d ms", TimerUtils.get().getElapsedTime(start)));
+ XltLogger.reportLogger.info(Console.endSection());
+ }
+ }
+
// TODO: Check if the similar method {@link TestLoadProfileConfiguration#getTotalRampUpPeriod()} is still needed.
static long computeRampUpOffset(final List profiles)
{
diff --git a/src/main/java/com/xceptance/xlt/report/XmlReportGenerator.java b/src/main/java/com/xceptance/xlt/report/XmlReportGenerator.java
index 8f100868d..842b05308 100644
--- a/src/main/java/com/xceptance/xlt/report/XmlReportGenerator.java
+++ b/src/main/java/com/xceptance/xlt/report/XmlReportGenerator.java
@@ -19,23 +19,16 @@
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
-import java.io.Writer;
import java.util.ArrayList;
import java.util.List;
import java.util.TimeZone;
-import org.apache.commons.text.StringEscapeUtils;
-
import com.thoughtworks.xstream.XStream;
import com.thoughtworks.xstream.converters.basic.DateConverter;
-import com.thoughtworks.xstream.core.util.QuickWriter;
-import com.thoughtworks.xstream.io.HierarchicalStreamWriter;
-import com.thoughtworks.xstream.io.naming.NameCoder;
-import com.thoughtworks.xstream.io.xml.DomDriver;
-import com.thoughtworks.xstream.io.xml.PrettyPrintWriter;
import com.xceptance.xlt.api.report.ReportCreator;
import com.xceptance.xlt.api.util.XltLogger;
import com.xceptance.xlt.common.XltConstants;
+import com.xceptance.xlt.report.util.xstream.SanitizingDomDriver;
/**
* Load test report generator.
@@ -103,37 +96,4 @@ private void saveTestReport(final TestReport testReport, final File xmlFile) thr
xstream.toXML(testReport, osw);
}
}
-
- /**
- * A custom {@link DomDriver} that uses a {@link SanitizingWriter} to write an XML file.
- */
- private static class SanitizingDomDriver extends DomDriver
- {
- @Override
- public HierarchicalStreamWriter createWriter(final Writer out)
- {
- return new SanitizingWriter(out, getNameCoder());
- }
- }
-
- /**
- * A custom {@link PrettyPrintWriter} that silently removes invalid XML 1.0 characters when writing text nodes.
- */
- private static class SanitizingWriter extends PrettyPrintWriter
- {
- public SanitizingWriter(final Writer writer, final NameCoder nameCoder)
- {
- super(writer, nameCoder);
- }
-
- @Override
- protected void writeText(final QuickWriter writer, final String text)
- {
- // escape special chars and remove invalid chars
- final String sanitizedText = StringEscapeUtils.escapeXml10(text);
-
- // don't call super.writeText() as this would escape the already escaped chars once more
- writer.write(sanitizedText);
- }
- }
}
diff --git a/src/main/java/com/xceptance/xlt/report/scorecard/Configuration.java b/src/main/java/com/xceptance/xlt/report/scorecard/Configuration.java
new file mode 100644
index 000000000..5316c4b57
--- /dev/null
+++ b/src/main/java/com/xceptance/xlt/report/scorecard/Configuration.java
@@ -0,0 +1,323 @@
+/*
+ * Copyright (c) 2005-2024 Xceptance Software Technologies GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.xceptance.xlt.report.scorecard;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.commons.lang3.StringUtils;
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+import com.thoughtworks.xstream.annotations.XStreamAsAttribute;
+import com.thoughtworks.xstream.annotations.XStreamConverter;
+import com.thoughtworks.xstream.converters.MarshallingContext;
+import com.thoughtworks.xstream.converters.UnmarshallingContext;
+import com.thoughtworks.xstream.converters.collections.MapConverter;
+import com.thoughtworks.xstream.io.HierarchicalStreamReader;
+import com.thoughtworks.xstream.io.HierarchicalStreamWriter;
+import com.thoughtworks.xstream.mapper.Mapper;
+
+public class Configuration
+{
+ @XStreamAsAttribute
+ private final int version;
+
+ @XStreamConverter(value = MappedValuesConverter.class)
+ private final Map selectors = new LinkedHashMap<>();
+
+ @XStreamConverter(value = MappedValuesConverter.class)
+ private final Map rules = new LinkedHashMap<>();
+
+ @XStreamConverter(value = MappedValuesConverter.class)
+ private final Map groups = new LinkedHashMap<>();
+
+ @XStreamConverter(value = MappedValuesConverter.class)
+ private final Map ratings = new LinkedHashMap<>();
+
+ public Configuration(final int version)
+ {
+ this.version = Math.max(1, version);
+ }
+
+ public void addSelector(final SelectorDefinition selector) throws ValidationException
+ {
+ final String selectorId = selector.getId();
+ validateId(selectorId);
+
+ selectors.put(selectorId, selector);
+ }
+
+ public boolean containsSelector(final String selectorId)
+ {
+ return selectors.containsKey(selectorId);
+ }
+
+ public SelectorDefinition getSelector(final String selectorId)
+ {
+ return selectors.get(selectorId);
+ }
+
+ public Collection getSelectors()
+ {
+ return Collections.unmodifiableCollection(selectors.values());
+ }
+
+ void addRule(final RuleDefinition rule) throws ValidationException
+ {
+ final String ruleId = rule.getId();
+ validateId(ruleId);
+
+ final List unknownSelectors = Stream.of(rule.getChecks()).map(RuleDefinition.Check::getSelectorId).filter(Objects::nonNull)
+ .filter(Predicate.not(this::containsSelector)).collect(Collectors.toList());
+ if (!unknownSelectors.isEmpty())
+ {
+ throw new ValidationException(String.format("Rule '%s' references unknown selector%s: %s", ruleId,
+ (unknownSelectors.size() > 1 ? "s" : ""), StringUtils.join(unknownSelectors)));
+ }
+
+ rules.put(ruleId, rule);
+ }
+
+ public boolean containsRule(final String ruleId)
+ {
+ return rules.containsKey(ruleId);
+ }
+
+ public Collection getRules()
+ {
+ return Collections.unmodifiableCollection(rules.values());
+ }
+
+ public RuleDefinition getRule(final String ruleId)
+ {
+ return rules.get(ruleId);
+ }
+
+ void addGroup(final GroupDefinition group) throws ValidationException
+ {
+ final String groupId = group.getId();
+ validateId(groupId);
+
+ final List unknownRules = group.getRuleIds().stream().filter(Predicate.not(this::containsRule))
+ .collect(Collectors.toList());
+ if (!unknownRules.isEmpty())
+ {
+ throw new ValidationException(String.format("Group '%s' references unknown rule%s: %s", groupId,
+ (unknownRules.size() > 1 ? "s" : ""), StringUtils.join(unknownRules)));
+ }
+
+ groups.put(groupId, group);
+ }
+
+ public boolean containsGroup(final String groupId)
+ {
+ return groups.containsKey(groupId);
+ }
+
+ public GroupDefinition getGroup(final String groupId)
+ {
+ return groups.get(groupId);
+ }
+
+ public Collection getGroups()
+ {
+ return Collections.unmodifiableCollection(groups.values());
+ }
+
+ void addRating(final RatingDefinition rating) throws ValidationException
+ {
+ final String ratingId = rating.getId();
+ validateId(ratingId);
+
+ ratings.put(ratingId, rating);
+ }
+
+ public boolean containsRating(final String ratingId)
+ {
+ return ratings.containsKey(ratingId);
+ }
+
+ public RatingDefinition getRating(final String ratingId)
+ {
+ return ratings.get(ratingId);
+ }
+
+ public Collection getRatings()
+ {
+ return Collections.unmodifiableCollection(ratings.values());
+ }
+
+ public int getVersion()
+ {
+ return version;
+ }
+
+ private void validateId(final String id) throws ValidationException
+ {
+ final String ambigousUDErrorMsg = "Some other %s shares the same ID: '%s'. IDs must be unique.";
+ if (containsSelector(id))
+ {
+ throw new ValidationException(String.format(ambigousUDErrorMsg, "selector", id));
+ }
+ if (containsRule(id))
+ {
+ throw new ValidationException(String.format(ambigousUDErrorMsg, "rule", id));
+ }
+ if (containsGroup(id))
+ {
+ throw new ValidationException(String.format(ambigousUDErrorMsg, "group", id));
+ }
+ if (containsRating(id))
+ {
+ throw new ValidationException(String.format(ambigousUDErrorMsg, "rating", id));
+ }
+ }
+
+ static Configuration fromJSON(final JSONObject jsonObject) throws ValidationException
+ {
+ final Configuration config = new Configuration(jsonObject.optInt("version", -1));
+ final JSONArray selectorArr = jsonObject.optJSONArray("selectors");
+ if (selectorArr != null)
+ {
+ for (int i = 0; i < selectorArr.length(); i++)
+ {
+ try
+ {
+ final SelectorDefinition selDef = SelectorDefinition.fromJSON(selectorArr.getJSONObject(i));
+
+ config.addSelector(selDef);
+ }
+ catch (final Exception e)
+ {
+ throw new ValidationException(String.format("Selector #%d is invalid: %s", i, e.getMessage()));
+ }
+ }
+ }
+
+ final JSONArray ruleArr = jsonObject.optJSONArray("rules");
+ if (ruleArr != null)
+ {
+ for (int i = 0; i < ruleArr.length(); i++)
+ {
+ try
+ {
+ final RuleDefinition ruleDef = RuleDefinition.fromJSON(ruleArr.getJSONObject(i));
+
+ config.addRule(ruleDef);
+ }
+ catch (final Exception e)
+ {
+ throw new ValidationException(String.format("Rule #%d is invalid: %s", i, e.getMessage()));
+ }
+ }
+ }
+
+ final JSONArray groupArr = jsonObject.getJSONArray("groups");
+ for (int i = 0; i < groupArr.length(); i++)
+ {
+ try
+ {
+ final GroupDefinition groupDef = GroupDefinition.fromJSON(groupArr.getJSONObject(i));
+
+ config.addGroup(groupDef);
+ }
+ catch (final Exception e)
+ {
+ throw new ValidationException(String.format("Group #%d is invalid: %s", i, e.getMessage()));
+ }
+ }
+
+ final JSONArray ratingArr = jsonObject.optJSONArray("ratings");
+ if (ratingArr != null)
+ {
+ for (int i = 0; i < ratingArr.length(); i++)
+ {
+ try
+ {
+ final RatingDefinition ratingDef = RatingDefinition.fromJSON(ratingArr.getJSONObject(i));
+
+ config.addRating(ratingDef);
+ }
+ catch (final Exception e)
+ {
+ throw new ValidationException(String.format("Rating #%d is invalid: %s", i, e.getMessage()));
+ }
+ }
+ }
+
+ if (!config.rules.values().stream().anyMatch(RuleDefinition::isEnabled))
+ {
+ throw new ValidationException("Configuration must contain at least one enabled rule");
+ }
+
+ if (!config.groups.values().stream().anyMatch((groupDef) -> groupDef.isEnabled() && !groupDef.getRuleIds().isEmpty()))
+ {
+ throw new ValidationException("Configuration must contain at least one enabled and non-empty group");
+ }
+
+ return config;
+ }
+
+ public static class MappedValuesConverter extends MapConverter
+ {
+
+ public MappedValuesConverter(final Mapper mapper, final Class type)
+ {
+ super(mapper, type);
+ }
+
+ public MappedValuesConverter(final Mapper mapper)
+ {
+ super(mapper);
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public boolean canConvert(final Class type)
+ {
+ if (Map.class.isAssignableFrom(type))
+ {
+ return true;
+ }
+
+ return super.canConvert(type);
+ }
+
+ @Override
+ public void marshal(final Object source, final HierarchicalStreamWriter writer, final MarshallingContext context)
+ {
+ final Map m = (Map) source;
+ for (final Object o : m.values())
+ {
+ writeCompleteItem(o, context, writer);
+ }
+ }
+
+ @Override
+ public Object unmarshal(final HierarchicalStreamReader reader, final UnmarshallingContext context)
+ {
+ return null;
+ }
+ }
+}
diff --git a/src/main/java/com/xceptance/xlt/report/scorecard/Evaluator.java b/src/main/java/com/xceptance/xlt/report/scorecard/Evaluator.java
new file mode 100644
index 000000000..b6930faf1
--- /dev/null
+++ b/src/main/java/com/xceptance/xlt/report/scorecard/Evaluator.java
@@ -0,0 +1,598 @@
+/*
+ * Copyright (c) 2005-2024 Xceptance Software Technologies GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.xceptance.xlt.report.scorecard;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.StringUtils;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONTokener;
+
+import com.google.common.io.Files;
+import com.thoughtworks.xstream.XStream;
+import com.xceptance.common.util.ParameterCheckUtils;
+import com.xceptance.xlt.common.XltConstants;
+import com.xceptance.xlt.report.scorecard.GroupDefinition.Mode;
+import com.xceptance.xlt.report.util.xstream.SanitizingDomDriver;
+
+import dev.harrel.jsonschema.Validator;
+import dev.harrel.jsonschema.ValidatorFactory;
+import dev.harrel.jsonschema.providers.OrgJsonNode;
+import net.sf.saxon.s9api.Processor;
+import net.sf.saxon.s9api.SaxonApiException;
+import net.sf.saxon.s9api.UnprefixedElementMatchingPolicy;
+import net.sf.saxon.s9api.XPathCompiler;
+import net.sf.saxon.s9api.XdmItem;
+import net.sf.saxon.s9api.XdmNode;
+import net.sf.saxon.s9api.XdmValue;
+
+public class Evaluator
+{
+ private static final String SCHEMA_RESOURCE_PATH = "configuration-schema.json";
+
+ private final File configFile;
+
+ private final Processor processor;
+
+ /**
+ * Creates a new evaluator instance that uses the given configuration JSON file.
+ *
+ * @param configFile
+ * the configuration JSON file to use
+ */
+ public Evaluator(final File configFile)
+ {
+ ParameterCheckUtils.isReadableFile(configFile, "configFile");
+
+ this.configFile = configFile;
+ this.processor = new Processor(false);
+ }
+
+ /**
+ * Evaluates the given XML file.
+ *
+ * @param documentFile
+ * the XML file to evaluate
+ * @return resulting scorecard
+ */
+ public Scorecard evaluate(final File documentFile)
+ {
+ ParameterCheckUtils.isNotNull(documentFile, "documentFile");
+
+ try
+ {
+ final Configuration config = parseConfiguration();
+ return doEvaluate(config, documentFile);
+ }
+ catch (final Exception ex)
+ {
+ return Scorecard.error(ex);
+ }
+
+ }
+
+ /**
+ * Writes the given scorecard as serialized XML to the given output file.
+ *
+ * @param scorecard
+ * the scorecard to be written
+ * @param outputFile
+ * the target output file
+ * @throws IOException
+ * thrown upon failure to write to given file
+ */
+ public void writeScorecardToFile(final Scorecard scorecard, final File outputFile) throws IOException
+ {
+ try (final OutputStreamWriter osw = new OutputStreamWriter(new FileOutputStream(outputFile), XltConstants.UTF8_ENCODING))
+ {
+ writeScorecard(scorecard, osw);
+ }
+ }
+
+ /**
+ * Writes the given scorecard as serialized XML to the given destination writer.
+ *
+ * @param scorecard
+ * the scorecard to be written
+ * @param writer
+ * the destination to write serialized XML to
+ * @throws IOException
+ * thrown if scorecard could not be written
+ */
+ public void writeScorecard(final Scorecard scorecard, final Writer writer) throws IOException
+ {
+ // writer the XML preamble
+ writer.write(XltConstants.XML_HEADER);
+
+ // create and configure XStream instance
+ final XStream xstream = new XStream(new SanitizingDomDriver());
+ xstream.autodetectAnnotations(true);
+ xstream.aliasSystemAttribute(null, "class");
+ xstream.setMode(XStream.NO_REFERENCES);
+
+ // let XStream do its job
+ xstream.toXML(scorecard, writer);
+ }
+
+ protected Configuration parseConfiguration() throws ValidationException, FileNotFoundException, IOException
+ {
+ final JSONObject schemaJSON;
+ try (final InputStream is = getClass().getResourceAsStream(SCHEMA_RESOURCE_PATH))
+ {
+ schemaJSON = new JSONObject(new JSONTokener(is));
+ }
+ catch (final JSONException je)
+ {
+ throw new ValidationException("Failed to parse JSON schema file", je);
+ }
+
+ final JSONObject configJSON;
+ try (final BufferedReader reader = Files.newReader(configFile, StandardCharsets.UTF_8))
+ {
+ configJSON = new JSONObject(new JSONTokener(reader));
+ }
+ catch (final JSONException je)
+ {
+ throw new ValidationException("Could not parse configuration file '" + configFile.getName() + "' as JSON", je);
+ }
+
+ final Validator validator = new ValidatorFactory().withJsonNodeFactory(new OrgJsonNode.Factory()).createValidator();
+
+ final URI schemaURI = validator.registerSchema(schemaJSON);
+ final Validator.Result validationResult = validator.validate(schemaURI, configJSON);
+ if (!validationResult.isValid())
+ {
+ throw new ValidationException("Configuration file '" + configFile.getName() + "' is malformed -> " +
+ validationResult.getErrors().stream().map(e -> e.getInstanceLocation() + ": " + e.getError())
+ .collect(Collectors.joining(", ")));
+ }
+
+ try
+ {
+ return Configuration.fromJSON(configJSON);
+ }
+ catch (final Exception e)
+ {
+ throw new ValidationException("Configuration file '" + configFile.getName() + "' is malformed", e);
+ }
+
+ }
+
+ protected Scorecard doEvaluate(final Configuration config, final File documentFile) throws SaxonApiException
+ {
+ final XdmNode docNode = processor.newDocumentBuilder().build(documentFile);
+ final XPathCompiler xpathCompiler = processor.newXPathCompiler();
+ xpathCompiler.setUnprefixedElementMatchingPolicy(UnprefixedElementMatchingPolicy.DEFAULT_NAMESPACE);
+ xpathCompiler.setCaching(true);
+
+ Integer points = 0, totalPoints = 0; // counters for achieved and achievable points
+ boolean testFailed = false; // whether to mark test as failed
+
+ // initialize scorecard and its result objects
+ final Scorecard scorecard = new Scorecard(config);
+ final Scorecard.Result result = scorecard.result;
+ // remember the erroneous groups to decide whether evaluation has a meaningful result later on
+ final List erroneousGroups = new ArrayList<>();
+
+ // loop through the list of configured groups (in definition order)
+ for (final GroupDefinition groupDef : config.getGroups())
+ {
+ // create a group result object
+ final Scorecard.Group group = new Scorecard.Group(groupDef);
+ // loop through the group's rule IDs (in definition order)
+ for (final String ruleId : groupDef.getRuleIds())
+ {
+ // lookup the rule's definition for this ID
+ final RuleDefinition ruleDef = config.getRule(ruleId);
+ // create a rule result object
+ final Scorecard.Rule rule = new Scorecard.Rule(ruleDef, groupDef.isEnabled());
+ // evaluate the rule
+ evaluateRule(rule, xpathCompiler, docNode, config::getSelector);
+
+ // add rule result to group
+ group.addRule(rule);
+
+ }
+
+ // conclude the evaluation of the group
+ testFailed = conclude(group) | testFailed;
+
+ // add number of group's achieved and achievable points to the counters
+ points += group.getPoints();
+ totalPoints += group.getTotalPoints();
+
+ if (group.getStatus().isError())
+ {
+ erroneousGroups.add(group);
+ }
+
+ // add group result to scorecard result
+ result.addGroup(group);
+ }
+
+ final boolean evaluationFailed = !erroneousGroups.isEmpty();
+ if (evaluationFailed)
+ {
+ // do not "overwrite" any previous error
+ if (StringUtils.isBlank(result.getError()))
+ {
+ // collect error messages from erroneous groups/rules and join them with two new-line characters
+ final String errorMessagesJoined = erroneousGroups.stream().flatMap(g -> g.getRules().stream())
+ .filter(r -> r.getStatus().isError() &&
+ StringUtils.isNotBlank(r.getMessage()))
+ .map(r -> String.format("Error evaluating rule '%s': %s", r.getId(),
+ r.getMessage()))
+ .collect(Collectors.joining("\n\n"));
+ if (StringUtils.isNotBlank(errorMessagesJoined))
+ {
+ result.setError(errorMessagesJoined);
+ }
+ }
+
+ }
+ else
+ {
+ // set overall number of achieved and achievable points
+ result.setPoints(points);
+ result.setTotalPoints(totalPoints);
+
+ // compute final score
+ final double pointsPercentage = getPercentage(points, totalPoints);
+
+ // determine the test's rating and whether it has failed
+ String rating = null;
+ for (final RatingDefinition ratingDef : config.getRatings())
+ {
+ if (ratingDef.isEnabled() && pointsPercentage <= ratingDef.getValue())
+ {
+ rating = ratingDef.getId();
+ testFailed = testFailed || ratingDef.isFailsTest();
+
+ break;
+ }
+
+ }
+
+ // set final values
+ result.setTestFailed(testFailed);
+ result.setPointsPercentage(pointsPercentage);
+ result.setRating(rating);
+ }
+
+ return scorecard;
+ }
+
+ private void evaluateRule(final Scorecard.Rule rule, final XPathCompiler compiler, final XdmNode document,
+ final Function selectorLookup)
+ {
+ for (final RuleDefinition.Check check : rule.getDefinition().getChecks())
+ {
+ final Scorecard.Rule.Check ruleCheck = new Scorecard.Rule.Check(check, rule.isEnabled());
+ if (ruleCheck.isEnabled())
+ {
+ evaluateRuleCheck(ruleCheck, compiler, document, selectorLookup);
+ }
+ rule.addCheck(ruleCheck);
+ }
+
+ conclude(rule);
+ }
+
+ private void evaluateRuleCheck(final Scorecard.Rule.Check check, final XPathCompiler compiler, final XdmNode document,
+ final Function selectorLookup)
+ {
+ final String selector;
+ // pick the right selector (specified directly or referenced by ID)
+ {
+ final String selectorId = check.getDefinition().getSelectorId();
+ if (selectorId != null)
+ {
+ selector = selectorLookup.apply(selectorId).getExpression();
+ }
+ else
+ {
+ selector = check.getDefinition().getSelector();
+ }
+ }
+ Status status = Status.FAILED;
+ String message = null, value = null;
+ try
+ {
+ final XdmValue result = compiler.evaluate(selector, document);
+ if (result.isEmpty())
+ {
+ status = Status.ERROR;
+ message = "No item found for selector '" + selector + "'";
+ }
+ else if (result.size() > 1)
+ {
+ status = Status.ERROR;
+ message = "Selector must match a single item but found " + result.size() + " items instead";
+ }
+
+ else
+ {
+ final XdmItem node = result.itemAt(0);
+ if (!(node.isAtomicValue() || node.isNode()))
+ {
+ status = Status.ERROR;
+ message = "Selected item is neither a node nor an atomic value";
+ }
+ else
+ {
+ value = node.getStringValue();
+ final boolean matches = evaluateConditionSafe(check.getDefinition().getCondition(), compiler, node);
+ if (matches)
+ {
+ status = Status.PASSED;
+ }
+ }
+
+ }
+ }
+ catch (final SaxonApiException sae)
+ {
+ status = Status.ERROR;
+ message = sae.getMessage();
+ }
+ check.setStatus(status);
+ check.setErrorMessage(message);
+ if (check.getDefinition().isDisplayValue())
+ {
+ check.setValue(value);
+ }
+ }
+
+ private boolean evaluateConditionSafe(final String condition, final XPathCompiler compiler, final XdmValue contextValue)
+ {
+ // strip any leading/trailing whitespace
+ String expr = StringUtils.strip(condition);
+ // if expression starts with a comparison operator, put a '.' in front of it
+ if (StringUtils.startsWithAny(expr, "=", "<", ">", "!="))
+ {
+ expr = ". " + expr;
+ }
+ try
+ {
+ return contextValue.select(compiler.compile(expr).asStep()).asAtomic().getBooleanValue();
+ }
+ catch (final ClassCastException | SaxonApiException e)
+ {
+ return false;
+ }
+ }
+
+ private void conclude(final Scorecard.Rule rule)
+ {
+ // nothing to do for disabled rules
+ if (!rule.isEnabled())
+ {
+ return;
+ }
+
+ Status lastStatus = Status.PASSED;
+ // loop through rule's checks
+ for (final Scorecard.Rule.Check c : rule.getChecks())
+ {
+ final Status checkStatus = c.getStatus();
+ // ignore skipped checks
+ if (checkStatus.isSkipped())
+ {
+ continue;
+ }
+
+ // remember most recent check status that doesn't indicate a passed check or just the very first check
+ // status if all checks did pass
+ if (!checkStatus.isPassed())
+ {
+ lastStatus = checkStatus;
+ // encountered erroneous check -> set rule message to the check's error message and stop looping
+ if (checkStatus.isError())
+ {
+ rule.setMessage(String.format("[Check #%d] %s", c.getIndex(), c.getErrorMessage()));
+ break;
+ }
+ }
+ }
+ // take action according to rule's final status
+ // (pass: set message to success message and points to all achievable points, fail: set message to fail message)
+
+ // negate rule status if desired
+ if (rule.getDefinition().isNegateResult())
+ {
+ lastStatus = lastStatus.negate();
+ }
+ rule.setStatus(lastStatus);
+ if (lastStatus.isPassed())
+ {
+ rule.setMessage(rule.getDefinition().getSuccessMessage());
+ rule.setPoints(rule.getDefinition().getPoints());
+ }
+ else if (lastStatus.isFailed())
+ {
+ rule.setMessage(rule.getDefinition().getFailMessage());
+ }
+
+ }
+
+ private boolean conclude(final Scorecard.Group group)
+ {
+ // nothing to do for disabled groups
+ if (!group.isEnabled())
+ {
+ return false;
+ }
+
+ Scorecard.Rule firstMatch = null, lastMatch = null;
+ int maxPoints = 0, sumPointsTotal = 0, sumPointsMatching = 0;
+
+ boolean somePassed = false, someFailed = false, someError = false;
+
+ // loop through group's rules and determine
+ // - the points of the first matching rule,
+ // - the points of the last matching rule,
+ // - the points of all matching rules (sum of),
+ // - the maximum number of all rules' points and
+ // - the overall sum of all rules' points
+ // - the rules' messages
+ // - the overall status of the group
+ // - + PASSED if at least one rule passed and mode is 'first' or 'last' OR if all rules did pass and mode is
+ // 'all'
+ // - + FAILED if at least one rule failed if mode is 'all' OR all rules did fail and mode is 'first' or 'last'
+ // - + ERROR if some rule was erroneous
+ final List rules = group.getRules();
+ for (final Scorecard.Rule rule : rules)
+ {
+ // rules must be enabled in order to participate in point calculation
+ // N.B. Rule status is SKIPPED if and only if rule is disabled
+ if (!rule.getDefinition().isEnabled()) // no need to check group for being enabled
+ {
+ continue;
+ }
+
+ final int pointsAchieved = rule.getPoints();
+ final int rulePoints = rule.getDefinition().getPoints();
+ final Status ruleStatus = rule.getStatus();
+
+ if (ruleStatus.isError())
+ {
+ someError = true;
+ break;
+ }
+
+ maxPoints = Math.max(maxPoints, rulePoints);
+ sumPointsTotal += rulePoints;
+
+ if (ruleStatus.isPassed())
+ {
+ somePassed = true;
+ if (firstMatch == null)
+ {
+ firstMatch = rule;
+ }
+ lastMatch = rule;
+
+ sumPointsMatching += pointsAchieved;
+
+ }
+ else if (ruleStatus.isFailed())
+ {
+ someFailed = true;
+ }
+
+ }
+
+ if (someError)
+ {
+ group.setStatus(Status.ERROR);
+ group.setPoints(0);
+ group.setTotalPoints(0);
+
+ return false;
+ }
+
+ // pick the correct values for group's points and total points according to its mode
+ final int points, totalPoints;
+ final Status groupStatus;
+
+ final List rulesThatMayFailTest;
+ final Mode mode = group.getDefinition().getMode();
+ if (mode == Mode.allPassed)
+ {
+ groupStatus = someFailed ? Status.FAILED : somePassed ? Status.PASSED : Status.SKIPPED;
+ points = sumPointsMatching;
+ totalPoints = sumPointsTotal;
+
+ rulesThatMayFailTest = rules;
+ }
+ else if (mode == Mode.firstPassed || mode == Mode.lastPassed)
+ {
+ final Scorecard.Rule triggerRule = (mode == Mode.firstPassed) ? firstMatch : lastMatch;
+ final int idx = triggerRule != null ? rules.indexOf(triggerRule) : -1;
+
+ groupStatus = somePassed ? Status.PASSED : someFailed ? Status.FAILED : Status.SKIPPED;
+ points = Optional.ofNullable(triggerRule).map(Scorecard.Rule::getPoints).orElse(0);
+ totalPoints = maxPoints;
+ // need to inspect the group's status trigger rule as well
+ rulesThatMayFailTest = idx < 0 ? rules : rules.subList(0, idx + 1);
+ }
+ else
+ {
+ groupStatus = Status.ERROR;
+ points = 0;
+ totalPoints = 0;
+
+ rulesThatMayFailTest = Collections.emptyList();
+ }
+
+ boolean testFailed = rulesThatMayFailTest.stream().filter((rule) -> {
+ final boolean ruleFailedTest = rule.mayFailTest();
+ if (ruleFailedTest)
+ {
+ rule.setTestFailed();
+ }
+ return ruleFailedTest;
+ }).count() > 0L;
+
+ if (group.mayFailTest())
+ {
+ group.setTestFailed();
+ testFailed = true;
+ }
+
+ group.setStatus(groupStatus);
+ group.setPoints(points);
+ group.setTotalPoints(totalPoints);
+
+ final String groupMessage = groupStatus.isPassed() ? group.getDefinition().getSuccessMessage()
+ : groupStatus.isFailed() ? group.getDefinition().getFailMessage() : null;
+ if (groupMessage != null)
+ {
+ group.setMessage(groupMessage);
+ }
+
+ return testFailed;
+ }
+
+ /**
+ * Returns the ratio in percent rounded to one decimal place.
+ *
+ * @param numerator
+ * @param denominator
+ * @return given ratio in percent
+ */
+ private static double getPercentage(final int numerator, final int denominator)
+ {
+ return denominator > 0 ? (Math.round((numerator * 1000.0) / denominator) / 10.0) : 0.0;
+ }
+
+}
diff --git a/src/main/java/com/xceptance/xlt/report/scorecard/GroupDefinition.java b/src/main/java/com/xceptance/xlt/report/scorecard/GroupDefinition.java
new file mode 100644
index 000000000..adda39403
--- /dev/null
+++ b/src/main/java/com/xceptance/xlt/report/scorecard/GroupDefinition.java
@@ -0,0 +1,242 @@
+/*
+ * Copyright (c) 2005-2024 Xceptance Software Technologies GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.xceptance.xlt.report.scorecard;
+
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.commons.lang3.EnumUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+import com.thoughtworks.xstream.annotations.XStreamAlias;
+import com.thoughtworks.xstream.annotations.XStreamAsAttribute;
+import com.thoughtworks.xstream.annotations.XStreamConverter;
+import com.thoughtworks.xstream.converters.MarshallingContext;
+import com.thoughtworks.xstream.converters.UnmarshallingContext;
+import com.thoughtworks.xstream.converters.collections.AbstractCollectionConverter;
+import com.thoughtworks.xstream.converters.enums.EnumToStringConverter;
+import com.thoughtworks.xstream.io.HierarchicalStreamReader;
+import com.thoughtworks.xstream.io.HierarchicalStreamWriter;
+import com.thoughtworks.xstream.mapper.Mapper;
+
+@XStreamAlias("group")
+public class GroupDefinition
+{
+ @XStreamAsAttribute
+ private final String id;
+
+ @XStreamAsAttribute
+ private final String name;
+
+ private String description;
+
+ @XStreamAlias("rules")
+ @XStreamConverter(RuleIdConverter.class)
+ private final List ruleIds;
+
+ @XStreamAsAttribute
+ private boolean enabled;
+
+ @XStreamAsAttribute
+ private boolean failsTest;
+
+ @XStreamAsAttribute
+ private TestFailTrigger failsOn;
+
+ @XStreamAsAttribute
+ @XStreamConverter(EnumToStringConverter.class)
+ private Mode mode;
+
+ private String successMessage;
+
+ private String failMessage;
+
+ public GroupDefinition(final String id, final String name, final List ruleIds)
+ {
+ this.id = Objects.requireNonNull(id, "Group ID must not be null");
+ this.name = name;
+ this.ruleIds = ruleIds;
+ }
+
+ public boolean isEnabled()
+ {
+ return enabled;
+ }
+
+ public void setEnabled(boolean enabled)
+ {
+ this.enabled = enabled;
+ }
+
+ public Mode getMode()
+ {
+ return mode;
+ }
+
+ public void setMode(Mode mode)
+ {
+ this.mode = mode;
+ }
+
+ public String getId()
+ {
+ return id;
+ }
+
+ public String getDescription()
+ {
+ return description;
+ }
+
+ public void setDescription(String description)
+ {
+ this.description = description;
+ }
+
+ public List getRuleIds()
+ {
+ return Collections.unmodifiableList(ruleIds);
+ }
+
+ public String getSuccessMessage()
+ {
+ return successMessage;
+ }
+
+ public void setSuccessMessage(String successMessage)
+ {
+ this.successMessage = successMessage;
+ }
+
+ public String getFailMessage()
+ {
+ return failMessage;
+ }
+
+ public void setFailMessage(String failMessage)
+ {
+ this.failMessage = failMessage;
+ }
+
+ public boolean isFailsTest()
+ {
+ return failsTest;
+ }
+
+ public void setFailsTest(boolean failsTest)
+ {
+ this.failsTest = failsTest;
+ }
+
+ public TestFailTrigger getFailsOn()
+ {
+ return failsOn;
+ }
+
+ public void setFailsOn(TestFailTrigger failsOn)
+ {
+ this.failsOn = failsOn;
+ }
+
+ public static enum Mode
+ {
+ firstPassed,
+ lastPassed,
+ allPassed;
+
+ }
+
+ public static GroupDefinition fromJSON(final JSONObject jsonObject) throws ValidationException
+ {
+ final String id = jsonObject.getString("id");
+ final String name = StringUtils.trimToNull(jsonObject.optString("name"));
+ final String description = jsonObject.optString("description", null);
+ final JSONArray ruleArr = jsonObject.getJSONArray("rules");
+ final boolean enabled = jsonObject.optBoolean("enabled", true);
+ final boolean failsTest = jsonObject.optBoolean("failsTest", false);
+ final String modeStr = StringUtils.trimToNull(jsonObject.optString("mode"));
+ final String failsOnStr = StringUtils.trimToNull(jsonObject.optString("failsOn"));
+
+ final Mode source = EnumUtils.getEnumIgnoreCase(Mode.class, modeStr, Mode.firstPassed);
+ final TestFailTrigger failsOn = EnumUtils.getEnumIgnoreCase(TestFailTrigger.class, failsOnStr);
+
+ final JSONObject messageObj = jsonObject.optJSONObject("messages");
+ final String successMessage = messageObj != null ? StringUtils.trimToNull(messageObj.optString("success")) : null;
+ final String failMessage = messageObj != null ? StringUtils.trimToNull(messageObj.optString("fail")) : null;
+
+ final LinkedList ruleIds = new LinkedList<>();
+ for (int i = 0; i < ruleArr.length(); i++)
+ {
+ ruleIds.add(ruleArr.getString(i));
+ }
+
+ if (ruleIds.isEmpty())
+ {
+ throw new ValidationException("Property 'rules' must contain at least one value");
+ }
+ if (ruleIds.stream().distinct().count() != ruleIds.size())
+ {
+ throw new ValidationException("Property 'rules' must contain distinct values");
+ }
+
+ final GroupDefinition group = new GroupDefinition(id, name, ruleIds);
+ group.setEnabled(enabled);
+ group.setDescription(description);
+ group.setFailsTest(failsTest);
+ group.setFailsOn(failsOn);
+ group.setMode(source);
+ group.setFailMessage(failMessage);
+ group.setSuccessMessage(successMessage);
+
+ return group;
+ }
+
+ public static class RuleIdConverter extends AbstractCollectionConverter
+ {
+ public RuleIdConverter(final Mapper mapper)
+ {
+ super(mapper);
+ }
+
+ @SuppressWarnings("rawtypes")
+ @Override
+ public boolean canConvert(final Class type)
+ {
+ return List.class.isAssignableFrom(type);
+ }
+
+ @Override
+ public void marshal(final Object source, final HierarchicalStreamWriter writer, final MarshallingContext context)
+ {
+ for (final Object o : (List) source)
+ {
+ writer.startNode("rule");
+ writer.addAttribute("ref-id", Objects.toString(o));
+ writer.endNode();
+ }
+ }
+
+ @Override
+ public Object unmarshal(final HierarchicalStreamReader reader, final UnmarshallingContext context)
+ {
+ return null;
+ }
+ }
+}
diff --git a/src/main/java/com/xceptance/xlt/report/scorecard/RatingDefinition.java b/src/main/java/com/xceptance/xlt/report/scorecard/RatingDefinition.java
new file mode 100644
index 000000000..237ed94e1
--- /dev/null
+++ b/src/main/java/com/xceptance/xlt/report/scorecard/RatingDefinition.java
@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2005-2024 Xceptance Software Technologies GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.xceptance.xlt.report.scorecard;
+
+import java.util.Objects;
+
+import org.apache.commons.lang3.StringUtils;
+import org.json.JSONObject;
+
+import com.thoughtworks.xstream.annotations.XStreamAlias;
+import com.thoughtworks.xstream.annotations.XStreamAsAttribute;
+
+@XStreamAlias("rating")
+public class RatingDefinition
+{
+ @XStreamAsAttribute
+ private final String id;
+
+ @XStreamAsAttribute
+ private final String name;
+
+ private final String description;
+
+ @XStreamAsAttribute
+ private final double value;
+
+ @XStreamAsAttribute
+ private final boolean enabled;
+
+ @XStreamAsAttribute
+ private final boolean failsTest;
+
+ RatingDefinition(final String id, final String name, final String description, final double value, final boolean enabled,
+ final boolean failsTest)
+ {
+ this.id = Objects.requireNonNull(id, "Rating ID must not be null");
+ this.name = name;
+ this.value = value;
+ this.description = description;
+ this.enabled = enabled;
+ this.failsTest = failsTest;
+ }
+
+ public String getId()
+ {
+ return id;
+ }
+
+ public String getName()
+ {
+ return name;
+ }
+
+ public String getDescription()
+ {
+ return description;
+ }
+
+ public double getValue()
+ {
+ return value;
+ }
+
+ public boolean isEnabled()
+ {
+ return enabled;
+ }
+
+ public boolean isFailsTest()
+ {
+ return failsTest;
+ }
+
+ static RatingDefinition fromJSON(final JSONObject jsonObject) throws ValidationException
+ {
+ final String id = jsonObject.getString("id");
+ final String name = StringUtils.trimToNull(jsonObject.optString("name"));
+ final String desc = jsonObject.optString("description", null);
+ final double value = jsonObject.getDouble("value");
+ final boolean enabled = jsonObject.optBoolean("enabled", true);
+ final boolean failsTest = jsonObject.optBoolean("failsTest", false);
+
+ if (Math.min(Math.max(0.0, value), 100.0) != value)
+ {
+ throw new ValidationException("Property 'value' must be greater than or equal to 0.0 and less than or equal to 100.0");
+ }
+
+ return new RatingDefinition(id, name, desc, value, enabled, failsTest);
+ }
+}
diff --git a/src/main/java/com/xceptance/xlt/report/scorecard/RuleDefinition.java b/src/main/java/com/xceptance/xlt/report/scorecard/RuleDefinition.java
new file mode 100644
index 000000000..9a1aca889
--- /dev/null
+++ b/src/main/java/com/xceptance/xlt/report/scorecard/RuleDefinition.java
@@ -0,0 +1,313 @@
+/*
+ * Copyright (c) 2005-2024 Xceptance Software Technologies GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.xceptance.xlt.report.scorecard;
+
+import java.util.LinkedList;
+import java.util.Objects;
+
+import org.apache.commons.lang3.EnumUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+import com.thoughtworks.xstream.annotations.XStreamAlias;
+import com.thoughtworks.xstream.annotations.XStreamAsAttribute;
+import com.thoughtworks.xstream.annotations.XStreamConverter;
+import com.thoughtworks.xstream.converters.Converter;
+import com.thoughtworks.xstream.converters.MarshallingContext;
+import com.thoughtworks.xstream.converters.UnmarshallingContext;
+import com.thoughtworks.xstream.io.HierarchicalStreamReader;
+import com.thoughtworks.xstream.io.HierarchicalStreamWriter;
+
+@XStreamAlias("rule")
+public class RuleDefinition
+{
+ @XStreamAsAttribute
+ private final String id;
+
+ @XStreamAsAttribute
+ private final String name;
+
+ @XStreamAsAttribute
+ private boolean enabled = true;
+
+ private String description;
+
+ @XStreamAsAttribute
+ private boolean failsTest;
+
+ @XStreamAsAttribute
+ private TestFailTrigger failsOn;
+
+ @XStreamAsAttribute
+ private boolean negateResult;
+
+ private final Check[] checks;
+
+ @XStreamAsAttribute
+ private int points;
+
+ private String successMessage;
+
+ private String failMessage;
+
+ public RuleDefinition(final String id, final String name, final Check[] checks)
+ {
+ this.id = Objects.requireNonNull(id, "Rule ID must not be null");
+ this.name = name;
+ this.checks = Objects.requireNonNull(checks, "Rule checks must not be null");
+ }
+
+ public boolean isEnabled()
+ {
+ return enabled;
+ }
+
+ public void setEnabled(final boolean enabled)
+ {
+ this.enabled = enabled;
+ }
+
+ public String getDescription()
+ {
+ return description;
+ }
+
+ public void setDescription(final String description)
+ {
+ this.description = description;
+ }
+
+ public boolean isFailsTest()
+ {
+ return failsTest;
+ }
+
+ public void setFailsTest(final boolean failsTest)
+ {
+ this.failsTest = failsTest;
+ }
+
+ public TestFailTrigger getFailsOn()
+ {
+ return failsOn;
+ }
+
+ public void setFailsOn(final TestFailTrigger failsOn)
+ {
+ this.failsOn = failsOn;
+ }
+
+ public boolean isNegateResult()
+ {
+ return negateResult;
+ }
+
+ public void setNegateResult(boolean negateResult)
+ {
+ this.negateResult = negateResult;
+ }
+
+ public int getPoints()
+ {
+ return points;
+ }
+
+ public void setPoints(final int points)
+ {
+ this.points = points;
+ }
+
+ public String getSuccessMessage()
+ {
+ return successMessage;
+ }
+
+ public void setSuccessMessage(final String successMessage)
+ {
+ this.successMessage = successMessage;
+ }
+
+ public String getFailMessage()
+ {
+ return failMessage;
+ }
+
+ public void setFailMessage(final String failMessage)
+ {
+ this.failMessage = failMessage;
+ }
+
+ public String getId()
+ {
+ return id;
+ }
+
+ public String getName()
+ {
+ return name;
+ }
+
+ public Check[] getChecks()
+ {
+ return checks;
+ }
+
+ @XStreamAlias("check")
+ public static class Check
+ {
+ @XStreamAsAttribute
+ private final int index;
+
+ private final String selector;
+
+ @XStreamConverter(SelectorIdConverter.class)
+ @XStreamAlias("selector")
+ private final String selectorId;
+
+ private final String condition;
+
+ @XStreamAsAttribute
+ private final boolean enabled;
+
+ @XStreamAsAttribute
+ private final boolean displayValue;
+
+ public Check(final int index, final String selector, final String selectorId, final String condition, final boolean enabled,
+ final boolean displayValue)
+ {
+ this.index = Math.max(0, index);
+ this.selectorId = selectorId;
+ this.selector = selector;
+ this.condition = Objects.requireNonNull(condition, "Rule check condition must not be null");
+ this.enabled = enabled;
+ this.displayValue = displayValue;
+ }
+
+ public int getIndex()
+ {
+ return index;
+ }
+
+ public boolean isEnabled()
+ {
+ return enabled;
+ }
+
+ public boolean isDisplayValue()
+ {
+ return displayValue;
+ }
+
+ public String getSelector()
+ {
+ return selector;
+ }
+
+ public String getSelectorId()
+ {
+ return selectorId;
+ }
+
+ public String getCondition()
+ {
+ return condition;
+ }
+
+ }
+
+ static RuleDefinition fromJSON(final JSONObject jsonObject) throws ValidationException
+ {
+ final String ruleId = jsonObject.getString("id");
+ final String ruleName = StringUtils.trimToNull(jsonObject.optString("name"));
+ final String ruleDesc = jsonObject.optString("description", null);
+ final boolean enabled = jsonObject.optBoolean("enabled", true);
+ final boolean failsTest = jsonObject.optBoolean("failsTest", false);
+ final boolean negateResult = jsonObject.optBoolean("negateResult", false);
+ final JSONObject messages = jsonObject.optJSONObject("messages");
+ final JSONArray checks = jsonObject.optJSONArray("checks");
+ final int rulePoints = jsonObject.optInt("points");
+
+ final String failsOnStr = StringUtils.trimToNull(jsonObject.optString("failsOn"));
+ final TestFailTrigger failsOn = EnumUtils.getEnumIgnoreCase(TestFailTrigger.class, failsOnStr);
+
+ final LinkedList checkList = new LinkedList<>();
+ if (checks != null)
+ {
+ for (int i = 0; i < checks.length(); i++)
+ {
+ final JSONObject checkObj = checks.getJSONObject(i);
+ final String selector = checkObj.optString("selector", null);
+ final String selectorId = checkObj.optString("selectorId", null);
+ final String condition = checkObj.getString("condition");
+
+ final boolean displayValue = checkObj.optBoolean("displayValue", true);
+ final boolean checkEnabled = checkObj.optBoolean("enabled", true);
+
+ if (!(selector == null ^ selectorId == null))
+ {
+ throw new ValidationException("Check #" + i +
+ " is ambiguous: either 'selector' or 'selectorId' property must be given");
+ }
+
+ checkList.add(new Check(i, selector, selectorId, condition, checkEnabled, displayValue));
+ }
+ }
+ if (rulePoints < 0)
+ {
+ throw new ValidationException("Property 'points' must be a non-negative integer");
+ }
+ if (!checkList.isEmpty() && !checkList.stream().anyMatch(Check::isEnabled))
+ {
+ throw new ValidationException("Property 'checks' must contain at least one enabled check definition");
+ }
+
+ final RuleDefinition ruleDef = new RuleDefinition(ruleId, ruleName, checkList.toArray(Check[]::new));
+ ruleDef.setDescription(ruleDesc);
+ ruleDef.setEnabled(enabled);
+ ruleDef.setFailsTest(failsTest);
+ ruleDef.setFailsOn(failsOn);
+ ruleDef.setNegateResult(negateResult);
+ ruleDef.setPoints(rulePoints);
+ ruleDef.setSuccessMessage(messages != null ? StringUtils.trimToNull(messages.optString("success")) : null);
+ ruleDef.setFailMessage(messages != null ? StringUtils.trimToNull(messages.optString("fail")) : null);
+
+ return ruleDef;
+ }
+
+ public static class SelectorIdConverter implements Converter
+ {
+ @SuppressWarnings("rawtypes")
+ @Override
+ public boolean canConvert(Class type)
+ {
+ return type == String.class;
+ }
+
+ @Override
+ public void marshal(Object source, HierarchicalStreamWriter writer, MarshallingContext context)
+ {
+ writer.addAttribute("ref-id", Objects.toString(source));
+ }
+
+ @Override
+ public Object unmarshal(HierarchicalStreamReader reader, UnmarshallingContext context)
+ {
+ return null;
+ }
+
+ }
+}
diff --git a/src/main/java/com/xceptance/xlt/report/scorecard/Scorecard.java b/src/main/java/com/xceptance/xlt/report/scorecard/Scorecard.java
new file mode 100644
index 000000000..ad8f2a2eb
--- /dev/null
+++ b/src/main/java/com/xceptance/xlt/report/scorecard/Scorecard.java
@@ -0,0 +1,446 @@
+/*
+ * Copyright (c) 2005-2024 Xceptance Software Technologies GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.xceptance.xlt.report.scorecard;
+
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.builder.ReflectionToStringBuilder;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import com.thoughtworks.xstream.annotations.XStreamAlias;
+import com.thoughtworks.xstream.annotations.XStreamAsAttribute;
+
+@XStreamAlias("scorecard")
+public class Scorecard
+{
+ public static class Result
+ {
+ @XStreamAsAttribute
+ private Double pointsPercentage;
+
+ @XStreamAsAttribute
+ private Integer points;
+
+ @XStreamAsAttribute
+ private Integer totalPoints;
+
+ @XStreamAsAttribute
+ private boolean testFailed;
+
+ private List groups;
+
+ private String error;
+
+ private String rating;
+
+ public String getError()
+ {
+ return error;
+ }
+
+ void setError(final String error)
+ {
+ this.error = Objects.requireNonNull(error);
+ }
+
+ public Integer getPoints()
+ {
+ return points;
+ }
+
+ void setPoints(final Integer points)
+ {
+ this.points = points;
+ }
+
+ public Double getPointsPercentage()
+ {
+ return pointsPercentage;
+ }
+
+ void setPointsPercentage(final Double pointsPercentage)
+ {
+ this.pointsPercentage = pointsPercentage;
+ }
+
+ public boolean isTestFailed()
+ {
+ return testFailed;
+ }
+
+ void setTestFailed(final boolean testFailed)
+ {
+ this.testFailed = testFailed;
+ }
+
+ public Integer getTotalPoints()
+ {
+ return totalPoints;
+ }
+
+ void setTotalPoints(final Integer totalPoints)
+ {
+ this.totalPoints = totalPoints;
+ }
+
+ void addGroup(final Group group)
+ {
+ Objects.requireNonNull(group);
+ if (groups == null)
+ {
+ groups = new LinkedList<>();
+ }
+ groups.add(group);
+ }
+
+ public List getGroups()
+ {
+ if (groups == null)
+ {
+ return Collections.emptyList();
+ }
+ return Collections.unmodifiableList(groups);
+ }
+
+ public String getRating()
+ {
+ return rating;
+ }
+
+ void setRating(final String rating)
+ {
+ this.rating = rating;
+ }
+
+ }
+
+ public final Configuration configuration;
+
+ @XStreamAlias("outcome")
+ public final Scorecard.Result result = new Scorecard.Result();
+
+ Scorecard(final Configuration configuration)
+ {
+ this.configuration = configuration;
+ }
+
+ static Scorecard error(final Throwable t)
+ {
+ final Scorecard r = new Scorecard(null);
+ final String errMsg = ExceptionUtils.stream(t).map(Throwable::getMessage).collect(Collectors.joining(" -> "));
+ r.result.setError(errMsg);
+ return r;
+ }
+
+ @Override
+ public String toString()
+ {
+ return ReflectionToStringBuilder.toString(this);
+ }
+
+ @XStreamAlias("group")
+ public static class Group
+ {
+ private final transient GroupDefinition definition;
+
+ @XStreamAsAttribute
+ @XStreamAlias("ref-id")
+ private final String id;
+
+ private final List rules = new LinkedList<>();
+
+ @XStreamAsAttribute
+ private int points;
+
+ @XStreamAsAttribute
+ private int totalPoints;
+
+ private String message;
+
+ @XStreamAlias("result")
+ private Status status = Status.SKIPPED;
+
+ @XStreamAsAttribute
+ private boolean testFailed;
+
+ Group(final GroupDefinition definition)
+ {
+ this.definition = Objects.requireNonNull(definition, "Group definition must not be null");
+ this.id = definition.getId();
+ }
+
+ public GroupDefinition getDefinition()
+ {
+ return definition;
+ }
+
+ public String getId()
+ {
+ return id;
+ }
+
+ public List getRules()
+ {
+ return Collections.unmodifiableList(rules);
+ }
+
+ void addRule(final Rule rule)
+ {
+ rules.add(Objects.requireNonNull(rule));
+ }
+
+ public int getPoints()
+ {
+ return points;
+ }
+
+ void setPoints(final int points)
+ {
+ this.points = Math.max(0, points);
+ }
+
+ public int getTotalPoints()
+ {
+ return totalPoints;
+ }
+
+ void setTotalPoints(final int totalPoints)
+ {
+ this.totalPoints = Math.max(0, totalPoints);
+ }
+
+ public String getMessage()
+ {
+ return message;
+ }
+
+ void setMessage(final String message)
+ {
+ this.message = StringUtils.trimToNull(message);
+ }
+
+ void setStatus(final Status status)
+ {
+ this.status = Objects.requireNonNull(status);
+ }
+
+ public Status getStatus()
+ {
+ return status;
+ }
+
+ void setTestFailed()
+ {
+ this.testFailed = true;
+ }
+
+ public boolean isEnabled()
+ {
+ return definition.isEnabled();
+ }
+
+ TestFailTrigger getFailsOn()
+ {
+ return Optional.ofNullable(definition.getFailsOn()).orElse(TestFailTrigger.NOTPASSED);
+ }
+
+ boolean mayFailTest()
+ {
+ return isEnabled() && definition.isFailsTest() && getFailsOn().isTriggeredBy(status);
+ }
+ }
+
+ @XStreamAlias("rule")
+ public static class Rule
+ {
+ private transient final RuleDefinition definition;
+
+ private transient final boolean groupEnabled;
+
+ private final List checks = new LinkedList<>();
+
+ @XStreamAlias("result")
+ private Status status = Status.SKIPPED;
+
+ private String message;
+
+ @XStreamAsAttribute
+ private int points;
+
+ @XStreamAsAttribute
+ @XStreamAlias("ref-id")
+ private final String id;
+
+ @XStreamAsAttribute
+ private boolean testFailed;
+
+ Rule(final RuleDefinition definition, final boolean groupEnabled)
+ {
+ this.definition = Objects.requireNonNull(definition, "Rule definition must not be null");
+ this.groupEnabled = groupEnabled;
+
+ this.id = definition.getId();
+ }
+
+ public String getId()
+ {
+ return id;
+ }
+
+ public RuleDefinition getDefinition()
+ {
+ return definition;
+ }
+
+ public List getChecks()
+ {
+ return Collections.unmodifiableList(checks);
+ }
+
+ void addCheck(final Check check)
+ {
+ checks.add(Objects.requireNonNull(check));
+ }
+
+ public Status getStatus()
+ {
+ return status;
+ }
+
+ void setStatus(final Status status)
+ {
+ this.status = Objects.requireNonNull(status);
+ }
+
+ public String getMessage()
+ {
+ return message;
+ }
+
+ void setMessage(final String message)
+ {
+ this.message = StringUtils.trimToNull(message);
+ }
+
+ public int getPoints()
+ {
+ return points;
+ }
+
+ void setPoints(final int points)
+ {
+ this.points = Math.max(0, Math.min(definition.getPoints(), points));
+ }
+
+ public boolean isEnabled()
+ {
+ return groupEnabled && definition.isEnabled();
+ }
+
+ TestFailTrigger getFailsOn()
+ {
+ return Optional.ofNullable(definition.getFailsOn()).orElse(TestFailTrigger.NOTPASSED);
+ }
+
+ boolean mayFailTest()
+ {
+ return isEnabled() && definition.isFailsTest() && getFailsOn().isTriggeredBy(status);
+ }
+
+ void setTestFailed()
+ {
+ this.testFailed = true;
+ }
+
+ @XStreamAlias("check")
+ public static class Check
+ {
+ @XStreamAsAttribute
+ private final int index;
+
+ private transient final RuleDefinition.Check definition;
+
+ private transient final boolean ruleEnabled;
+
+ @XStreamAlias("result")
+ private Status status = Status.SKIPPED;
+
+ private String errorMessage;
+
+ private String value;
+
+ Check(final RuleDefinition.Check definition, final boolean ruleEnabled)
+ {
+ this.definition = Objects.requireNonNull(definition, "Check definition must not be null");
+ this.ruleEnabled = ruleEnabled;
+
+ this.index = definition.getIndex();
+ }
+
+ public int getIndex()
+ {
+ return index;
+ }
+
+ public RuleDefinition.Check getDefinition()
+ {
+ return definition;
+ }
+
+ public Status getStatus()
+ {
+ return status;
+ }
+
+ void setStatus(final Status status)
+ {
+ this.status = Objects.requireNonNull(status);
+ }
+
+ public String getErrorMessage()
+ {
+ return errorMessage;
+ }
+
+ void setErrorMessage(final String errorMessage)
+ {
+ this.errorMessage = StringUtils.trimToNull(errorMessage);
+ }
+
+ public String getValue()
+ {
+ return value;
+ }
+
+ void setValue(final String value)
+ {
+ this.value = value;
+ }
+
+ public boolean isEnabled()
+ {
+ return ruleEnabled && definition.isEnabled();
+ }
+ }
+
+ }
+}
diff --git a/src/main/java/com/xceptance/xlt/report/scorecard/SelectorDefinition.java b/src/main/java/com/xceptance/xlt/report/scorecard/SelectorDefinition.java
new file mode 100644
index 000000000..08ba07ee2
--- /dev/null
+++ b/src/main/java/com/xceptance/xlt/report/scorecard/SelectorDefinition.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2005-2024 Xceptance Software Technologies GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.xceptance.xlt.report.scorecard;
+
+import java.util.Objects;
+
+import org.json.JSONObject;
+
+import com.thoughtworks.xstream.annotations.XStreamAlias;
+import com.thoughtworks.xstream.annotations.XStreamAsAttribute;
+
+@XStreamAlias("selector")
+public class SelectorDefinition
+{
+ @XStreamAsAttribute
+ private final String id;
+
+ private final String expression;
+
+ SelectorDefinition(final String id, final String expression)
+ {
+ this.id = Objects.requireNonNull(id, "Selector ID must not be null");
+ this.expression = Objects.requireNonNull(expression, "Selector expression must not be null");
+ }
+
+ public String getId()
+ {
+ return id;
+ }
+
+ public String getExpression()
+ {
+ return expression;
+ }
+
+ static SelectorDefinition fromJSON(final JSONObject jsonObject) throws ValidationException
+ {
+ final String id = jsonObject.getString("id");
+ final String expression = jsonObject.getString("expression");
+
+ return new SelectorDefinition(id, expression);
+ }
+}
diff --git a/src/main/java/com/xceptance/xlt/report/scorecard/Status.java b/src/main/java/com/xceptance/xlt/report/scorecard/Status.java
new file mode 100644
index 000000000..e9fd82bbd
--- /dev/null
+++ b/src/main/java/com/xceptance/xlt/report/scorecard/Status.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2005-2024 Xceptance Software Technologies GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.xceptance.xlt.report.scorecard;
+
+import com.thoughtworks.xstream.annotations.XStreamConverter;
+import com.thoughtworks.xstream.converters.enums.EnumToStringConverter;
+
+@XStreamConverter(EnumToStringConverter.class)
+enum Status
+{
+ SKIPPED,
+ PASSED,
+ FAILED("NOTPASSED"),
+ ERROR;
+
+ private final String displayValue;
+
+ private Status(final String value)
+ {
+ this.displayValue = value;
+ }
+
+ private Status()
+ {
+ this(null);
+ }
+
+ public boolean isPassed()
+ {
+ return Status.PASSED == this;
+ }
+
+ public boolean isFailed()
+ {
+ return Status.FAILED == this;
+ }
+
+ public boolean isError()
+ {
+ return Status.ERROR == this;
+ }
+
+ public boolean isSkipped()
+ {
+ return Status.SKIPPED == this;
+ }
+
+ @Override
+ public String toString()
+ {
+ return displayValue == null ? name() : displayValue;
+ }
+
+ /**
+ * Returns the negated status of this status (PASSED → FAILED; FAILED → PASSED; SKIPPED and ERROR stay as
+ * is).
+ *
+ * @return this status negated
+ */
+ Status negate()
+ {
+ if (isPassed())
+ {
+ return FAILED;
+ }
+ if (isFailed())
+ {
+ return PASSED;
+ }
+ return this;
+ }
+
+}
diff --git a/src/main/java/com/xceptance/xlt/report/scorecard/TestFailTrigger.java b/src/main/java/com/xceptance/xlt/report/scorecard/TestFailTrigger.java
new file mode 100644
index 000000000..c77d984ae
--- /dev/null
+++ b/src/main/java/com/xceptance/xlt/report/scorecard/TestFailTrigger.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2005-2024 Xceptance Software Technologies GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.xceptance.xlt.report.scorecard;
+
+enum TestFailTrigger
+{
+ PASSED,
+ NOTPASSED;
+
+ boolean isTriggeredBy(final Status status)
+ {
+ if (status != null)
+ {
+ switch (this)
+ {
+ case NOTPASSED:
+ return status.isFailed();
+ case PASSED:
+ return status.isPassed();
+ }
+ }
+
+ return false;
+ }
+}
diff --git a/src/main/java/com/xceptance/xlt/report/scorecard/UpdateMain.java b/src/main/java/com/xceptance/xlt/report/scorecard/UpdateMain.java
new file mode 100644
index 000000000..0f495cbf0
--- /dev/null
+++ b/src/main/java/com/xceptance/xlt/report/scorecard/UpdateMain.java
@@ -0,0 +1,165 @@
+/*
+ * Copyright (c) 2005-2024 Xceptance Software Technologies GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.xceptance.xlt.report.scorecard;
+
+import java.io.File;
+import java.util.Locale;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.vfs2.VFS;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.xceptance.common.util.Console;
+import com.xceptance.common.util.ProcessExitCodes;
+import com.xceptance.xlt.api.util.XltLogger;
+import com.xceptance.xlt.common.XltConstants;
+import com.xceptance.xlt.engine.XltEngine;
+import com.xceptance.xlt.engine.XltExecutionContext;
+import com.xceptance.xlt.report.ReportGenerator;
+import com.xceptance.xlt.util.Timer;
+
+public final class UpdateMain
+{
+ /** Class logger */
+ private static final Logger LOG = LoggerFactory.getLogger(UpdateMain.class);
+
+ public static void main(final String[] args) throws Throwable
+ {
+ Locale.setDefault(Locale.US);
+
+ XltLogger.reportLogger.info(Console.horizontalBar());
+ XltLogger.reportLogger.info(Console.startSection("XLT Scorecard Update"));
+ XltLogger.reportLogger.info(Console.endSection());
+
+ final UpdateMain main = new UpdateMain();
+ try
+ {
+ main.init(args);
+ }
+ catch (final Exception e)
+ {
+ final String errorMessage = "Failed to initialize report generator";
+ System.err.println(errorMessage + ": " + e.getMessage());
+ LOG.error(errorMessage, e);
+
+ main.printUsage();
+ System.exit(ProcessExitCodes.PARAMETER_ERROR);
+ }
+
+ try
+ {
+
+ main.run();
+
+ System.exit(ProcessExitCodes.SUCCESS);
+ }
+ catch (final Exception e)
+ {
+ final String errorMessage = "Failed to update scorecard";
+ System.err.println(errorMessage + ": " + e.getMessage());
+ LOG.error(errorMessage, e);
+
+ System.exit(ProcessExitCodes.GENERAL_ERROR);
+ }
+
+ }
+
+ private final Options options;
+
+ private File inputDir;
+
+ private File outputDir;
+
+ private UpdateMain()
+ {
+ options = createCommandLineOptions();
+ }
+
+ private void init(final String[] arguments) throws ParseException
+ {
+ XltLogger.reportLogger.info(Console.horizontalBar());
+ XltLogger.reportLogger.info(Console.startSection("Initializing..."));
+
+ final Timer timer = Timer.start();
+
+ XltEngine.get();
+
+ final CommandLine cmdLine = new DefaultParser().parse(options, arguments);
+
+ final String outputDirName = cmdLine.getOptionValue(XltConstants.COMMANDLINE_OPTION_OUTPUT_DIR);
+ if (outputDirName != null)
+ {
+ outputDir = new File(outputDirName);
+ }
+
+ final String[] remainingArguments = cmdLine.getArgs();
+ if (remainingArguments.length != 1)
+ {
+ throw new IllegalArgumentException("Please specify a single report directory");
+ }
+
+ inputDir = new File(remainingArguments[0]);
+
+ XltExecutionContext.getCurrent().setTestSuiteHomeDir(inputDir);
+ XltExecutionContext.getCurrent().setTestSuiteConfigDir(new File(inputDir, XltConstants.CONFIG_DIR_NAME));
+
+ XltLogger.reportLogger.info(timer.stop().get("...finished"));
+ XltLogger.reportLogger.info(Console.endSection());
+ }
+
+ private void run() throws Exception
+ {
+ XltLogger.reportLogger.info(Console.horizontalBar());
+ XltLogger.reportLogger.info(Console.startSection("Setup..."));
+
+ final Timer timer = Timer.start();
+
+ final ReportGenerator reportGenerator = new ReportGenerator(VFS.getManager().resolveFile(inputDir.toURI()), outputDir, true);
+
+ XltLogger.reportLogger.info(timer.stop().get("...finished"));
+ XltLogger.reportLogger.info(Console.endSection());
+
+ reportGenerator.updateScorecard();
+ }
+
+ private void printUsage()
+ {
+ final HelpFormatter helpFormatter = new HelpFormatter();
+ helpFormatter.setSyntaxPrefix("Usage: ");
+ helpFormatter.setWidth(79);
+
+ System.out.println();
+ helpFormatter.printHelp("update_scorecard () reportdir", "Updates the scorecard of the given report\n\nOptions:\n",
+ options, null);
+ System.out.println();
+ }
+
+ private static Options createCommandLineOptions()
+ {
+ final Options opts = new Options();
+ final Option targetDir = new Option(XltConstants.COMMANDLINE_OPTION_OUTPUT_DIR, true, "the output directory");
+ targetDir.setArgName("dir");
+ opts.addOption(targetDir);
+
+ return opts;
+ }
+}
diff --git a/src/main/java/com/xceptance/xlt/report/scorecard/ValidationException.java b/src/main/java/com/xceptance/xlt/report/scorecard/ValidationException.java
new file mode 100644
index 000000000..6c6659272
--- /dev/null
+++ b/src/main/java/com/xceptance/xlt/report/scorecard/ValidationException.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2005-2024 Xceptance Software Technologies GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.xceptance.xlt.report.scorecard;
+
+public class ValidationException extends Exception
+{
+
+ public ValidationException(final String message, final Throwable cause)
+ {
+ super(message, cause);
+ }
+
+ public ValidationException(final String message)
+ {
+ super(message);
+ }
+
+ public ValidationException(final Throwable cause)
+ {
+ super(cause);
+ }
+
+}
diff --git a/src/main/java/com/xceptance/xlt/report/util/xstream/SanitizingDomDriver.java b/src/main/java/com/xceptance/xlt/report/util/xstream/SanitizingDomDriver.java
new file mode 100644
index 000000000..385e1aaa4
--- /dev/null
+++ b/src/main/java/com/xceptance/xlt/report/util/xstream/SanitizingDomDriver.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2005-2024 Xceptance Software Technologies GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.xceptance.xlt.report.util.xstream;
+
+import java.io.Writer;
+
+import com.thoughtworks.xstream.io.HierarchicalStreamWriter;
+import com.thoughtworks.xstream.io.xml.DomDriver;
+
+/**
+ * A custom {@link DomDriver} that uses a {@link SanitizingWriter} to write an XML file.
+ */
+public class SanitizingDomDriver extends DomDriver
+{
+ @Override
+ public HierarchicalStreamWriter createWriter(final Writer out)
+ {
+ return new SanitizingWriter(out, getNameCoder());
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/xceptance/xlt/report/util/xstream/SanitizingWriter.java b/src/main/java/com/xceptance/xlt/report/util/xstream/SanitizingWriter.java
new file mode 100644
index 000000000..3e5b030df
--- /dev/null
+++ b/src/main/java/com/xceptance/xlt/report/util/xstream/SanitizingWriter.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2005-2024 Xceptance Software Technologies GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.xceptance.xlt.report.util.xstream;
+
+import java.io.Writer;
+
+import org.apache.commons.text.StringEscapeUtils;
+
+import com.thoughtworks.xstream.core.util.QuickWriter;
+import com.thoughtworks.xstream.io.naming.NameCoder;
+import com.thoughtworks.xstream.io.xml.PrettyPrintWriter;
+
+/**
+ * A custom {@link PrettyPrintWriter} that silently removes invalid XML 1.0 characters when writing text nodes.
+ */
+class SanitizingWriter extends PrettyPrintWriter
+{
+ public SanitizingWriter(final Writer writer, final NameCoder nameCoder)
+ {
+ super(writer, nameCoder);
+ }
+
+ @Override
+ protected void writeText(final QuickWriter writer, final String text)
+ {
+ // escape special chars and remove invalid chars
+ final String sanitizedText = StringEscapeUtils.escapeXml10(text);
+
+ // don't call super.writeText() as this would escape the already escaped chars once more
+ writer.write(sanitizedText);
+ }
+}
\ No newline at end of file
diff --git a/src/main/resources/com/xceptance/xlt/report/scorecard/configuration-schema.json b/src/main/resources/com/xceptance/xlt/report/scorecard/configuration-schema.json
new file mode 100644
index 000000000..4d391a9e7
--- /dev/null
+++ b/src/main/resources/com/xceptance/xlt/report/scorecard/configuration-schema.json
@@ -0,0 +1,259 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://xlt.xceptance.com/schemas/scorecard-config.json",
+ "title": "XLT Scorecard Configuration",
+ "description": "Schema for XLT's Scorecard configuration files",
+ "type": "object",
+ "$defs": {
+ "boolDefaultFalse": {
+ "type": "boolean",
+ "default": false
+ },
+ "boolDefaultTrue": {
+ "type": "boolean",
+ "default": true
+ },
+ "nonEmptyString": {
+ "type": "string",
+ "minLength": 1
+ },
+ "messageDict": {
+ "type": "object",
+ "properties": {
+ "success": {
+ "type": "string"
+ },
+ "fail": {
+ "type": "string"
+ }
+ },
+ "minProperties": 1,
+ "additionalProperties": false
+ },
+ "testFailTrigger": {
+ "enum": [
+ "NOTPASSED",
+ "PASSED"
+ ],
+ "default": "NOTPASSED"
+ }
+ },
+ "properties": {
+ "comment": {
+ "type": "string"
+ },
+ "version": {
+ "type": "integer",
+ "minimum": 1
+ },
+ "selectors": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "id": {
+ "$ref": "#/$defs/nonEmptyString"
+ },
+ "expression": {
+ "$ref": "#/$defs/nonEmptyString"
+ },
+ "comment": {
+ "type": "string"
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "id",
+ "expression"
+ ]
+ }
+ },
+ "rules": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "id": {
+ "$ref": "#/$defs/nonEmptyString"
+ },
+ "enabled": {
+ "$ref": "#/$defs/boolDefaultTrue"
+ },
+ "comment": {
+ "type": "string"
+ },
+ "description": {
+ "type": "string"
+ },
+ "failsTest": {
+ "$ref": "#/$defs/boolDefaultFalse"
+ },
+ "failsOn": {
+ "$ref": "#/$defs/testFailTrigger"
+ },
+ "messages": {
+ "$ref": "#/$defs/messageDict"
+ },
+ "name": {
+ "$ref": "#/$defs/nonEmptyString"
+ },
+ "negateResult": {
+ "$ref": "#/$defs/boolDefaultFalse"
+ },
+ "checks": {
+ "type": "array",
+ "items": {
+ "oneOf": [
+ {
+ "properties": {
+ "selectorId": {
+ "$ref": "#/$defs/nonEmptyString"
+ }
+ },
+ "required": [
+ "selectorId"
+ ]
+ },
+ {
+ "properties": {
+ "selector": {
+ "$ref": "#/$defs/nonEmptyString"
+ }
+ },
+ "required": [
+ "selector"
+ ]
+ }
+ ],
+ "type": "object",
+ "properties": {
+ "enabled": {
+ "$ref": "#/$defs/boolDefaultTrue"
+ },
+ "displayValue": {
+ "$ref": "#/$defs/boolDefaultFalse"
+ },
+ "condition": {
+ "$ref": "#/$defs/nonEmptyString"
+ },
+ "comment": {
+ "type": "string"
+ }
+ },
+ "unevaluatedProperties": false,
+ "required": [
+ "condition"
+ ]
+ },
+ "uniqueItems": true
+ },
+ "points": {
+ "type": "integer",
+ "minimum": 0
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "id"
+ ]
+ },
+ "minItems": 1,
+ "uniqueItems": true
+ },
+ "groups": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "id": {
+ "$ref": "#/$defs/nonEmptyString"
+ },
+ "name": {
+ "$ref": "#/$defs/nonEmptyString"
+ },
+ "comment": {
+ "type": "string"
+ },
+ "description": {
+ "type": "string"
+ },
+ "enabled": {
+ "$ref": "#/$defs/boolDefaultTrue"
+ },
+ "failsTest": {
+ "$ref": "#/$defs/boolDefaultFalse"
+ },
+ "failsOn": {
+ "$ref": "#/$defs/testFailTrigger"
+ },
+ "messages": {
+ "$ref": "#/$defs/messageDict"
+ },
+ "mode": {
+ "enum": [
+ "firstPassed",
+ "lastPassed",
+ "allPassed"
+ ],
+ "default": "firstPassed"
+ },
+ "rules": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "minItems": 1,
+ "uniqueItems": true
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "id",
+ "rules"
+ ]
+ },
+ "minItems": 1,
+ "uniqueItems": true
+ },
+ "ratings": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "id": {
+ "$ref": "#/$defs/nonEmptyString"
+ },
+ "name": {
+ "$ref": "#/$defs/nonEmptyString"
+ },
+ "comment": {
+ "type": "string"
+ },
+ "description": {
+ "type": "string"
+ },
+ "value": {
+ "type": "number",
+ "minimum": 0.0,
+ "maximum": 100.0
+ },
+ "enabled": {
+ "$ref": "#/$defs/boolDefaultTrue"
+ },
+ "failsTest": {
+ "$ref": "#/$defs/boolDefaultFalse"
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "id",
+ "value"
+ ]
+ }
+ }
+ },
+ "required": [
+ "rules",
+ "groups"
+ ]
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ | Group | +Description | +Enabled | +Fails Test | +Mode | +Rule Count | +Result | +Message | +Max Points | +Points | +
|---|---|---|---|---|---|---|---|---|---|
| Totals | ++ | + | + | + | + | + | |||
|
+ |
+
+
+ |
+
+
+ |
+
+
+ |
+
+
+ |
+
+
+ |
+
+
+ |
+
+
+ |
+
+
+ |
+
+
+ |
+
| No data available | +|||||||||
+
+
+
+
+
+
+
+
+
+
+
+
+
+ | Rating | +Description | +Enabled | +Fails Test | +Value | +
+ |
+
+
+ |
+
+
+ |
+
+
+ |
+
+
+ |
+
|---|---|---|---|---|
| No data available | +||||
+
+
+
+
+
+
+
+
+
+ | Rule | +Check Number | +Enabled | +Display Value | +Selector | +Condition | +Value | +Result | +
|---|---|---|---|---|---|---|---|
|
+ |
+
+
+ |
+
+
+ |
+
+
+ |
+
+
+ |
+
+
+ |
+
+
+ |
+ |
| No data available | +|||||||
+
+
+
+
+
+
+
+
+
+ | Group | +Rule | +Description | +Enabled | +Fails Test | +Negate Result | +Check Count | +Result | +Message | +Max Points | +Points | +
|---|---|---|---|---|---|---|---|---|---|---|
|
+ |
+
+
+ |
+
+
+ |
+
+
+ |
+
+
+ |
+
+
+ |
+
+
+ |
+
+
+ |
+
+
+ |
+
+
+ |
+ |
| No data available | +||||||||||
+
+
+
+
+
+
+
+
+
+
+
+ The test could not be evaluated for the following reason:
+
+ +
+
+ The test got
+
+
+
+ succeeded with
+
+
+
+
+ would have succeeded with
+
+
+
+ would have succeeded with
+
+
+
+ .
+
+
+ failed with
+
+
+
+
+ The test failed due to a rule or group qualifying the test as failed.
+
+
+
+
+
+ Scorecard Result
+
+
+A performance test scorecard is a quick and easy way to evaluate a test result.
+
+
+Based on the total score and the maximum possible score, a percentage score is calculated that serves as the final verdict for the test.
+This summary page gives you the insight you need into the assessment and helps you to derive recommendations for next steps. It also helps you identifying areas for improvement.
+Rating
+
+
+A rating defines the result of the scoring. It is basically a quick verdict on the created scorecard.
+
+
+A score defines a verbal test result and can also mark the test as failed if necessary. The score uses the percentage value of the points achieved versus the maximum points achievable to calculate the score. The rating value defines the upper limit for the application of this score.
+Rules
+
+
+Rules consist of data checks on the test result. A rule must be assigned to at least one group to be effective.
+
+
+A rule passes if all its conditions evaluate to true, otherwise it fails.
If a rule is marked as "Error", the evaluation statement was incorrect and the rule is ignored. A rule can mark the entire test as failed, regardless of the rating result, if the "fails test" attribute is set to true.
Rule Checks
+
+
+Rule checks act as queries against the load test result data.
+
+
+Any data provided by XLT in its XML result format can be used to create rule checks. A check must be associated with a rule to be effective, and it can be used in more than one rule. A check consists of a selector against the XML document and a condition that it must satisfy. This table lists the details of the rule and its results within the group to which it is assigned.
+Groups
+
+
+Groups define which rules are evaluated and how the overall result of multiple related rules is determined. A rule may be used in more than one group. Each group requires at least one rule to be considered.
+