From 5d412029f70abb67866807d6069a6843bc9d746b Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Tue, 20 Dec 2022 19:30:00 -0500 Subject: [PATCH 01/28] global log4j config (#23631) --- bin/setclasspath.bat | 104 +++++++++++++++++ bin/setclasspath.sh | 105 ++++++++++++++++++ dotCMS/build.gradle | 33 +++++- dotCMS/dependencies.gradle | 20 ++-- dotCMS/gradle.properties | 1 + .../listeners/ContextLifecycleListener.java | 10 +- .../com/dotmarketing/loggers/Log4jUtil.java | 3 +- dotCMS/src/main/resources/log4j2-tomcat.xml | 41 +++++++ 8 files changed, 305 insertions(+), 12 deletions(-) create mode 100644 bin/setclasspath.bat create mode 100755 bin/setclasspath.sh create mode 100644 dotCMS/src/main/resources/log4j2-tomcat.xml diff --git a/bin/setclasspath.bat b/bin/setclasspath.bat new file mode 100644 index 000000000000..a27956b13b90 --- /dev/null +++ b/bin/setclasspath.bat @@ -0,0 +1,104 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +rem --------------------------------------------------------------------------- +rem Set JAVA_HOME or JRE_HOME if not already set, ensure any provided settings +rem are valid and consistent with the selected start-up options and set up the +rem endorsed directory. +rem --------------------------------------------------------------------------- + +rem Make sure prerequisite environment variables are set + +rem In debug mode we need a real JDK (JAVA_HOME) +if ""%1"" == ""debug"" goto needJavaHome + +rem Otherwise either JRE or JDK are fine +if not "%JRE_HOME%" == "" goto gotJreHome +if not "%JAVA_HOME%" == "" goto gotJavaHome +echo Neither the JAVA_HOME nor the JRE_HOME environment variable is defined +echo At least one of these environment variable is needed to run this program +goto exit + +:needJavaHome +rem Check if we have a usable JDK +if "%JAVA_HOME%" == "" goto noJavaHome +if not exist "%JAVA_HOME%\bin\java.exe" goto noJavaHome +if not exist "%JAVA_HOME%\bin\jdb.exe" goto noJavaHome +if not exist "%JAVA_HOME%\bin\javac.exe" goto noJavaHome +set "JRE_HOME=%JAVA_HOME%" +goto okJava + +:noJavaHome +echo The JAVA_HOME environment variable is not defined correctly. +echo It is needed to run this program in debug mode. +echo NB: JAVA_HOME should point to a JDK not a JRE. +goto exit + +:gotJavaHome +rem No JRE given, use JAVA_HOME as JRE_HOME +set "JRE_HOME=%JAVA_HOME%" + +:gotJreHome +rem Check if we have a usable JRE +if not exist "%JRE_HOME%\bin\java.exe" goto noJreHome +goto okJava + +:noJreHome +rem Needed at least a JRE +echo The JRE_HOME environment variable is not defined correctly +echo This environment variable is needed to run this program +goto exit + +:okJava +rem Don't override the endorsed dir if the user has set it previously +if not "%JAVA_ENDORSED_DIRS%" == "" goto gotEndorseddir +rem Java 9 no longer supports the java.endorsed.dirs +rem system property. Only try to use it if +rem CATALINA_HOME/endorsed exists. +if not exist "%CATALINA_HOME%\endorsed" goto gotEndorseddir +set "JAVA_ENDORSED_DIRS=%CATALINA_HOME%\endorsed" +:gotEndorseddir + +rem Don't override _RUNJAVA if the user has set it previously +if not "%_RUNJAVA%" == "" goto gotRunJava +rem Set standard command for invoking Java. +rem Also note the quoting as JRE_HOME may contain spaces. +set _RUNJAVA="%JRE_HOME%\bin\java.exe" +:gotRunJava + +rem Don't override _RUNJDB if the user has set it previously +rem Also note the quoting as JAVA_HOME may contain spaces. +if not "%_RUNJDB%" == "" goto gotRunJdb +set _RUNJDB="%JAVA_HOME%\bin\jdb.exe" +:gotRunJdb + +if not "%USE_GLOBAL_LOG4J" == "" goto gotGlobalLog4j +if [ "$USE_GLOBAL_LOG4J" = true ]; then + echo Using Global Log4j + set CLASSPATH="%CATALINA_HOME%\log4j2\lib\*;%CATALINA_HOME%\log4j2\conf$CLASSPATH" + set JAVA_OPTS=" -DLog4jContextSelector=org.apache.logging.log4j.core.async.BasicAsyncLoggerContextSelector %JAVA_OPTS%" + set JAVA_OPTS=" -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager %JAVA_OPTS%" +fi +:gotGlobalLog4j + + +goto end + +:exit +exit /b 1 + +:end +exit /b 0 diff --git a/bin/setclasspath.sh b/bin/setclasspath.sh new file mode 100755 index 000000000000..4980d3164a88 --- /dev/null +++ b/bin/setclasspath.sh @@ -0,0 +1,105 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ----------------------------------------------------------------------------- +# Set JAVA_HOME or JRE_HOME if not already set, ensure any provided settings +# are valid and consistent with the selected start-up options and set up the +# endorsed directory. +# ----------------------------------------------------------------------------- + +# Make sure prerequisite environment variables are set +if [ -z "$JAVA_HOME" ] && [ -z "$JRE_HOME" ]; then + if $darwin; then + # Bugzilla 54390 + if [ -x '/usr/libexec/java_home' ] ; then + export JAVA_HOME=`/usr/libexec/java_home` + # Bugzilla 37284 (reviewed). + elif [ -d "/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home" ]; then + export JAVA_HOME="/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home" + fi + else + JAVA_PATH=`which java 2>/dev/null` + if [ "x$JAVA_PATH" != "x" ]; then + JAVA_PATH=`dirname "$JAVA_PATH" 2>/dev/null` + JRE_HOME=`dirname "$JAVA_PATH" 2>/dev/null` + fi + if [ "x$JRE_HOME" = "x" ]; then + # XXX: Should we try other locations? + if [ -x /usr/bin/java ]; then + JRE_HOME=/usr + fi + fi + fi + if [ -z "$JAVA_HOME" ] && [ -z "$JRE_HOME" ]; then + echo "Neither the JAVA_HOME nor the JRE_HOME environment variable is defined" + echo "At least one of these environment variable is needed to run this program" + exit 1 + fi +fi +if [ -z "$JAVA_HOME" ] && [ "$1" = "debug" ]; then + echo "JAVA_HOME should point to a JDK in order to run in debug mode." + exit 1 +fi +if [ -z "$JRE_HOME" ]; then + JRE_HOME="$JAVA_HOME" +fi + +# If we're running under jdb, we need a full jdk. +if [ "$1" = "debug" ] ; then + if [ "$os400" = "true" ]; then + if [ ! -x "$JAVA_HOME"/bin/java ] || [ ! -x "$JAVA_HOME"/bin/javac ]; then + echo "The JAVA_HOME environment variable is not defined correctly" + echo "This environment variable is needed to run this program" + echo "NB: JAVA_HOME should point to a JDK not a JRE" + exit 1 + fi + else + if [ ! -x "$JAVA_HOME"/bin/java ] || [ ! -x "$JAVA_HOME"/bin/jdb ] || [ ! -x "$JAVA_HOME"/bin/javac ]; then + echo "The JAVA_HOME environment variable is not defined correctly" + echo "This environment variable is needed to run this program" + echo "NB: JAVA_HOME should point to a JDK not a JRE" + exit 1 + fi + fi +fi + +# Don't override the endorsed dir if the user has set it previously +if [ -z "$JAVA_ENDORSED_DIRS" ]; then + # Java 9 no longer supports the java.endorsed.dirs + # system property. Only try to use it if + # CATALINA_HOME/endorsed exists. + if [ -d "$CATALINA_HOME"/endorsed ]; then + JAVA_ENDORSED_DIRS="$CATALINA_HOME"/endorsed + fi +fi + +# Set standard commands for invoking Java, if not already set. +if [ -z "$_RUNJAVA" ]; then + _RUNJAVA="$JRE_HOME"/bin/java +fi +if [ "$os400" != "true" ]; then + if [ -z "$_RUNJDB" ]; then + _RUNJDB="$JAVA_HOME"/bin/jdb + fi +fi + echo In set classpath +if [ "$USE_GLOBAL_LOG4J" = true ]; then + echo Using Global Log4j + CLASSPATH="$CATALINA_HOME/log4j2/lib/*:$CATALINA_HOME/log4j2/conf$CLASSPATH" + JAVA_OPTS=" -DLog4jContextSelector=org.apache.logging.log4j.core.async.BasicAsyncLoggerContextSelector $JAVA_OPTS" + JAVA_OPTS=" -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager $JAVA_OPTS" +fi \ No newline at end of file diff --git a/dotCMS/build.gradle b/dotCMS/build.gradle index df363d636487..8c2490697325 100644 --- a/dotCMS/build.gradle +++ b/dotCMS/build.gradle @@ -493,12 +493,37 @@ class DeployWarTomcatTask extends DefaultTask{ } project.copy{ - from '../libs/buildlibs' + from project.file("../libs/buildlibs") into buildLibLocation + '/lib' include 'mail.jar' include 'javax.activation-1.2.0.jar' } + + project.copy { + from "$project.buildDir/resources/main" + into buildLibLocation + '/log4j2/conf' + include "log4j2-tomcat*.xml" + } + + project.copy{ + from project.configurations.compileClasspath + into buildLibLocation + '/log4j2/lib' + include 'commons-logging-*.jar' + include 'log4j-*.jar' + include 'disruptor-*.jar' + include 'slf4j-api-*.jar' + exclude 'log4j-web-*.jar' + } + + project.copy{ + from project.file("../bin") + into buildLibLocation + '/bin' + include 'setclasspath.sh' + include 'setclasspath.bat' + } + + } } @@ -904,6 +929,12 @@ task createDistPrep(dependsOn: ['clonePullTomcatDist', 'deployWarTomcatDist']) { exclude "com", "org", "release.properties" } + copy { + from "$rootDir/../conf/log4j" + into "$distLocation$log4j2ConfDistLocation" + include "log4j*.xml" + } + copy { into "$distLocation$pluginsDistLocation" from "src/main/plugins" diff --git a/dotCMS/dependencies.gradle b/dotCMS/dependencies.gradle index 20177924ec51..364471238e4a 100644 --- a/dotCMS/dependencies.gradle +++ b/dotCMS/dependencies.gradle @@ -5,6 +5,8 @@ dependencies { def immutablesVersion = "2.9.1" def byteBuddyVersion = "1.12.6" + def log4jVersion = "2.19.0" + annotationProcessor "org.immutables:value:$immutablesVersion" // <--- this is important implementation('junit:junit:4.13.2') { @@ -49,15 +51,19 @@ dependencies { implementation group: 'com.dotcms.lib', name: 'dot.counter-ejb', version:'ukv_2' implementation group: 'com.dotcms.lib', name: 'dot.dwr', version:'3rc2modified_3' - implementation group: 'com.lmax', name: 'disruptor', version: '3.3.4' + implementation group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1' implementation group: 'org.apache.commons', name: 'commons-numbers-gamma', version: '1.0' - implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.17.1' - implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.17.1' - implementation group: 'org.apache.logging.log4j', name: 'log4j-1.2-api', version: '2.17.1' - implementation group: 'org.apache.logging.log4j', name: 'log4j-jcl', version: '2.17.1' - implementation group: 'org.apache.logging.log4j', name: 'log4j-web', version: '2.17.1' - implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j-impl', version: '2.17.1' + + implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: log4jVersion + implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: log4jVersion + implementation group: 'org.apache.logging.log4j', name: 'log4j-1.2-api', version: log4jVersion + implementation group: 'org.apache.logging.log4j', name: 'log4j-jcl', version: log4jVersion + implementation group: 'org.apache.logging.log4j', name: 'log4j-web', version: log4jVersion + implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j-impl', version: log4jVersion + implementation group: 'org.apache.logging.log4j', name: 'log4j-jul', version: log4jVersion + implementation group: 'org.apache.logging.log4j', name: 'log4j-appserver', version: log4jVersion + implementation group: 'com.lmax', name: 'disruptor', version: '3.3.4' implementation group: 'com.dotcms.lib', name: 'dot.fileupload-ext', version:'ukv_3' implementation group: 'fop', name: 'fop', version:'0.20.3' diff --git a/dotCMS/gradle.properties b/dotCMS/gradle.properties index a4f9e914ca92..d01f76cd1ee4 100644 --- a/dotCMS/gradle.properties +++ b/dotCMS/gradle.properties @@ -29,6 +29,7 @@ distBinLocation=/bin pluginsDistLocation=/plugins docsDistLocation=/docs confDistLocation=/bin/system/src-conf +log4j2ConfDistLocation=/log4j2/conf tomcatDistInstallLocation=dotserver/tomcat outputDistLocation=../dist-output/ distUpdate=false diff --git a/dotCMS/src/main/java/com/dotmarketing/listeners/ContextLifecycleListener.java b/dotCMS/src/main/java/com/dotmarketing/listeners/ContextLifecycleListener.java index 3f123b49a89e..075b240bf992 100644 --- a/dotCMS/src/main/java/com/dotmarketing/listeners/ContextLifecycleListener.java +++ b/dotCMS/src/main/java/com/dotmarketing/listeners/ContextLifecycleListener.java @@ -10,9 +10,9 @@ import com.dotmarketing.loggers.Log4jUtil; import com.dotmarketing.quartz.QuartzUtils; import com.dotmarketing.util.Config; -import com.dotmarketing.util.ConfigUtils; import com.dotmarketing.util.Logger; import io.vavr.control.Try; +import org.apache.logging.log4j.core.async.BasicAsyncLoggerContextSelector; import javax.servlet.ServletContext; import javax.servlet.ServletContextEvent; @@ -77,8 +77,12 @@ public void contextInitialized(ServletContextEvent arg0) { Logger.error(this,e.getMessage(),e); } - //Initialises/reconfigures log4j based on a given log4j configuration file - Log4jUtil.initializeFromPath(path); + // Do not reconfigure if using global configuration. Remove this if we move + // a full global configuration + if (System.getProperty("Log4jContextSelector").equals(BasicAsyncLoggerContextSelector.class.getName())) + Log4jUtil.initializeFromPath(path); + else + Logger.debug(this, "Reinitializing configuration from "+path); installWebSocket(arg0.getServletContext()); diff --git a/dotCMS/src/main/java/com/dotmarketing/loggers/Log4jUtil.java b/dotCMS/src/main/java/com/dotmarketing/loggers/Log4jUtil.java index ac941ba8604d..d4757718631d 100644 --- a/dotCMS/src/main/java/com/dotmarketing/loggers/Log4jUtil.java +++ b/dotCMS/src/main/java/com/dotmarketing/loggers/Log4jUtil.java @@ -51,7 +51,7 @@ public static void createAndAddConsoleAppender () { Configuration configuration = loggerContext.getConfiguration(); //Init log4j to see the messages in ant's output - if ( !appenderMap.isEmpty() ) { + if ( appenderMap.isEmpty() ) { //Create a simple layout for our appender Layout simpleLayout = PatternLayout.newBuilder() @@ -61,6 +61,7 @@ public static void createAndAddConsoleAppender () { //Create and add a console appender to the configuration ConsoleAppender consoleAppender = ConsoleAppender.createDefaultAppenderForLayout(simpleLayout); configuration.addAppender(consoleAppender); + LogManager.getLogger().debug("Added a new ConsoleAppender to the log4j configuration"); } } diff --git a/dotCMS/src/main/resources/log4j2-tomcat.xml b/dotCMS/src/main/resources/log4j2-tomcat.xml new file mode 100644 index 000000000000..a23e0f0f97e9 --- /dev/null +++ b/dotCMS/src/main/resources/log4j2-tomcat.xml @@ -0,0 +1,41 @@ + + + + + + + ${sys:catalina.base}/logs + %d %highlight{[%-6p]} [%t] %c{1.} - %m%n + + + + + + + + + + + + + + + + + + + + + + From 50ff092017c8238ba25ed2361f73176ff3365fff Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Wed, 21 Dec 2022 11:20:57 -0500 Subject: [PATCH 02/28] formatting fixes --- .../listeners/ContextLifecycleListener.java | 62 ++++++++++--------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/dotCMS/src/main/java/com/dotmarketing/listeners/ContextLifecycleListener.java b/dotCMS/src/main/java/com/dotmarketing/listeners/ContextLifecycleListener.java index 075b240bf992..d5f8519ee9d8 100644 --- a/dotCMS/src/main/java/com/dotmarketing/listeners/ContextLifecycleListener.java +++ b/dotCMS/src/main/java/com/dotmarketing/listeners/ContextLifecycleListener.java @@ -21,79 +21,81 @@ import java.io.File; /** - * * @author Andres Olarte - * */ public class ContextLifecycleListener implements ServletContextListener { - public ContextLifecycleListener() { - AsciiArt.doArt(); - } + public ContextLifecycleListener() { + AsciiArt.doArt(); + } public void contextDestroyed(ServletContextEvent arg0) { Logger.info(this, "Shutdown : Started, executing a clean shutdown."); Try.run(() -> QuartzUtils.stopSchedulers()) - .onFailure(e -> Logger.warn(ContextLifecycleListener.class, "Shutdown : " + e.getMessage())); - + .onFailure(e -> Logger.warn(ContextLifecycleListener.class, + "Shutdown : " + e.getMessage())); + Try.run(() -> LicenseUtil.freeLicenseOnRepo()) - .onFailure(e -> Logger.warn(ContextLifecycleListener.class, "Shutdown : " + e.getMessage())); + .onFailure(e -> Logger.warn(ContextLifecycleListener.class, + "Shutdown : " + e.getMessage())); - Try.run(() -> CacheLocator.getCacheAdministrator().shutdown()) - .onFailure(e -> Logger.warn(ContextLifecycleListener.class, "Shutdown : " + e.getMessage())); - - + .onFailure(e -> Logger.warn(ContextLifecycleListener.class, + "Shutdown : " + e.getMessage())); Try.run(() -> DotConcurrentFactory.getInstance().shutdownAndDestroy()) - .onFailure(e -> Logger.warn(ContextLifecycleListener.class, "Shutdown : " + e.getMessage())); + .onFailure(e -> Logger.warn(ContextLifecycleListener.class, + "Shutdown : " + e.getMessage())); Try.run(() -> ReindexThread.stopThread()) - .onFailure(e -> Logger.warn(ContextLifecycleListener.class, "Shutdown : " + e.getMessage())); + .onFailure(e -> Logger.warn(ContextLifecycleListener.class, + "Shutdown : " + e.getMessage())); Logger.info(this, "Shutdown : Finished."); } - public void contextInitialized(ServletContextEvent arg0) { + public void contextInitialized(ServletContextEvent arg0) { ByteBuddyFactory.init(); - Config.setMyApp(arg0.getServletContext()); - + Config.setMyApp(arg0.getServletContext()); String path = null; - try { + try { String contextPath = Config.CONTEXT_PATH; - if ( !contextPath.endsWith( File.separator ) ) { + if (!contextPath.endsWith(File.separator)) { contextPath += File.separator; } - File file = new File(contextPath + "WEB-INF" + File.separator + "log4j" + File.separator + "log4j2.xml"); - path = file.toURI().toString(); + File file = new File(contextPath + "WEB-INF" + File.separator + "log4j" + File.separator + + "log4j2.xml"); + path = file.toURI().toString(); } catch (Exception e) { - Logger.error(this,e.getMessage(),e); - } + Logger.error(this, e.getMessage(), e); + } - // Do not reconfigure if using global configuration. Remove this if we move + // Do not reconfigure if using global configuration. Remove this if we move // a full global configuration - if (System.getProperty("Log4jContextSelector").equals(BasicAsyncLoggerContextSelector.class.getName())) + if (System.getProperty("Log4jContextSelector") + .equals(BasicAsyncLoggerContextSelector.class.getName())) { Log4jUtil.initializeFromPath(path); - else - Logger.debug(this, "Reinitializing configuration from "+path); - + } else { + Logger.debug(this, "Reinitializing configuration from " + path); + } installWebSocket(arg0.getServletContext()); - } + } private void installWebSocket(final ServletContext serverContext) { try { Logger.info(this, "Installing the web socket"); - var container = (ServerContainer) serverContext.getAttribute("javax.websocket.server.ServerContainer"); + var container = (ServerContainer) serverContext.getAttribute( + "javax.websocket.server.ServerContainer"); container.addEndpoint(SystemEventsWebSocketEndPoint.class); } catch (Throwable e) { From 6c108b16c41f49a701156f19efacf595d42d84a6 Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Thu, 22 Dec 2022 11:14:27 -0500 Subject: [PATCH 03/28] Add null check for system property --- .../listeners/ContextLifecycleListener.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/dotCMS/src/main/java/com/dotmarketing/listeners/ContextLifecycleListener.java b/dotCMS/src/main/java/com/dotmarketing/listeners/ContextLifecycleListener.java index d5f8519ee9d8..5b360fb05c07 100644 --- a/dotCMS/src/main/java/com/dotmarketing/listeners/ContextLifecycleListener.java +++ b/dotCMS/src/main/java/com/dotmarketing/listeners/ContextLifecycleListener.java @@ -12,6 +12,7 @@ import com.dotmarketing.util.Config; import com.dotmarketing.util.Logger; import io.vavr.control.Try; +import java.util.Optional; import org.apache.logging.log4j.core.async.BasicAsyncLoggerContextSelector; import javax.servlet.ServletContext; @@ -19,6 +20,7 @@ import javax.servlet.ServletContextListener; import javax.websocket.server.ServerContainer; import java.io.File; +import org.apache.logging.log4j.core.selector.BasicContextSelector; /** * @author Andres Olarte @@ -79,11 +81,13 @@ public void contextInitialized(ServletContextEvent arg0) { // Do not reconfigure if using global configuration. Remove this if we move // a full global configuration - if (System.getProperty("Log4jContextSelector") - .equals(BasicAsyncLoggerContextSelector.class.getName())) { - Log4jUtil.initializeFromPath(path); - } else { + // This is called infrequently on context startup so do not need to cache + // getting system property + String log4jContextSelector = System.getProperty("Log4jContextSelector"); + if (log4jContextSelector != null && log4jContextSelector.equals(BasicAsyncLoggerContextSelector.class.getName()) + || log4jContextSelector.equals(BasicContextSelector.class.getName())) { Logger.debug(this, "Reinitializing configuration from " + path); + Log4jUtil.initializeFromPath(path); } installWebSocket(arg0.getServletContext()); From 723538525549de4160de1c74baaeb137c189032f Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Tue, 31 Jan 2023 14:10:41 -0500 Subject: [PATCH 04/28] Fixup null pointer and cleanup sonar lint --- .../listeners/ContextLifecycleListener.java | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/dotCMS/src/main/java/com/dotmarketing/listeners/ContextLifecycleListener.java b/dotCMS/src/main/java/com/dotmarketing/listeners/ContextLifecycleListener.java index 5b360fb05c07..368cfe7c8748 100644 --- a/dotCMS/src/main/java/com/dotmarketing/listeners/ContextLifecycleListener.java +++ b/dotCMS/src/main/java/com/dotmarketing/listeners/ContextLifecycleListener.java @@ -12,7 +12,6 @@ import com.dotmarketing.util.Config; import com.dotmarketing.util.Logger; import io.vavr.control.Try; -import java.util.Optional; import org.apache.logging.log4j.core.async.BasicAsyncLoggerContextSelector; import javax.servlet.ServletContext; @@ -27,6 +26,8 @@ */ public class ContextLifecycleListener implements ServletContextListener { + public static final String SHUTDOWN = "Shutdown : "; + public ContextLifecycleListener() { AsciiArt.doArt(); } @@ -34,25 +35,25 @@ public ContextLifecycleListener() { public void contextDestroyed(ServletContextEvent arg0) { Logger.info(this, "Shutdown : Started, executing a clean shutdown."); - Try.run(() -> QuartzUtils.stopSchedulers()) + Try.run(QuartzUtils::stopSchedulers) .onFailure(e -> Logger.warn(ContextLifecycleListener.class, - "Shutdown : " + e.getMessage())); + SHUTDOWN + e.getMessage())); - Try.run(() -> LicenseUtil.freeLicenseOnRepo()) + Try.run(LicenseUtil::freeLicenseOnRepo) .onFailure(e -> Logger.warn(ContextLifecycleListener.class, - "Shutdown : " + e.getMessage())); + SHUTDOWN + e.getMessage())); Try.run(() -> CacheLocator.getCacheAdministrator().shutdown()) .onFailure(e -> Logger.warn(ContextLifecycleListener.class, - "Shutdown : " + e.getMessage())); + SHUTDOWN + e.getMessage())); Try.run(() -> DotConcurrentFactory.getInstance().shutdownAndDestroy()) .onFailure(e -> Logger.warn(ContextLifecycleListener.class, - "Shutdown : " + e.getMessage())); + SHUTDOWN + e.getMessage())); - Try.run(() -> ReindexThread.stopThread()) + Try.run(ReindexThread::stopThread) .onFailure(e -> Logger.warn(ContextLifecycleListener.class, - "Shutdown : " + e.getMessage())); + SHUTDOWN + e.getMessage())); Logger.info(this, "Shutdown : Finished."); @@ -84,8 +85,8 @@ public void contextInitialized(ServletContextEvent arg0) { // This is called infrequently on context startup so do not need to cache // getting system property String log4jContextSelector = System.getProperty("Log4jContextSelector"); - if (log4jContextSelector != null && log4jContextSelector.equals(BasicAsyncLoggerContextSelector.class.getName()) - || log4jContextSelector.equals(BasicContextSelector.class.getName())) { + if (log4jContextSelector != null && (log4jContextSelector.equals(BasicAsyncLoggerContextSelector.class.getName()) + || log4jContextSelector.equals(BasicContextSelector.class.getName()))) { Logger.debug(this, "Reinitializing configuration from " + path); Log4jUtil.initializeFromPath(path); } From d29c14b8ad4ad53a6e3f1bec762ba1106a6c8f72 Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Wed, 1 Feb 2023 16:27:43 -0500 Subject: [PATCH 05/28] Fixup Caffeine cache concurrency --- .../cache/provider/caffine/CaffineCache.java | 301 ++++++++---------- 1 file changed, 132 insertions(+), 169 deletions(-) diff --git a/dotCMS/src/main/java/com/dotmarketing/business/cache/provider/caffine/CaffineCache.java b/dotCMS/src/main/java/com/dotmarketing/business/cache/provider/caffine/CaffineCache.java index 672d0013fd50..6a05898bbbec 100644 --- a/dotCMS/src/main/java/com/dotmarketing/business/cache/provider/caffine/CaffineCache.java +++ b/dotCMS/src/main/java/com/dotmarketing/business/cache/provider/caffine/CaffineCache.java @@ -13,29 +13,28 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.Expiry; -import com.github.benmanes.caffeine.cache.LoadingCache; import com.google.common.collect.ImmutableSet; import io.vavr.control.Try; import java.text.DecimalFormat; import java.text.NumberFormat; import java.util.HashSet; import java.util.Iterator; -import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; /** - * In-Memory Cache implementation using https://github.com/ben-manes/caffeine - * + * In-Memory Cache implementation using ... + *

* Supports key-specific time invalidations by providing an {@link Expirable} object * in the {@link #put(String, String, Object)} method with the desired TTL. - * + *

* A group-wide invalidation time can also be set by config properties. - * + *

* i.e., for the "graphqlquerycache" group - * + *

* cache.graphqlquerycache.chain=com.dotmarketing.business.cache.provider.caffine.CaffineCache * cache.graphqlquerycache.seconds=15 * @@ -43,8 +42,12 @@ public class CaffineCache extends CacheProvider { private static final long serialVersionUID = 1348649382678659786L; + public static final String SECONDS = ".seconds"; + public static final String SIZE = "size"; + public static final String CACHE = "cache."; - private Boolean isInitialized = false; + private AtomicBoolean isInitialized=new AtomicBoolean(false); + private AtomicBoolean initializing=new AtomicBoolean(false); static final String DEFAULT_CACHE = CacheProviderAPI.DEFAULT_CACHE; @@ -70,36 +73,40 @@ public boolean isDistributed() { @Override public void init() { - HashSet _availableCaches = new HashSet<>(); - _availableCaches.add(DEFAULT_CACHE); + if(!isInitialized.get() && initializing.compareAndSet(false, true)) { - Iterator it = Config.getKeys(); - while (it.hasNext()) { + HashSet tempAvailableCaches = new HashSet<>(); + tempAvailableCaches.add(DEFAULT_CACHE); - String key = it.next(); - if (key == null) { - continue; - } - - if (key.startsWith("cache.")) { + Iterator it = Config.getKeys(); + while (it.hasNext()) { - String cacheName = key.split("\\.")[1]; - if (key.endsWith(".size")) { - int inMemory = Config.getIntProperty(key, 0); - _availableCaches.add(cacheName.toLowerCase()); - Logger.info(this.getClass(), - "***\t Cache Config Memory : " + cacheName + ": " + inMemory); + String key = it.next(); + if (key == null) { + continue; } + if (key.startsWith(CACHE)) { + + String cacheName = key.split("\\.")[1]; + if (key.endsWith("." + SIZE)) { + int inMemory = Config.getIntProperty(key, 0); + tempAvailableCaches.add(cacheName.toLowerCase()); + Logger.debug(this.getClass(), + "***\t Cache Config Memory : " + cacheName + ": " + inMemory); + } + + } } + this.availableCaches = ImmutableSet.copyOf(tempAvailableCaches); + isInitialized.set(true); + initializing.set(false); } - this.availableCaches = ImmutableSet.copyOf(_availableCaches); - isInitialized = true; } @Override public boolean isInitialized() throws Exception { - return isInitialized; + return isInitialized.get(); } @Override @@ -121,195 +128,151 @@ public Object get(String group, String key) { @Override public void remove(String group) { - // Get the cache for the given group - Cache cache = getCache(group); - - // Invalidates the Cache for the given group - cache.invalidateAll(); - - // Remove this group from the global list of cache groups - groups.remove(group); + groups.computeIfPresent(group, (k, v) -> { + v.invalidateAll(); + return null;}); } @Override public void remove(String group, String key) { - - // Get the cache for the given group - Cache cache = getCache(group); - - // Invalidates from Cache a key from a given group - cache.invalidate(key); + getCache(group).invalidate(key); } @Override public void removeAll() { - - Set currentGroups = new HashSet<>(); - currentGroups.addAll(getGroups()); - - for (String group : currentGroups) { - remove(group); - } - - groups.clear(); + groups.forEach((k, v) -> v.invalidateAll()); } @Override public Set getGroups() { - return groups.keySet(); + return Set.copyOf(groups.keySet()); } @Override public Set getKeys(String group) { - - Set keys = new HashSet<>(); - - Cache cache = getCache(group); - Map m = cache.asMap(); - - if (m != null) { - keys = m.keySet(); - } - - return keys; + return Set.copyOf(getCache(group).asMap().keySet()); } @Override public CacheProviderStats getStats() { CacheStats providerStats = new CacheStats(); - CacheProviderStats ret = new CacheProviderStats(providerStats,getName()); - - Set currentGroups = new HashSet<>(); - currentGroups.addAll(getGroups()); - NumberFormat nf = DecimalFormat.getInstance(); - DecimalFormat pf = new DecimalFormat("##.##%"); - - CacheSizingUtil sizer = new CacheSizingUtil(); - - - - for (String group : currentGroups) { - final CacheStats stats = new CacheStats(); + final CacheProviderStats ret = new CacheProviderStats(providerStats,getName()); - final Cache foundCache = getCache(group); + final NumberFormat nf = NumberFormat.getInstance(); + final DecimalFormat pf = new DecimalFormat("##.##%"); + final CacheSizingUtil sizer = new CacheSizingUtil(); - final boolean isDefault = (Config.getIntProperty("cache." + group + ".size", -1) == -1); + groups.forEach(1, (group, foundCache) -> processCacheStats(ret, nf, pf, sizer, group, foundCache) + ); + return ret; + } - final int size = isDefault ? Config.getIntProperty("cache." + DEFAULT_CACHE + ".size") - : (Config.getIntProperty("cache." + group + ".size", -1) != -1) - ? Config.getIntProperty("cache." + group + ".size") - : Config.getIntProperty("cache." + DEFAULT_CACHE + ".size"); - - final int seconds = isDefault ? Config.getIntProperty("cache." + DEFAULT_CACHE + ".seconds", 100) - : (Config.getIntProperty("cache." + group + ".seconds", -1) != -1) - ? Config.getIntProperty("cache." + group + ".seconds") - : Config.getIntProperty("cache." + DEFAULT_CACHE + ".seconds", 100); - - com.github.benmanes.caffeine.cache.stats.CacheStats cstats = foundCache.stats(); - stats.addStat(CacheStats.REGION, group); - stats.addStat(CacheStats.REGION_DEFAULT, isDefault + ""); - stats.addStat(CacheStats.REGION_CONFIGURED_SIZE, "size:" + nf.format(size) + " / " + seconds + "s"); - stats.addStat(CacheStats.REGION_SIZE, nf.format(foundCache.estimatedSize())); - stats.addStat(CacheStats.REGION_LOAD, nf.format(cstats.missCount()+cstats.hitCount())); - stats.addStat(CacheStats.REGION_HITS, nf.format(cstats.hitCount())); - stats.addStat(CacheStats.REGION_HIT_RATE, pf.format(cstats.hitRate())); - stats.addStat(CacheStats.REGION_AVG_LOAD_TIME, nf.format(cstats.averageLoadPenalty()/1000000) + " ms"); - stats.addStat(CacheStats.REGION_EVICTIONS, nf.format(cstats.evictionCount())); - - long averageObjectSize = Try.of(()-> sizer.averageSize(foundCache.asMap())).getOrElse(-1L); - long totalObjectSize = averageObjectSize * foundCache.estimatedSize(); + private static void processCacheStats(CacheProviderStats ret, NumberFormat nf, DecimalFormat pf, + CacheSizingUtil sizer, String group, Cache foundCache) { + final CacheStats stats = new CacheStats(); + boolean isDefault=false; + int size=Config.getIntProperty(CACHE + group + "." + SIZE, -1); + if (size==-1) + { + isDefault=true; + size = Config.getIntProperty(CACHE + DEFAULT_CACHE + "." + SIZE,-1); + } - stats.addStat(CacheStats.REGION_MEM_PER_OBJECT, "

" + String.format("%010d", averageObjectSize) + "
" + UtilMethods.prettyByteify(averageObjectSize)); - stats.addStat(CacheStats.REGION_MEM_TOTAL_PRETTY, "
" + String.format("%010d", totalObjectSize) + "
" + UtilMethods.prettyByteify(totalObjectSize)); - ret.addStatRecord(stats); + int seconds=Config.getIntProperty(CACHE + group + SECONDS, -1); + if (seconds==-1) { + seconds = Config.getIntProperty(CACHE + DEFAULT_CACHE + SECONDS, 100); } - return ret; + com.github.benmanes.caffeine.cache.stats.CacheStats cstats = foundCache.stats(); + stats.addStat(CacheStats.REGION, group); + stats.addStat(CacheStats.REGION_DEFAULT, isDefault + ""); + stats.addStat(CacheStats.REGION_CONFIGURED_SIZE, SIZE + ":" + nf.format(size) + " / " + seconds + "s"); + stats.addStat(CacheStats.REGION_SIZE, nf.format(foundCache.estimatedSize())); + stats.addStat(CacheStats.REGION_LOAD, nf.format(cstats.missCount()+cstats.hitCount())); + stats.addStat(CacheStats.REGION_HITS, nf.format(cstats.hitCount())); + stats.addStat(CacheStats.REGION_HIT_RATE, pf.format(cstats.hitRate())); + stats.addStat(CacheStats.REGION_AVG_LOAD_TIME, nf.format(cstats.averageLoadPenalty()/1000000) + " ms"); + stats.addStat(CacheStats.REGION_EVICTIONS, nf.format(cstats.evictionCount())); + + long averageObjectSize = Try.of(()-> sizer.averageSize(foundCache.asMap())).getOrElse(-1L); + long totalObjectSize = averageObjectSize * foundCache.estimatedSize(); + + stats.addStat(CacheStats.REGION_MEM_PER_OBJECT, "
" + String.format("%010d", averageObjectSize) + "
" + UtilMethods.prettyByteify(averageObjectSize)); + stats.addStat(CacheStats.REGION_MEM_TOTAL_PRETTY, "
" + String.format("%010d", totalObjectSize) + "
" + UtilMethods.prettyByteify(totalObjectSize)); + ret.addStatRecord(stats); } @Override public void shutdown() { Logger.info(this.getClass(), "===== Calling shutdown [" + getName() + "]."); - isInitialized = false; + isInitialized.set(false); } - private Cache getCache(String cacheName) { - + private Cache getCache(final String cacheName) { if (cacheName == null) { throw new DotStateException("Null cache region passed in"); } + return groups.computeIfAbsent(cacheName, k -> createCache(cacheName.toLowerCase())); + } + + private Cache createCache(String cacheName) { + Cache cache; + boolean separateCache = (Config.getBooleanProperty( + "cache.separate.caches.for.non.defined.regions", true) + || availableCaches.contains(cacheName) + || DEFAULT_CACHE.equals(cacheName)); + + if (separateCache) { + int size = Config.getIntProperty(CACHE + cacheName + "." + SIZE, -1); + + if (size == -1) { + size = Config.getIntProperty(CACHE + DEFAULT_CACHE + "." + SIZE, 100); + } - cacheName = cacheName.toLowerCase(); - Cache cache = groups.get(cacheName); + final int defaultTTL = Config.getIntProperty(CACHE + cacheName + SECONDS, -1); - // init cache if it does not exist - if (cache == null) { - synchronized (cacheName.intern()) { - cache = groups.get(cacheName); - if (cache == null) { - boolean separateCache = (Config.getBooleanProperty( - "cache.separate.caches.for.non.defined.regions", true) - || availableCaches.contains(cacheName) - || DEFAULT_CACHE.equals(cacheName)); + Logger.debug(this.getClass(), + "***\t Building Cache : " + cacheName + ", " + SIZE + ":" + size + + ",Concurrency:" + + Config.getIntProperty("cache.concurrencylevel", 32)); - if (separateCache) { - int size = Config.getIntProperty("cache." + cacheName + ".size", -1); + cache = Caffeine.newBuilder() + .maximumSize(size) + .recordStats() + .expireAfter(new Expiry() { + public long expireAfterCreate(String key, Object value, long currentTime) { + long ttlInSeconds; - if (size == -1) { - size = Config.getIntProperty("cache." + DEFAULT_CACHE + ".size", 100); + if(value instanceof Expirable + && ((Expirable) value).getTtl() > 0) { + ttlInSeconds = ((Expirable) value).getTtl(); + } else if (defaultTTL > 0) { + ttlInSeconds = defaultTTL; + } else { + ttlInSeconds = Long.MAX_VALUE; + } + + return TimeUnit.SECONDS.toNanos(ttlInSeconds); + } + public long expireAfterUpdate(String key, Object value, + long currentTime, long currentDuration) { + return currentDuration; + } + public long expireAfterRead(String key, Object value, + long currentTime, long currentDuration) { + return currentDuration; } + }) + .build(key -> null); - final int defaultTTL = Config.getIntProperty("cache." + cacheName + ".seconds", -1); - - - Logger.info(this.getClass(), - "***\t Building Cache : " + cacheName + ", size:" + size - + ",Concurrency:" - + Config.getIntProperty("cache.concurrencylevel", 32)); - - cache = Caffeine.newBuilder() - .maximumSize(size) - .recordStats() - .expireAfter(new Expiry() { - public long expireAfterCreate(String key, Object value, long currentTime) { - long ttlInSeconds; - - if(value instanceof Expirable - && ((Expirable) value).getTtl() > 0) { - ttlInSeconds = ((Expirable) value).getTtl(); - } else if (defaultTTL > 0) { - ttlInSeconds = defaultTTL; - } else { - ttlInSeconds = Long.MAX_VALUE; - } - - return TimeUnit.SECONDS.toNanos(ttlInSeconds); - } - public long expireAfterUpdate(String key, Object value, - long currentTime, long currentDuration) { - return currentDuration; - } - public long expireAfterRead(String key, Object value, - long currentTime, long currentDuration) { - return currentDuration; - } - }) - .build(key -> null); - - groups.put(cacheName, cache); - - } else { - Logger.info(this.getClass(), - "***\t No Cache for : " + cacheName + ", using " + DEFAULT_CACHE); - cache = getCache(DEFAULT_CACHE); - groups.put(cacheName, cache); - } - } - } + } else { + Logger.debug(this.getClass(), + "***\t No Cache for : " + cacheName + ", using " + DEFAULT_CACHE); + cache = getCache(DEFAULT_CACHE); } return cache; } From c422e25d602a83bc938d926c2ee7ffbd9c2f6e95 Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Fri, 3 Feb 2023 17:09:25 -0500 Subject: [PATCH 06/28] merge upstream --- .../src/integration-test/java/com/dotcms/MainSuite.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dotCMS/src/integration-test/java/com/dotcms/MainSuite.java b/dotCMS/src/integration-test/java/com/dotcms/MainSuite.java index d507b77ca632..aa5821de51c9 100644 --- a/dotCMS/src/integration-test/java/com/dotcms/MainSuite.java +++ b/dotCMS/src/integration-test/java/com/dotcms/MainSuite.java @@ -227,15 +227,21 @@ @RunWith(MainBaseSuite.class) @SuiteClasses({ + /* StartEndScheduledExperimentsJobTest.class, + RulesAPIImplIntegrationTest.class, Task220825CreateVariantFieldTest.class, + */ // AnalyticsAPIImplTest.class, + /* Task221007AddVariantIntoPrimaryKeyTest.class, ESContentletAPIImplTest.class, ExperimentAPIImpIT.class, ExperimentWebAPIImplIT.class, + */ // AccessTokenRenewJobTest.class, + /* ContentletWebAPIImplIntegrationTest.class, // moved to top because of failures on GHA DependencyBundlerTest.class, // moved to top because of failures on GHA SiteAndFolderResolverImplTest.class, //Moved up to avoid conflicts with CT deletion @@ -428,7 +434,8 @@ com.dotmarketing.common.reindex.ReindexAPITest.class, com.dotmarketing.common.db.DotDatabaseMetaDataTest.class, com.dotmarketing.common.db.ParamsSetterTest.class, - com.dotmarketing.cms.urlmap.URLMapAPIImplTest.class, + com.dotmarketing.cms.urlmap.URLMapAPIImplTest.class + ,*/ com.dotmarketing.factories.PublishFactoryTest.class, com.dotmarketing.factories.WebAssetFactoryTest.class, com.dotmarketing.factories.MultiTreeAPITest.class, From e253ec7dead3170fc6b39ae4e95310636acb2a21 Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Fri, 3 Feb 2023 17:12:23 -0500 Subject: [PATCH 07/28] fixup enterprise --- dotCMS/src/main/enterprise | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotCMS/src/main/enterprise b/dotCMS/src/main/enterprise index ff05f4a91308..8b94e46df785 160000 --- a/dotCMS/src/main/enterprise +++ b/dotCMS/src/main/enterprise @@ -1 +1 @@ -Subproject commit ff05f4a91308e5af1b7fbb246a2dbb56d2f083fd +Subproject commit 8b94e46df78503f7ad0319fc57facc0472c3021e From 23b2b5a8778cdb5c606db2fc50f3cd83c7cb9825 Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Wed, 1 Feb 2023 21:52:05 -0500 Subject: [PATCH 08/28] Ensure Integration Test context available to data providers --- .../junit/CustomDataProviderRunner.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/dotCMS/src/integration-test/java/com/dotcms/junit/CustomDataProviderRunner.java b/dotCMS/src/integration-test/java/com/dotcms/junit/CustomDataProviderRunner.java index a06d21031203..c184dde8ab0c 100644 --- a/dotCMS/src/integration-test/java/com/dotcms/junit/CustomDataProviderRunner.java +++ b/dotCMS/src/integration-test/java/com/dotcms/junit/CustomDataProviderRunner.java @@ -1,16 +1,13 @@ package com.dotcms.junit; import com.dotcms.business.bytebuddy.ByteBuddyFactory; +import com.dotcms.util.IntegrationTestInitService; import com.dotmarketing.db.DbConnectionFactory; import com.dotmarketing.util.Logger; import com.tngtech.java.junit.dataprovider.DataProviderRunner; -import java.util.Arrays; - -import net.bytebuddy.agent.ByteBuddyAgent; -import net.bytebuddy.agent.builder.AgentBuilder; +import java.util.List; import org.junit.Ignore; import org.junit.rules.RunRules; -import org.junit.rules.TestRule; import org.junit.runner.Description; import org.junit.runner.notification.RunNotifier; import org.junit.runners.model.FrameworkMethod; @@ -21,6 +18,15 @@ public class CustomDataProviderRunner extends DataProviderRunner { public CustomDataProviderRunner(Class clazz) throws InitializationError { super(clazz); ByteBuddyFactory.init(); + // Make sure integration test init service is initialized for Data Providers + // these run when the tests to run are determined, not when the tests are run + // so @BeforeClass methods are not run before this. + // It is better to have DataProviders not do any database work, but if they do + try { + IntegrationTestInitService.getInstance().init(); + } catch (Exception e) { + throw new InitializationError(e); + } } @Override @@ -30,13 +36,12 @@ protected void runChild(final FrameworkMethod method, RunNotifier notifier) { notifier.fireTestIgnored(description); } else { RunRules runRules = new RunRules(methodBlock(method), - Arrays.asList(new TestRule[]{new RuleWatcher()}), description); + List.of(new RuleWatcher()), description); runLeaf(runRules, description, notifier); if (DbConnectionFactory.connectionExists()) { Logger.error(CustomDataProviderRunner.class,"Connection remains after test = "+description+":"+DbConnectionFactory.connectionExists()); } - } } From e1c8f81f52d7335541252f3b6a3328adf69d4c40 Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Wed, 1 Feb 2023 23:45:34 -0500 Subject: [PATCH 09/28] Split tests half to locate side effects-2 --- .../java/com/dotcms/MainSuite.java | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/dotCMS/src/integration-test/java/com/dotcms/MainSuite.java b/dotCMS/src/integration-test/java/com/dotcms/MainSuite.java index aa5821de51c9..75dafc156e2c 100644 --- a/dotCMS/src/integration-test/java/com/dotcms/MainSuite.java +++ b/dotCMS/src/integration-test/java/com/dotcms/MainSuite.java @@ -8,14 +8,12 @@ import com.dotcms.cache.lettuce.RedisClientTest; import com.dotcms.content.business.ObjectMapperTest; import com.dotcms.content.business.json.ContentletJsonAPITest; -import com.dotcms.content.elasticsearch.business.ESContentletAPIImplTest; import com.dotcms.content.elasticsearch.business.ESIndexAPITest; import com.dotcms.content.elasticsearch.business.ElasticsearchUtilTest; import com.dotcms.content.elasticsearch.util.ESMappingUtilHelperTest; import com.dotcms.content.model.hydration.MetadataDelegateTest; import com.dotcms.contenttype.business.ContentTypeInitializerTest; import com.dotcms.contenttype.business.DotAssetBaseTypeToContentTypeStrategyImplTest; -import com.dotcms.contenttype.business.SiteAndFolderResolverImplTest; import com.dotcms.contenttype.business.StoryBlockAPITest; import com.dotcms.contenttype.test.DotAssetAPITest; import com.dotcms.csspreproc.CSSCacheTest; @@ -25,16 +23,13 @@ import com.dotcms.ema.EMAWebInterceptorTest; import com.dotcms.enterprise.cluster.ClusterFactoryTest; import com.dotcms.enterprise.publishing.bundler.URLMapBundlerTest; -import com.dotcms.enterprise.publishing.remote.PushPublishBundleGeneratorTest; import com.dotcms.enterprise.publishing.remote.StaticPushPublishBundleGeneratorTest; import com.dotcms.enterprise.publishing.remote.bundler.ContainerBundlerTest; import com.dotcms.enterprise.publishing.remote.bundler.ContentBundlerTest; import com.dotcms.enterprise.publishing.remote.bundler.ContentTypeBundlerTest; -import com.dotcms.enterprise.publishing.remote.bundler.DependencyBundlerTest; import com.dotcms.enterprise.publishing.remote.bundler.FolderBundlerTest; import com.dotcms.enterprise.publishing.remote.bundler.HostBundlerTest; import com.dotcms.enterprise.publishing.remote.bundler.LinkBundlerTest; -import com.dotcms.enterprise.publishing.remote.bundler.RuleBundlerTest; import com.dotcms.enterprise.publishing.remote.bundler.TemplateBundlerTest; import com.dotcms.enterprise.publishing.remote.bundler.WorkflowBundlerTest; import com.dotcms.enterprise.publishing.remote.handler.ContentHandlerTest; @@ -42,12 +37,7 @@ import com.dotcms.enterprise.publishing.remote.handler.HandlerUtilTest; import com.dotcms.enterprise.publishing.staticpublishing.AWSS3PublisherTest; import com.dotcms.enterprise.publishing.staticpublishing.LanguageFolderTest; -import com.dotcms.enterprise.publishing.staticpublishing.StaticPublisherIntegrationTest; -import com.dotcms.enterprise.rules.RulesAPIImplIntegrationTest; -import com.dotcms.experiments.business.ExperimentAPIImpIT; -import com.dotcms.experiments.business.web.ExperimentWebAPIImplIT; import com.dotcms.filters.interceptor.meta.MetaWebInterceptorTest; -import com.dotcms.graphql.DotGraphQLHttpServletTest; import com.dotcms.integritycheckers.HostIntegrityCheckerTest; import com.dotcms.integritycheckers.IntegrityUtilTest; import com.dotcms.junit.MainBaseSuite; @@ -56,14 +46,11 @@ import com.dotcms.publisher.bundle.business.BundleAPITest; import com.dotcms.publisher.bundle.business.BundleFactoryTest; import com.dotcms.publisher.business.PublishAuditAPITest; -import com.dotcms.publisher.business.PublishQueueElementTransformerTest; import com.dotcms.publisher.receiver.BundlePublisherTest; import com.dotcms.publisher.util.DependencyManagerTest; -import com.dotcms.publisher.util.DependencyModDateUtilTest; import com.dotcms.publishing.BundlerUtilTest; import com.dotcms.publishing.PublisherFilterImplTest; import com.dotcms.publishing.PushPublishFiltersInitializerTest; -import com.dotcms.publishing.job.SiteSearchJobImplTest; import com.dotcms.publishing.manifest.CSVManifestBuilderTest; import com.dotcms.publishing.manifest.CSVManifestReaderTest; import com.dotcms.publishing.manifest.ManifestReaderFactoryTest; @@ -75,7 +62,6 @@ import com.dotcms.rendering.velocity.viewtools.JSONToolTest; import com.dotcms.rendering.velocity.viewtools.MessageToolTest; import com.dotcms.rendering.velocity.viewtools.XmlToolTest; -import com.dotcms.rendering.velocity.viewtools.XsltToolTest; import com.dotcms.rendering.velocity.viewtools.content.StoryBlockMapTest; import com.dotcms.rendering.velocity.viewtools.content.StoryBlockTest; import com.dotcms.rest.BundlePublisherResourceIntegrationTest; @@ -102,7 +88,6 @@ import com.dotcms.storage.StoragePersistenceAPITest; import com.dotcms.storage.repository.HashedLocalFileRepositoryManagerTest; import com.dotcms.translate.GoogleTranslationServiceIntegrationTest; -import com.dotcms.uuid.shorty.LegacyShortyIdApiTest; import com.dotcms.variant.VariantAPITest; import com.dotcms.variant.VariantFactoryTest; import com.dotcms.variant.business.VariantCacheTest; @@ -112,7 +97,6 @@ import com.dotmarketing.business.PermissionBitFactoryImplTest; import com.dotmarketing.business.VersionableFactoryImplTest; import com.dotmarketing.business.helper.PermissionHelperTest; -import com.dotmarketing.cache.FolderCacheImplIntegrationTest; import com.dotmarketing.common.db.DBTimeZoneCheckTest; import com.dotmarketing.filters.AutoLoginFilterTest; import com.dotmarketing.image.filter.ImageFilterAPIImplTest; @@ -122,9 +106,7 @@ import com.dotmarketing.portlets.categories.business.CategoryFactoryTest; import com.dotmarketing.portlets.cmsmaintenance.factories.CMSMaintenanceFactoryTest; import com.dotmarketing.portlets.containers.business.ContainerFactoryImplTest; -import com.dotmarketing.portlets.containers.business.ContainerStructureFinderStrategyResolverTest; import com.dotmarketing.portlets.contentlet.business.ContentletCacheImplTest; -import com.dotmarketing.portlets.contentlet.business.web.ContentletWebAPIImplIntegrationTest; import com.dotmarketing.portlets.contentlet.model.ContentletDependenciesTest; import com.dotmarketing.portlets.contentlet.model.IntegrationResourceLinkTest; import com.dotmarketing.portlets.fileassets.business.FileAssetAPIImplIntegrationTest; @@ -202,12 +184,10 @@ import com.dotmarketing.startup.runonce.Task220606UpdatePushNowActionletNameTest; import com.dotmarketing.startup.runonce.Task220822CreateVariantTableTest; import com.dotmarketing.startup.runonce.Task220824CreateDefaultVariantTest; -import com.dotmarketing.startup.runonce.Task220825CreateVariantFieldTest; import com.dotmarketing.startup.runonce.Task220825MakeSomeSystemFieldsRemovableTest; import com.dotmarketing.startup.runonce.Task220829CreateExperimentsTableTest; import com.dotmarketing.startup.runonce.Task220912UpdateCorrectShowOnMenuPropertyTest; import com.dotmarketing.startup.runonce.Task220928AddLookbackWindowColumnToExperimentTest; -import com.dotmarketing.startup.runonce.Task221007AddVariantIntoPrimaryKeyTest; import com.dotmarketing.startup.runonce.Task230110MakeSomeSystemFieldsRemovableByBaseTypeTest; import com.dotmarketing.util.ConfigTest; import com.dotmarketing.util.HashBuilderTest; From 5a231b1486da04e2e20121fd07150b3476cc2a67 Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Thu, 2 Feb 2023 01:27:31 -0500 Subject: [PATCH 10/28] Null pointer from hosts with null hostname during testing --- .../rest/api/v1/apps/SiteViewPaginator.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dotCMS/src/main/java/com/dotcms/rest/api/v1/apps/SiteViewPaginator.java b/dotCMS/src/main/java/com/dotcms/rest/api/v1/apps/SiteViewPaginator.java index 6907621e41df..51e754e00b1c 100644 --- a/dotCMS/src/main/java/com/dotcms/rest/api/v1/apps/SiteViewPaginator.java +++ b/dotCMS/src/main/java/com/dotcms/rest/api/v1/apps/SiteViewPaginator.java @@ -5,7 +5,6 @@ import com.dotcms.util.pagination.PaginationException; import com.dotcms.util.pagination.PaginatorOrdered; import com.dotmarketing.beans.Host; -import com.dotmarketing.business.APILocator; import com.dotmarketing.business.PermissionAPI; import com.dotmarketing.exception.DotDataException; import com.dotmarketing.exception.DotRuntimeException; @@ -75,13 +74,13 @@ public PaginatedArrayList getItems(final User user, final String filte final long totalCount = allSitesIdentifiers.size(); - //This values are fed from the outside through the appsAPI. + //These values are fed from the outside through the appsAPI. final Set sitesWithConfigurations = configuredSitesSupplier.get().stream() .map(String::toLowerCase).collect(Collectors.toSet()); final LinkedHashSet allSites = new LinkedHashSet<>(allSitesIdentifiers); //By doing this we remove from the configured-sites collection whatever sites didn't match the search. - //If it isn't part of the search results also discard from the configured sites we intent to show. + //If it isn't part of the search results also discard from the configured sites we intend to show. final LinkedHashSet configuredSites = sitesWithConfigurations.stream() .filter(allSites::contains) .collect(Collectors.toCollection(LinkedHashSet::new)); @@ -125,12 +124,12 @@ public PaginatedArrayList getItems(final User user, final String filte * First-one the Set coming from the serviceIntegrations-API containing all the sites with configurations. * Second-one a List with all the sites coming from querying the index. * Meaning this list is expected to come filtered and sorted. - * The resulting list will have all the configured items first and then he rest of the entries. - * Additionally to that SYSTEM_HOST is always expected to appear first if it ever existed on the allSites list. + * The resulting list will have all the configured items first and then the rest of the entries. + * Additionally, to that SYSTEM_HOST is always expected to appear first if it ever existed on the allSites list. * (Cuz it could have been removed from applying filtering). * @param configuredSites sites with configurations coming from service Integration API. * @param allSites all-sites sorted and filtered loaded from ES - * @return a brand new List ordered. + * @return a brand-new List ordered. */ private List join(final Set configuredSites, final List allSites) { final List newList = new LinkedList<>(); @@ -178,8 +177,9 @@ private List getHostIdentifiers(final User user, final String filter) hostStream = hostStream.filter(host -> host.getHostname().matches(regexFilter)); } return hostStream.filter(host -> Try.of(() -> permissionAPI - .doesUserHavePermission(host, PermissionAPI.PERMISSION_READ, user)) - .getOrElse(false)).sorted(Comparator.comparing(Host::getHostname)) + .doesUserHavePermission(host, PermissionAPI.PERMISSION_READ, user)) + .getOrElse(false)).filter(h -> h.getHostname() != null) + .sorted(Comparator.comparing(Host::getHostname)) .map(Contentlet::getIdentifier).filter(Objects::nonNull).map(String::toLowerCase) .collect(Collectors.toList()); From ea4b05c89e51a977bebc9efa02d892149d4646b5 Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Thu, 2 Feb 2023 14:56:15 -0500 Subject: [PATCH 11/28] Commit to diagnose ci build IT errors --- .../api/v1/folder/FolderResourceTest.java | 7 +++++++ .../contentlet/business/HostAPIImpl.java | 8 +++++++- .../contentlet/business/HostFactoryImpl.java | 20 +++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/dotCMS/src/integration-test/java/com/dotcms/rest/api/v1/folder/FolderResourceTest.java b/dotCMS/src/integration-test/java/com/dotcms/rest/api/v1/folder/FolderResourceTest.java index a9f28dc10efb..f5b9d0f62ca8 100644 --- a/dotCMS/src/integration-test/java/com/dotcms/rest/api/v1/folder/FolderResourceTest.java +++ b/dotCMS/src/integration-test/java/com/dotcms/rest/api/v1/folder/FolderResourceTest.java @@ -16,9 +16,11 @@ import com.dotmarketing.business.PermissionAPI; import com.dotmarketing.exception.DotDataException; import com.dotmarketing.exception.DotSecurityException; +import com.dotmarketing.portlets.contentlet.business.HostAPIImpl; import com.dotmarketing.portlets.folders.business.FolderAPI; import com.dotmarketing.portlets.folders.exception.InvalidFolderNameException; import com.dotmarketing.portlets.folders.model.Folder; +import com.dotmarketing.util.Logger; import com.dotmarketing.util.UUIDGenerator; import com.liferay.portal.model.User; import com.liferay.util.Base64; @@ -40,6 +42,7 @@ public class FolderResourceTest { static FolderAPI folderAPI; static User adminUser; + @BeforeClass public static void prepare() throws Exception { IntegrationTestInitService.getInstance().init(); @@ -301,6 +304,10 @@ public void test_loadFolderAndSubFoldersByPath_UserNoPermissionsOverSubFolder_re final List foldersToCreate = Arrays.asList("test_"+currentTime+"/folder_"+currentTime); final Host newHost = new SiteDataGen().nextPersisted(); + Logger.info(this, "STEVETEST: Host Identifier: " + newHost.getIdentifier()+ " Host Name: " + newHost.getHostname()); + HostAPIImpl.testThreadLocal.set(newHost.getIdentifier()); + + //Create Folders and SubFolders Response responseResource = resource.createFolders(getHttpRequest(adminUser.getEmailAddress(),"admin"),response,foldersToCreate,newHost.getHostname()); diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java index 358630d245ee..676e2683aac0 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java @@ -60,6 +60,8 @@ */ public class HostAPIImpl implements HostAPI, Flushable { + public static ThreadLocal testThreadLocal = new ThreadLocal<>(); + private HostCache hostCache = CacheLocator.getHostCache(); private Host systemHost; private final SystemEventsAPI systemEventsAPI; @@ -192,9 +194,13 @@ public Host findByName(final String siteName, final boolean respectFrontendRoles) throws DotDataException, DotSecurityException { try { + return findByNameNotDefault(siteName, user, respectFrontendRoles); } catch (Exception e) { - + if (HostAPIImpl.testThreadLocal.get().equals(siteName)) + { + Logger.info(this, "Default Host not found, returning system host"); + } try { return findDefaultHost(APILocator.systemUser(), respectFrontendRoles); } catch(Exception ex){ diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostFactoryImpl.java b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostFactoryImpl.java index c0685ad24a9f..1194787c0e0e 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostFactoryImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostFactoryImpl.java @@ -164,7 +164,15 @@ protected ContentletAPI getContentletAPI() { @Override public Host bySiteName(final String siteName) { + if (HostAPIImpl.testThreadLocal.get().equals(siteName)) + { + Logger.info(this, "Checking for site: " + siteName + " in cache"); + } Host site = siteCache.get(siteName); + if (HostAPIImpl.testThreadLocal.get().equals(siteName)) + { + Logger.info(this, "Found site: " + siteName + " in cache with identifier: " + (site != null ? site.getIdentifier() : "null")); + } if (null == site || !UtilMethods.isSet(site.getIdentifier())) { final DotConnect dc = new DotConnect(); final StringBuilder sqlQuery = new StringBuilder().append(SELECT_SITE_INODE) @@ -175,10 +183,18 @@ public Host bySiteName(final String siteName) { try { final List> dbResults = dc.loadResults(); if (dbResults.isEmpty()) { + if (HostAPIImpl.testThreadLocal.get().equals(siteName)) + { + Logger.info(this, "site: " + siteName + " not found in db"); + } return null; } final String siteInode = dbResults.get(0).get("inode"); if (dbResults.size() > 1) { + if (HostAPIImpl.testThreadLocal.get().equals(siteName)) + { + Logger.info(this, "site: " + siteName + " Found Multiple sites"); + } // This situation should NOT happen at all final StringBuilder warningMsg = new StringBuilder().append("ERROR: ").append(dbResults.size()) .append(" Sites have the same name '").append(siteName).append("':\n"); @@ -190,6 +206,10 @@ public Host bySiteName(final String siteName) { } final Contentlet siteAsContentlet = this.contentFactory.find(siteInode); site = new Host(siteAsContentlet); + if (HostAPIImpl.testThreadLocal.get().equals(siteName)) + { + Logger.info(this, "site: " + siteName + " Adding to sitecache"); + } this.siteCache.add(site); } catch (final Exception e) { final String errorMsg = String.format("An error occurred when retrieving Site by name '%s': %s", From f0a1ca3da773fd64ae09197208ad930a9053668b Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Thu, 2 Feb 2023 17:11:19 -0500 Subject: [PATCH 12/28] Commit to diagnose ci build IT errors --- .../java/com/dotcms/rest/api/v1/folder/FolderResourceTest.java | 2 +- .../dotmarketing/portlets/contentlet/business/HostAPIImpl.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dotCMS/src/integration-test/java/com/dotcms/rest/api/v1/folder/FolderResourceTest.java b/dotCMS/src/integration-test/java/com/dotcms/rest/api/v1/folder/FolderResourceTest.java index f5b9d0f62ca8..114df3b72453 100644 --- a/dotCMS/src/integration-test/java/com/dotcms/rest/api/v1/folder/FolderResourceTest.java +++ b/dotCMS/src/integration-test/java/com/dotcms/rest/api/v1/folder/FolderResourceTest.java @@ -305,7 +305,7 @@ public void test_loadFolderAndSubFoldersByPath_UserNoPermissionsOverSubFolder_re final Host newHost = new SiteDataGen().nextPersisted(); Logger.info(this, "STEVETEST: Host Identifier: " + newHost.getIdentifier()+ " Host Name: " + newHost.getHostname()); - HostAPIImpl.testThreadLocal.set(newHost.getIdentifier()); + HostAPIImpl.testThreadLocal.set(newHost.getHostname()); //Create Folders and SubFolders diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java index 676e2683aac0..02d9d6f620b4 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java @@ -199,7 +199,7 @@ public Host findByName(final String siteName, } catch (Exception e) { if (HostAPIImpl.testThreadLocal.get().equals(siteName)) { - Logger.info(this, "Default Host not found, returning system host"); + Logger.error(this, "Default Host not found, returning system host ",e); } try { return findDefaultHost(APILocator.systemUser(), respectFrontendRoles); From 9386e496b413de02633a37dd6004bd3173f5c618 Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Thu, 2 Feb 2023 21:49:37 -0500 Subject: [PATCH 13/28] fix session connection closing --- .../java/com/dotcms/IntegrationTestBase.java | 2 +- .../main/java/com/dotcms/rest/LicenseResource.java | 2 +- .../java/com/dotmarketing/db/HibernateUtil.java | 14 ++++++++++---- .../structure/business/StructureAPIImpl.java | 2 +- .../workflows/business/WorkflowFactoryImpl.java | 2 +- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/dotCMS/src/integration-test/java/com/dotcms/IntegrationTestBase.java b/dotCMS/src/integration-test/java/com/dotcms/IntegrationTestBase.java index 09f122b88b2b..772a6078632b 100644 --- a/dotCMS/src/integration-test/java/com/dotcms/IntegrationTestBase.java +++ b/dotCMS/src/integration-test/java/com/dotcms/IntegrationTestBase.java @@ -132,7 +132,7 @@ public void after () { //Closing the session try { HibernateUtil.closeAndCommitTransaction(); - if (null != HibernateUtil.getSession()) { + if (null != HibernateUtil.getSession(false)) { HibernateUtil.getSession().connection().close(); HibernateUtil.getSession().close(); } diff --git a/dotCMS/src/main/java/com/dotcms/rest/LicenseResource.java b/dotCMS/src/main/java/com/dotcms/rest/LicenseResource.java index f224b639cae9..9271f544028f 100644 --- a/dotCMS/src/main/java/com/dotcms/rest/LicenseResource.java +++ b/dotCMS/src/main/java/com/dotcms/rest/LicenseResource.java @@ -337,7 +337,7 @@ public Response freeLicense(@Context HttpServletRequest request, @Context final } catch(Exception exception) { Logger.error(this, "can't free license ",exception); try { - if(HibernateUtil.getSession().isOpen()){ + if(HibernateUtil.getSession(false).isOpen()){ HibernateUtil.rollbackTransaction(); } } catch (DotHibernateException dotHibernateException) { diff --git a/dotCMS/src/main/java/com/dotmarketing/db/HibernateUtil.java b/dotCMS/src/main/java/com/dotmarketing/db/HibernateUtil.java index 21ef6e6d7bf8..78d01f3d50ed 100644 --- a/dotCMS/src/main/java/com/dotmarketing/db/HibernateUtil.java +++ b/dotCMS/src/main/java/com/dotmarketing/db/HibernateUtil.java @@ -759,12 +759,14 @@ public static void setSession(final Session newSession) { throw new DotRuntimeException(e.getMessage(), e); } } - + public static Session getSession() { + return getSession(true); + } /** * Attempts to find a session associated with the Thread. If there isn't a * session, it will create one. */ - public static Session getSession() { + public static Session getSession(boolean createIfClosed) { try{ final Optional sessionOptional = getSessionIfOpened(); Session session = sessionOptional.isPresent() ? sessionOptional.get() : null; @@ -776,11 +778,15 @@ public static Session getSession() { if (session.connection().isClosed()) { try { session.close(); + // is it an issue to close the session if connection is already closed, should this have been closed before } catch (HibernateException e1) { Logger.error(HibernateUtil.class,e1.getMessage(),e1); } session = null; - session = sessionFactory.openSession(DbConnectionFactory.getConnection()); + if (createIfClosed) { + session = sessionFactory.openSession( + DbConnectionFactory.getConnection()); + } } } catch (Exception e) { try { @@ -1045,7 +1051,7 @@ public static void closeSession() throws DotHibernateException{ if (null == sessionHolder.get()){ return; } - Session session = getSession(); + Session session = getSession(false); if (null != session) { session.flush(); diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/structure/business/StructureAPIImpl.java b/dotCMS/src/main/java/com/dotmarketing/portlets/structure/business/StructureAPIImpl.java index 70263dc14c1e..4feb2742258a 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/structure/business/StructureAPIImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/structure/business/StructureAPIImpl.java @@ -142,7 +142,7 @@ public void olddelete(Structure st, User user) throws DotSecurityException, DotD // remove structure permissions perAPI.removePermissions(st); - HibernateUtil.getSession().clear(); + HibernateUtil.getSession(false).clear(); // remove structure itself StructureFactory.deleteStructure(st); diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowFactoryImpl.java b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowFactoryImpl.java index 66374cf6d975..2f5393270faf 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowFactoryImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowFactoryImpl.java @@ -2359,7 +2359,7 @@ public List findExpiredTasks() throws DotDataException, DotSecurit Logger.error(this, e.getMessage(), e); } finally { - HibernateUtil.getSession().clear(); + HibernateUtil.getSession(false).clear(); } return list; } From 208b5b59eb20582236fa39348c302fc7306fd068 Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Thu, 2 Feb 2023 21:50:02 -0500 Subject: [PATCH 14/28] add test log4j-test.xml --- .../integration-test/resources/log4j2-test.xml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 dotCMS/src/integration-test/resources/log4j2-test.xml diff --git a/dotCMS/src/integration-test/resources/log4j2-test.xml b/dotCMS/src/integration-test/resources/log4j2-test.xml new file mode 100644 index 000000000000..eb539c692416 --- /dev/null +++ b/dotCMS/src/integration-test/resources/log4j2-test.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + From 45bf080705bac5002e6bef4a7e211b53876dca98 Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Thu, 2 Feb 2023 21:51:38 -0500 Subject: [PATCH 15/28] Ensure bytebuddy loaded when running IT tests individually --- .../java/com/dotcms/util/IntegrationTestInitService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dotCMS/src/integration-test/java/com/dotcms/util/IntegrationTestInitService.java b/dotCMS/src/integration-test/java/com/dotcms/util/IntegrationTestInitService.java index 3a63c036d848..c49deca945f7 100644 --- a/dotCMS/src/integration-test/java/com/dotcms/util/IntegrationTestInitService.java +++ b/dotCMS/src/integration-test/java/com/dotcms/util/IntegrationTestInitService.java @@ -1,5 +1,6 @@ package com.dotcms.util; +import com.dotcms.business.bytebuddy.ByteBuddyFactory; import com.dotcms.config.DotInitializationService; import com.dotcms.repackage.org.apache.struts.Globals; import com.dotcms.repackage.org.apache.struts.config.ModuleConfig; @@ -33,6 +34,7 @@ private IntegrationTestInitService() { } public static IntegrationTestInitService getInstance() { + ByteBuddyFactory.init(); return service; } From 68a2a8aaf7b159de13e871b80c18a4b815baa393 Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Thu, 2 Feb 2023 21:52:07 -0500 Subject: [PATCH 16/28] connection logging --- .../dotmarketing/db/DbConnectionFactory.java | 65 +++++++++++++++---- 1 file changed, 52 insertions(+), 13 deletions(-) diff --git a/dotCMS/src/main/java/com/dotmarketing/db/DbConnectionFactory.java b/dotCMS/src/main/java/com/dotmarketing/db/DbConnectionFactory.java index 148c899a55a0..f633385875f6 100644 --- a/dotCMS/src/main/java/com/dotmarketing/db/DbConnectionFactory.java +++ b/dotCMS/src/main/java/com/dotmarketing/db/DbConnectionFactory.java @@ -24,12 +24,15 @@ import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; import javax.naming.Binding; import javax.naming.Context; import javax.naming.InitialContext; import javax.naming.NamingEnumeration; import javax.naming.NamingException; import javax.sql.DataSource; +import org.apache.commons.lang3.exception.ExceptionUtils; public class DbConnectionFactory { @@ -41,6 +44,7 @@ public class DbConnectionFactory { private static DataSource defaultDataSource = null; + private static AtomicBoolean connectionLogger = new AtomicBoolean(false); /** * Gets the autoCommit for the current connection * @return boolean @@ -105,6 +109,10 @@ public enum DataBaseType { connectionsHolder = new ThreadLocal>(); + private static final ThreadLocal + creatorStack = + new ThreadLocal<>(); + public static DataSource getDataSource() { if (null == defaultDataSource) { @@ -227,9 +235,13 @@ public static Connection getConnection() { DataSource db = getDataSource(); connection = db.getConnection(); connectionsList.put(DATABASE_DEFAULT_DATASOURCE, connection); - Logger.debug(DbConnectionFactory.class, - "Connection opened for thread " + Thread.currentThread().getId() + "-" + - DATABASE_DEFAULT_DATASOURCE); + if (connectionLogger.get()) { + Logger.error(DbConnectionFactory.class, + "Connection opened for thread " + Thread.currentThread().getId() + "-" + + DATABASE_DEFAULT_DATASOURCE, + new Throwable("Connection created")); + } + creatorStack.set(new Throwable("Connection created here")); } // _dbType would only be null until the getDbType was called, then it is static @@ -269,6 +281,14 @@ public static boolean connectionExists() { return isCreated; } // connectionExists. + public static Optional getCreatorStack() { + if (connectionExists()) { + return Optional.ofNullable(creatorStack.get()); + } else { + return Optional.empty(); + } + } + /** * Returns if the db is in a transaction - it will not open a db connection * if there is not one already open - instead, it will return false @@ -362,13 +382,16 @@ public static Connection getConnection(String dataSource) { if (connection == null || connection.isClosed()) { DataSource db = getDataSource(dataSource); - Logger.debug(DbConnectionFactory.class, - ()-> "Opening connection for thread " + Thread.currentThread().getId() + "-" + - dataSource + "\n" + UtilMethods.getDotCMSStackTrace()); + if (connectionLogger.get()) { + Logger.error(DbConnectionFactory.class, + "Opening connection for thread " + Thread.currentThread().getName() + + "[" + Thread.currentThread().getId() + "]" + + dataSource + "\n" + UtilMethods.getDotCMSStackTrace()); + } connection = db.getConnection(); connectionsList.put(dataSource, connection); Logger.debug(DbConnectionFactory.class, - "Connection opened for thread " + Thread.currentThread().getId() + "-" + + "Connection opened for thread " + Thread.currentThread().getName()+"["+Thread.currentThread().getId()+"]" + "-" + dataSource); } @@ -395,8 +418,13 @@ public static void closeConnection() { connectionsHolder.set(connectionsList); } - Logger.debug(DbConnectionFactory.class, ()-> "Closing all connections for " + Thread.currentThread().getId() + - "\n" + UtilMethods.getDotCMSStackTrace()); + if (connectionLogger.get()) { + Logger.error(DbConnectionFactory.class, + "Closing all connections for " + Thread.currentThread().getName() + "[" + + Thread.currentThread().getId() + "]" + + "\n" + ExceptionUtils.getStackTrace(creatorStack.get()), + new Throwable("Close Connection")); + } for (Entry entry : connectionsList.entrySet()) { String ds = entry.getKey(); @@ -412,7 +440,10 @@ public static void closeConnection() { } } - Logger.debug(DbConnectionFactory.class, ()-> "All connections closed for " + Thread.currentThread().getId()); + if (connectionLogger.get()) { + Logger.debug(DbConnectionFactory.class, + () -> "All connections closed for " + Thread.currentThread().getId()); + } connectionsList.clear(); } catch (Exception e) { @@ -440,9 +471,13 @@ public static void closeConnection(String ds) { Connection cn = connectionsList.get(ds); if (cn != null) { - Logger.debug(DbConnectionFactory.class, - "Closing connection for ()-> " + Thread.currentThread().getId() + "-" + ds + - "\n" + UtilMethods.getDotCMSStackTrace()); + if (connectionLogger.get()) { + Logger.error(DbConnectionFactory.class, + "Closing all connections for " + Thread.currentThread().getName() + "[" + + Thread.currentThread().getId() + "]" + + "\n" + ExceptionUtils.getStackTrace(creatorStack.get()), + new Throwable("Close Connection")); + } cn.close(); connectionsList.remove(ds); Logger.debug(DbConnectionFactory.class, @@ -457,6 +492,10 @@ public static void closeConnection(String ds) { } + public static void setConnectionLogger(boolean connectionLogger) { + DbConnectionFactory.connectionLogger.set(connectionLogger); + } + public static String getDBType() { /* From b737129ebecb0e146f76c443c4b0e66e4aab8a46 Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Thu, 2 Feb 2023 21:54:21 -0500 Subject: [PATCH 17/28] add loging to fix test --- .../api/v1/folder/FolderResourceTest.java | 180 +++++++++++++----- .../util/IntegrationTestInitService.java | 1 - .../contentlet/business/HostAPIImpl.java | 13 +- .../contentlet/business/HostFactoryImpl.java | 10 +- 4 files changed, 152 insertions(+), 52 deletions(-) diff --git a/dotCMS/src/integration-test/java/com/dotcms/rest/api/v1/folder/FolderResourceTest.java b/dotCMS/src/integration-test/java/com/dotcms/rest/api/v1/folder/FolderResourceTest.java index 114df3b72453..95589da83426 100644 --- a/dotCMS/src/integration-test/java/com/dotcms/rest/api/v1/folder/FolderResourceTest.java +++ b/dotCMS/src/integration-test/java/com/dotcms/rest/api/v1/folder/FolderResourceTest.java @@ -14,6 +14,7 @@ import com.dotmarketing.beans.Permission; import com.dotmarketing.business.APILocator; import com.dotmarketing.business.PermissionAPI; +import com.dotmarketing.db.DbConnectionFactory; import com.dotmarketing.exception.DotDataException; import com.dotmarketing.exception.DotSecurityException; import com.dotmarketing.portlets.contentlet.business.HostAPIImpl; @@ -31,6 +32,7 @@ import javax.servlet.http.HttpServletResponse; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -300,55 +302,145 @@ public void test_loadFolderAndSubFoldersByPath_UserNoPermissionsOverHost_return4 @Test public void test_loadFolderAndSubFoldersByPath_UserNoPermissionsOverSubFolder_returnFoldersWithUserPermissions() throws DotDataException, DotSecurityException { - final long currentTime = System.currentTimeMillis(); - final List foldersToCreate = Arrays.asList("test_"+currentTime+"/folder_"+currentTime); - final Host newHost = new SiteDataGen().nextPersisted(); - - Logger.info(this, "STEVETEST: Host Identifier: " + newHost.getIdentifier()+ " Host Name: " + newHost.getHostname()); - HostAPIImpl.testThreadLocal.set(newHost.getHostname()); - - - //Create Folders and SubFolders - Response responseResource = resource.createFolders(getHttpRequest(adminUser.getEmailAddress(),"admin"),response,foldersToCreate,newHost.getHostname()); + // system host means all hosts - //Check that the response is 200, OK - Assert.assertEquals(Status.OK.getStatusCode(),responseResource.getStatus()); - - //Get all the folders and subfolders using the admin - responseResource = resource.loadFolderAndSubFoldersByPath(getHttpRequest(adminUser.getEmailAddress(),"admin"),response,newHost.getIdentifier(),"test_"+currentTime); + try { + DbConnectionFactory.setConnectionLogger(false); - //Get The parent folder Id to give permission over it - ResponseEntityView responseEntityView = ResponseEntityView.class.cast(responseResource.getEntity()); - FolderView responseFolderView = FolderView.class.cast(responseEntityView.getEntity()); - final Folder folder = folderAPI.find(responseFolderView.getIdentifier(),adminUser,false); - - final User limitedUser = new UserDataGen().roles(TestUserUtils.getFrontendRole(), TestUserUtils.getBackendRole()).nextPersisted(); - final String password = "admin"; - limitedUser.setPassword(password); - APILocator.getUserAPI().save(limitedUser,APILocator.systemUser(),false); + Logger.error(FolderResourceTest.class, + "Connection exists before: " + DbConnectionFactory.connectionExists()); + DbConnectionFactory.getCreatorStack() + .ifPresent(s -> Logger.error(this, "Pre Connection Creator: ", s)); - //Give Permissions Over the Host - Permission permissions = new Permission(PermissionAPI.INDIVIDUAL_PERMISSION_TYPE, - newHost.getPermissionId(), - APILocator.getRoleAPI().loadRoleByKey(limitedUser.getUserId()).getId(), - PermissionAPI.PERMISSION_READ, true); - APILocator.getPermissionAPI().save(permissions, newHost, APILocator.systemUser(), false); + Logger.error(FolderResourceTest.class, + "Transaction exists before: " + DbConnectionFactory.inTransaction()); - //Give Permissions Over the Folder - permissions = new Permission(PermissionAPI.INDIVIDUAL_PERMISSION_TYPE, - folder.getPermissionId(), - APILocator.getRoleAPI().loadRoleByKey(limitedUser.getUserId()).getId(), - PermissionAPI.PERMISSION_READ, true); - APILocator.getPermissionAPI().save(permissions, folder, APILocator.systemUser(), false); + List allHostsCache = APILocator.getHostAPI() + .findAllFromDB(APILocator.getUserAPI().getSystemUser(), false); + + logHosts(allHostsCache, "allHostsCacheBefore"); + + List allHostsDb = APILocator.getHostAPI() + .findAllFromDB(APILocator.getUserAPI().getSystemUser(), false); + logHosts(allHostsDb, "allHostsDbBefore"); + + final long currentTime = System.currentTimeMillis(); + final List foldersToCreate = Arrays.asList( + "test_" + currentTime + "/folder_" + currentTime); + final Host newHost = new SiteDataGen().nextPersisted(); + + Logger.error(this, "Connection exists after nextPeristed: " + + DbConnectionFactory.connectionExists()); + Logger.error(this, "Transaction exists after nextPeristed: " + + DbConnectionFactory.inTransaction()); + + allHostsCache = APILocator.getHostAPI() + .findAllFromDB(APILocator.getUserAPI().getSystemUser(), false); + + logHosts(allHostsCache, "allHostsCacheAfter"); + + Logger.error(this, "Connection exists before findAllFromDB: " + + DbConnectionFactory.connectionExists()); + Logger.error(this, "Transaction exists before findAllFromDB: " + + DbConnectionFactory.inTransaction()); + + allHostsDb = APILocator.getHostAPI() + .findAllFromDB(APILocator.getUserAPI().getSystemUser(), false); + Logger.error(this, "Connection exists after findAllFromDB: " + + DbConnectionFactory.connectionExists()); + Logger.error(this, "Transaction exists after findAllFromDB: " + + DbConnectionFactory.inTransaction()); + + logHosts(allHostsDb, "allHostsDbAfter"); + + Logger.info(this, "STEVETEST: Host created Identifier: " + newHost.getIdentifier() + + " Host Name: " + newHost.getHostname()); + HostAPIImpl.setTestThreadLocal(newHost.getHostname()); + + //Create Folders and SubFolders + Response responseResource = resource.createFolders( + getHttpRequest(adminUser.getEmailAddress(), "admin"), response, foldersToCreate, + newHost.getHostname()); + + allHostsCache = APILocator.getHostAPI() + .findAllFromDB(APILocator.getUserAPI().getSystemUser(), false); + + logHosts(allHostsCache, "allHostsCacheAfterCreateFolders"); + + allHostsDb = APILocator.getHostAPI() + .findAllFromDB(APILocator.getUserAPI().getSystemUser(), false); + logHosts(allHostsDb, "allHostsDbAfterCreateFolders"); + + Logger.info(this, "STEVETEST: Host created Identifier: " + newHost.getIdentifier() + + " Host Name: " + newHost.getHostname()); + + //Check that the response is 200, OK + Assert.assertEquals(Status.OK.getStatusCode(), responseResource.getStatus()); + + //Get all the folders and subfolders using the admin + responseResource = resource.loadFolderAndSubFoldersByPath( + getHttpRequest(adminUser.getEmailAddress(), "admin"), response, + newHost.getIdentifier(), "test_" + currentTime); + + //Get The parent folder Id to give permission over it + ResponseEntityView responseEntityView = ResponseEntityView.class.cast( + responseResource.getEntity()); + FolderView responseFolderView = FolderView.class.cast(responseEntityView.getEntity()); + final Folder folder = folderAPI.find(responseFolderView.getIdentifier(), adminUser, + false); + + final User limitedUser = new UserDataGen().roles(TestUserUtils.getFrontendRole(), + TestUserUtils.getBackendRole()).nextPersisted(); + final String password = "admin"; + limitedUser.setPassword(password); + APILocator.getUserAPI().save(limitedUser, APILocator.systemUser(), false); + + //Give Permissions Over the Host + Permission permissions = new Permission(PermissionAPI.INDIVIDUAL_PERMISSION_TYPE, + newHost.getPermissionId(), + APILocator.getRoleAPI().loadRoleByKey(limitedUser.getUserId()).getId(), + PermissionAPI.PERMISSION_READ, true); + APILocator.getPermissionAPI() + .save(permissions, newHost, APILocator.systemUser(), false); + + //Give Permissions Over the Folder + permissions = new Permission(PermissionAPI.INDIVIDUAL_PERMISSION_TYPE, + folder.getPermissionId(), + APILocator.getRoleAPI().loadRoleByKey(limitedUser.getUserId()).getId(), + PermissionAPI.PERMISSION_READ, true); + APILocator.getPermissionAPI().save(permissions, folder, APILocator.systemUser(), false); + + //Get all the folders and subfolders using the limited user + responseResource = resource.loadFolderAndSubFoldersByPath( + getHttpRequest(limitedUser.getEmailAddress(), password), response, + newHost.getIdentifier(), "test_" + currentTime); + + //Check Results + responseEntityView = ResponseEntityView.class.cast(responseResource.getEntity()); + responseFolderView = FolderView.class.cast(responseEntityView.getEntity()); + Assert.assertNotNull("Title is null", responseFolderView.getTitle()); + Assert.assertEquals("Title is not the same as the one requested", + responseFolderView.getTitle(), "test_" + currentTime); + Assert.assertTrue("There is no subfolders since user don't have permissions", + responseFolderView.getSubFolders().isEmpty()); + + Logger.error(this, + "Connection exists at endTest: " + DbConnectionFactory.connectionExists()); + Logger.error(this, + "Transaction exists at endTest: " + DbConnectionFactory.inTransaction()); + + }finally { + DbConnectionFactory.setConnectionLogger(false); + HostAPIImpl.setTestThreadLocal(null); + } - //Get all the folders and subfolders using the limited user - responseResource = resource.loadFolderAndSubFoldersByPath(getHttpRequest(limitedUser.getEmailAddress(),password),response,newHost.getIdentifier(),"test_"+currentTime); + } - //Check Results - responseEntityView = ResponseEntityView.class.cast(responseResource.getEntity()); - responseFolderView = FolderView.class.cast(responseEntityView.getEntity()); - Assert.assertNotNull("Title is null",responseFolderView.getTitle()); - Assert.assertEquals("Title is not the same as the one requested",responseFolderView.getTitle(),"test_"+currentTime); - Assert.assertTrue("There is no subfolders since user don't have permissions",responseFolderView.getSubFolders().isEmpty()); + private static void logHosts(List allHostsDb, String message) { + Logger.error(FolderResourceTest.class,"Logging Hosts: " + message); + for (Host host : allHostsDb) { + Logger.error(FolderResourceTest.class, " Host Identifier: " + host.getIdentifier()+ " Host Name: " + host.getHostname() + "Is System Host: " + host.isSystemHost() + " Is Default: " + host.isDefault() + + "Tag Storage=" + host.getTagStorage()); + } } } diff --git a/dotCMS/src/integration-test/java/com/dotcms/util/IntegrationTestInitService.java b/dotCMS/src/integration-test/java/com/dotcms/util/IntegrationTestInitService.java index c49deca945f7..1ca1f5398d0b 100644 --- a/dotCMS/src/integration-test/java/com/dotcms/util/IntegrationTestInitService.java +++ b/dotCMS/src/integration-test/java/com/dotcms/util/IntegrationTestInitService.java @@ -13,7 +13,6 @@ import com.liferay.util.SystemProperties; import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.felix.framework.OSGISystem; import org.mockito.Mockito; /** diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java index 02d9d6f620b4..ede56d06f3a8 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java @@ -60,7 +60,7 @@ */ public class HostAPIImpl implements HostAPI, Flushable { - public static ThreadLocal testThreadLocal = new ThreadLocal<>(); + private static ThreadLocal testThreadLocal = new ThreadLocal<>(); private HostCache hostCache = CacheLocator.getHostCache(); private Host systemHost; @@ -80,6 +80,15 @@ private ContentType hostType() throws DotDataException, DotSecurityException{ return APILocator.getContentTypeAPI(APILocator.systemUser()).find(Host.HOST_VELOCITY_VAR_NAME); } + public static void setTestThreadLocal(final String testThreadLocal) { + HostAPIImpl.testThreadLocal.set(testThreadLocal); + } + + public static boolean istTestThreadLocal(String test) { + String value = HostAPIImpl.testThreadLocal.get(); + return value != null && value.equals(test); + } + /** * Lazy initialization of the Host Factory service. This helps prevent startup issues when several factories or APIs * are initialized during the initialization phase of the Host Factory. @@ -197,7 +206,7 @@ public Host findByName(final String siteName, return findByNameNotDefault(siteName, user, respectFrontendRoles); } catch (Exception e) { - if (HostAPIImpl.testThreadLocal.get().equals(siteName)) + if (HostAPIImpl.istTestThreadLocal(siteName)) { Logger.error(this, "Default Host not found, returning system host ",e); } diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostFactoryImpl.java b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostFactoryImpl.java index 1194787c0e0e..dfc9255a7512 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostFactoryImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostFactoryImpl.java @@ -164,12 +164,12 @@ protected ContentletAPI getContentletAPI() { @Override public Host bySiteName(final String siteName) { - if (HostAPIImpl.testThreadLocal.get().equals(siteName)) + if (HostAPIImpl.istTestThreadLocal(siteName)) { Logger.info(this, "Checking for site: " + siteName + " in cache"); } Host site = siteCache.get(siteName); - if (HostAPIImpl.testThreadLocal.get().equals(siteName)) + if (HostAPIImpl.istTestThreadLocal(siteName)) { Logger.info(this, "Found site: " + siteName + " in cache with identifier: " + (site != null ? site.getIdentifier() : "null")); } @@ -183,7 +183,7 @@ public Host bySiteName(final String siteName) { try { final List> dbResults = dc.loadResults(); if (dbResults.isEmpty()) { - if (HostAPIImpl.testThreadLocal.get().equals(siteName)) + if (HostAPIImpl.istTestThreadLocal(siteName)) { Logger.info(this, "site: " + siteName + " not found in db"); } @@ -191,7 +191,7 @@ public Host bySiteName(final String siteName) { } final String siteInode = dbResults.get(0).get("inode"); if (dbResults.size() > 1) { - if (HostAPIImpl.testThreadLocal.get().equals(siteName)) + if (HostAPIImpl.istTestThreadLocal(siteName)) { Logger.info(this, "site: " + siteName + " Found Multiple sites"); } @@ -206,7 +206,7 @@ public Host bySiteName(final String siteName) { } final Contentlet siteAsContentlet = this.contentFactory.find(siteInode); site = new Host(siteAsContentlet); - if (HostAPIImpl.testThreadLocal.get().equals(siteName)) + if (HostAPIImpl.istTestThreadLocal(siteName)) { Logger.info(this, "site: " + siteName + " Adding to sitecache"); } From 12dfe0af1f99edbb08cea84c43798423e0d92afe Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Thu, 2 Feb 2023 22:43:54 -0500 Subject: [PATCH 18/28] fixup log4j-test.xml to output correct log file location --- .../resources/log4j2-test.xml | 36 ++++++++++++++----- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/dotCMS/src/integration-test/resources/log4j2-test.xml b/dotCMS/src/integration-test/resources/log4j2-test.xml index eb539c692416..f67b876593fe 100644 --- a/dotCMS/src/integration-test/resources/log4j2-test.xml +++ b/dotCMS/src/integration-test/resources/log4j2-test.xml @@ -1,16 +1,36 @@ - + + + + dotcms.log + dotcms-%i.log.gz + [%d{dd/MM/yy HH:mm:ss:SSS z}] %5p %c{2}: %m%n + - + + + + + + + + + + + + + + - - + + + - - - - + \ No newline at end of file From 7aeb20885ba3180adfcc6a2b536123fa936109b7 Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Fri, 3 Feb 2023 01:21:14 -0500 Subject: [PATCH 19/28] add loging to fix test --- .../rest/api/v1/folder/FolderResourceTest.java | 18 +----------------- .../contentlet/business/HostAPIImpl.java | 1 + .../contentlet/business/HostFactoryImpl.java | 14 ++++++++++++-- .../java/com/dotmarketing/util/Config.java | 1 + 4 files changed, 15 insertions(+), 19 deletions(-) diff --git a/dotCMS/src/integration-test/java/com/dotcms/rest/api/v1/folder/FolderResourceTest.java b/dotCMS/src/integration-test/java/com/dotcms/rest/api/v1/folder/FolderResourceTest.java index 95589da83426..9ecd1317848b 100644 --- a/dotCMS/src/integration-test/java/com/dotcms/rest/api/v1/folder/FolderResourceTest.java +++ b/dotCMS/src/integration-test/java/com/dotcms/rest/api/v1/folder/FolderResourceTest.java @@ -32,7 +32,6 @@ import javax.servlet.http.HttpServletResponse; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; -import org.apache.commons.lang3.exception.ExceptionUtils; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -315,11 +314,6 @@ public void test_loadFolderAndSubFoldersByPath_UserNoPermissionsOverSubFolder_re Logger.error(FolderResourceTest.class, "Transaction exists before: " + DbConnectionFactory.inTransaction()); - List allHostsCache = APILocator.getHostAPI() - .findAllFromDB(APILocator.getUserAPI().getSystemUser(), false); - - logHosts(allHostsCache, "allHostsCacheBefore"); - List allHostsDb = APILocator.getHostAPI() .findAllFromDB(APILocator.getUserAPI().getSystemUser(), false); logHosts(allHostsDb, "allHostsDbBefore"); @@ -334,11 +328,6 @@ public void test_loadFolderAndSubFoldersByPath_UserNoPermissionsOverSubFolder_re Logger.error(this, "Transaction exists after nextPeristed: " + DbConnectionFactory.inTransaction()); - allHostsCache = APILocator.getHostAPI() - .findAllFromDB(APILocator.getUserAPI().getSystemUser(), false); - - logHosts(allHostsCache, "allHostsCacheAfter"); - Logger.error(this, "Connection exists before findAllFromDB: " + DbConnectionFactory.connectionExists()); Logger.error(this, "Transaction exists before findAllFromDB: " @@ -362,11 +351,6 @@ public void test_loadFolderAndSubFoldersByPath_UserNoPermissionsOverSubFolder_re getHttpRequest(adminUser.getEmailAddress(), "admin"), response, foldersToCreate, newHost.getHostname()); - allHostsCache = APILocator.getHostAPI() - .findAllFromDB(APILocator.getUserAPI().getSystemUser(), false); - - logHosts(allHostsCache, "allHostsCacheAfterCreateFolders"); - allHostsDb = APILocator.getHostAPI() .findAllFromDB(APILocator.getUserAPI().getSystemUser(), false); logHosts(allHostsDb, "allHostsDbAfterCreateFolders"); @@ -440,7 +424,7 @@ private static void logHosts(List allHostsDb, String message) { Logger.error(FolderResourceTest.class,"Logging Hosts: " + message); for (Host host : allHostsDb) { Logger.error(FolderResourceTest.class, " Host Identifier: " + host.getIdentifier()+ " Host Name: " + host.getHostname() + "Is System Host: " + host.isSystemHost() + " Is Default: " + host.isDefault() - + "Tag Storage=" + host.getTagStorage()); + + " Tag Storage=" + host.getTagStorage()); } } } diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java index ede56d06f3a8..bffc01f3b029 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java @@ -213,6 +213,7 @@ public Host findByName(final String siteName, try { return findDefaultHost(APILocator.systemUser(), respectFrontendRoles); } catch(Exception ex){ + Logger.error(this, "Default Host not found",ex); throw new DotRuntimeException(e.getMessage(), e); } } diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostFactoryImpl.java b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostFactoryImpl.java index dfc9255a7512..30292afa207a 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostFactoryImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostFactoryImpl.java @@ -171,8 +171,12 @@ public Host bySiteName(final String siteName) { Host site = siteCache.get(siteName); if (HostAPIImpl.istTestThreadLocal(siteName)) { - Logger.info(this, "Found site: " + siteName + " in cache with identifier: " + (site != null ? site.getIdentifier() : "null")); - } + if (site != null){ + Logger.info(this, "Found site: " + siteName + " in cache with identifier: " + site.getIdentifier()); + } else { + Logger.info(this, "Site "+ siteName + " not found in cache"); + } + } if (null == site || !UtilMethods.isSet(site.getIdentifier())) { final DotConnect dc = new DotConnect(); final StringBuilder sqlQuery = new StringBuilder().append(SELECT_SITE_INODE) @@ -204,6 +208,10 @@ public Host bySiteName(final String siteName) { warningMsg.append("Defaulting to Site '").append(siteInode).append("'"); Logger.fatal(this, warningMsg.toString()); } + if (HostAPIImpl.istTestThreadLocal(siteName)) + { + Logger.info(this, "site: " + siteName + " found inode in db: " + siteInode); + } final Contentlet siteAsContentlet = this.contentFactory.find(siteInode); site = new Host(siteAsContentlet); if (HostAPIImpl.istTestThreadLocal(siteName)) @@ -212,8 +220,10 @@ public Host bySiteName(final String siteName) { } this.siteCache.add(site); } catch (final Exception e) { + final String errorMsg = String.format("An error occurred when retrieving Site by name '%s': %s", siteName, e.getMessage()); + Logger.error(this, errorMsg, e); throw new DotRuntimeException(errorMsg, e); } } diff --git a/dotCMS/src/main/java/com/dotmarketing/util/Config.java b/dotCMS/src/main/java/com/dotmarketing/util/Config.java index 747f46fd06de..1a2b8b6717f7 100644 --- a/dotCMS/src/main/java/com/dotmarketing/util/Config.java +++ b/dotCMS/src/main/java/com/dotmarketing/util/Config.java @@ -554,6 +554,7 @@ public static boolean getBooleanProperty (String name, boolean defaultVal) { */ public static void setProperty(String key, Object value) { if(props!=null) { + Logger.info(Config.class, "Setting property: " + key + " = " + value); props.setProperty(key, value); } } From eb37c50bf096b548f66eafdf3c30886700babcc5 Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Fri, 3 Feb 2023 02:19:40 -0500 Subject: [PATCH 20/28] test remove ContainerFactoryImplTest --- dotCMS/src/integration-test/java/com/dotcms/MainSuite.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotCMS/src/integration-test/java/com/dotcms/MainSuite.java b/dotCMS/src/integration-test/java/com/dotcms/MainSuite.java index 75dafc156e2c..2f8c2caee095 100644 --- a/dotCMS/src/integration-test/java/com/dotcms/MainSuite.java +++ b/dotCMS/src/integration-test/java/com/dotcms/MainSuite.java @@ -444,7 +444,7 @@ Task05210CreateDefaultDotAssetTest.class, CleanUpFieldReferencesJobTest.class, CachedParameterDecoratorTest.class, - ContainerFactoryImplTest.class, + //ContainerFactoryImplTest.class, test remove looks like creating new default host TemplateFactoryImplTest.class, TestConfig.class, ConfigTest.class, From 1cef387f234bde61ec7a9b8ecdcafb1687a031b2 Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Fri, 3 Feb 2023 10:10:34 -0500 Subject: [PATCH 21/28] More debugging for ci build, remove creation of extra default host --- .../contentlet/business/HostAPIImpl.java | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java index bffc01f3b029..49c6534470a5 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java @@ -21,6 +21,7 @@ import com.dotmarketing.business.PermissionLevel; import com.dotmarketing.business.Role; import com.dotmarketing.business.Treeable; +import com.dotmarketing.db.DbConnectionFactory; import com.dotmarketing.db.HibernateUtil; import com.dotmarketing.exception.DotDataException; import com.dotmarketing.exception.DotRuntimeException; @@ -47,6 +48,7 @@ import java.util.StringTokenizer; import java.util.concurrent.Future; import java.util.stream.Collectors; +import org.apache.commons.lang.exception.ExceptionUtils; /** * This API allows developers to access information related to Sites objects in your dotCMS content repository. @@ -671,7 +673,32 @@ private synchronized Host getOrCreateDefaultHost() throws DotDataException, DotS if (defaultHostOpt.isPresent()) { return defaultHostOpt.get(); } + Logger.error(HostAPIImpl.class, "Default Host not found. defaultField: " + defaultField + " siteContentType: " + siteContentType); + Logger.error(HostAPIImpl.class, "Inode: " + siteContentType.inode()); + Logger.error(HostAPIImpl.class, "Fields: " + fields); + Logger.error(HostAPIImpl.class, "dbColum: " + defaultField.map(Field::dbColumn).orElse("not found")); + + Optional.ofNullable(hostCache.getAllSites()).ifPresent(hosts -> hosts.forEach(host -> { + Logger.error(HostAPIImpl.class, "Host in cache: " + host.getHostname()); + })); + + findAllFromDB(APILocator.getUserAPI().getSystemUser(),false).stream().forEach(host -> { + Logger.error(HostAPIImpl.class, "Host in db: " + host.getHostname()); + }); + + Logger.error(HostAPIImpl.class, "Attempting again to get the default host"); + + + Logger.error(HostAPIImpl.class, "DbConnection created "+ DbConnectionFactory.getCreatorStack().map(ExceptionUtils::getStackTrace).orElse("No Connection") ); + final Optional defaultHostOpt2 = this.getHostFactory().findDefaultHost(siteContentType.inode(), defaultField.get().dbColumn()); + if (defaultHostOpt2.isPresent()) { + return defaultHostOpt2.get(); + } + + return null; + // If the Default Host doesn't exist or was removed, just go ahead and re-create it + /* Host defaultHost = new Host(); defaultHost.setDefault(true); defaultHost.setHostname("noDefault-" + System.currentTimeMillis()); @@ -684,6 +711,8 @@ private synchronized Host getOrCreateDefaultHost() throws DotDataException, DotS defaultHost = save(defaultHost, APILocator.systemUser(), false); sendNotification(); return defaultHost; + + */ } private void sendNotification() { From dfc2b16ccf629e060b2974adc9e71465faaf63b6 Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Fri, 3 Feb 2023 10:57:27 -0500 Subject: [PATCH 22/28] More debugging for ci build, remove creation of extra default host --- .../portlets/contentlet/business/HostAPIImpl.java | 10 +++++++--- .../portlets/contentlet/business/HostFactoryImpl.java | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java index 49c6534470a5..30b0ad45fb8f 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java @@ -506,6 +506,7 @@ public Host findSystemHost(final User user, final boolean respectFrontendRoles) } this.systemHost = this.getHostFactory().findSystemHost(user, respectFrontendRoles); if (null == this.systemHost) { + Logger.error("System Host not found. Creating one...",new Throwable()); this.systemHost = this.createSystemHost(); } return this.systemHost; @@ -679,11 +680,13 @@ private synchronized Host getOrCreateDefaultHost() throws DotDataException, DotS Logger.error(HostAPIImpl.class, "dbColum: " + defaultField.map(Field::dbColumn).orElse("not found")); Optional.ofNullable(hostCache.getAllSites()).ifPresent(hosts -> hosts.forEach(host -> { - Logger.error(HostAPIImpl.class, "Host in cache: " + host.getHostname()); + Logger.error(HostAPIImpl.class, "Cache Host Identifier: " + host.getIdentifier()+ " Host Name: " + host.getHostname() + "Is System Host: " + host.isSystemHost() + " Is Default: " + host.isDefault() + + " Tag Storage=" + host.getTagStorage()); })); findAllFromDB(APILocator.getUserAPI().getSystemUser(),false).stream().forEach(host -> { - Logger.error(HostAPIImpl.class, "Host in db: " + host.getHostname()); + Logger.error(HostAPIImpl.class, "DB Host Identifier: " + host.getIdentifier()+ " Host Name: " + host.getHostname() + "Is System Host: " + host.isSystemHost() + " Is Default: " + host.isDefault() + + " Tag Storage=" + host.getTagStorage()); }); Logger.error(HostAPIImpl.class, "Attempting again to get the default host"); @@ -695,7 +698,8 @@ private synchronized Host getOrCreateDefaultHost() throws DotDataException, DotS return defaultHostOpt2.get(); } - return null; + throw new RuntimeException("Unable to locate the default host. Please check the logs for more information."); + // If the Default Host doesn't exist or was removed, just go ahead and re-create it /* diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostFactoryImpl.java b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostFactoryImpl.java index 30292afa207a..4151c3586917 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostFactoryImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostFactoryImpl.java @@ -647,6 +647,7 @@ public Optional findDefaultHost(final String contentTypeId, final String c inode = dotConnect.getString("working_inode"); } if (UtilMethods.isNotSet(inode)) { + Logger.debug(HostAPIImpl.class, "Unable to find default host inode from db using json ="+APILocator.getContentletJsonAPI().isPersistContentAsJson()); return Optional.empty(); } defaultHost = new Host(APILocator.getContentletAPI().find(inode, APILocator.systemUser(), false)); From bc7ab135b147c26e203ccfbfbf250d24b0f307db Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Fri, 3 Feb 2023 11:45:47 -0500 Subject: [PATCH 23/28] fix config rest of SAVE_CONTENTLET_AS_JSON --- .../dotmarketing/quartz/job/CleanUpFieldReferencesJobTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotCMS/src/integration-test/java/com/dotmarketing/quartz/job/CleanUpFieldReferencesJobTest.java b/dotCMS/src/integration-test/java/com/dotmarketing/quartz/job/CleanUpFieldReferencesJobTest.java index a01ff5972c36..e36fe878e737 100644 --- a/dotCMS/src/integration-test/java/com/dotmarketing/quartz/job/CleanUpFieldReferencesJobTest.java +++ b/dotCMS/src/integration-test/java/com/dotmarketing/quartz/job/CleanUpFieldReferencesJobTest.java @@ -181,7 +181,7 @@ public void testCleanUpFieldJob(TestCase testCase) } finally { - Config.getBooleanProperty(SAVE_CONTENTLET_AS_JSON, saveFieldsAsJson); + Config.setProperty(SAVE_CONTENTLET_AS_JSON, saveFieldsAsJson); if (contentType != null) { contentTypeAPI.delete(contentType); From 66e3d465a1b460db0362fe7a83d5b3f7bf2c06f5 Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Fri, 3 Feb 2023 12:42:04 -0500 Subject: [PATCH 24/28] fix config rest of SAVE_CONTENTLET_AS_JSON --- .../portlets/contentlet/business/HostAPIImpl.java | 4 ++++ .../portlets/contentlet/business/HostFactoryImpl.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java index 30b0ad45fb8f..05e3c5b18723 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostAPIImpl.java @@ -1,5 +1,7 @@ package com.dotmarketing.portlets.contentlet.business; +import static com.dotcms.content.business.json.ContentletJsonAPI.SAVE_CONTENTLET_AS_JSON; + import com.dotcms.api.system.event.Payload; import com.dotcms.api.system.event.SystemEventType; import com.dotcms.api.system.event.SystemEventsAPI; @@ -32,6 +34,7 @@ import com.dotmarketing.portlets.contentlet.model.IndexPolicyProvider; import com.dotmarketing.portlets.folders.model.Folder; import com.dotmarketing.portlets.links.model.Link; +import com.dotmarketing.util.Config; import com.dotmarketing.util.DateUtil; import com.dotmarketing.util.Logger; import com.dotmarketing.util.PaginatedArrayList; @@ -674,6 +677,7 @@ private synchronized Host getOrCreateDefaultHost() throws DotDataException, DotS if (defaultHostOpt.isPresent()) { return defaultHostOpt.get(); } + Logger.error(HostAPIImpl.class,"persist.contentlet.as.json is "+Config.getStringProperty(SAVE_CONTENTLET_AS_JSON, "not set")); Logger.error(HostAPIImpl.class, "Default Host not found. defaultField: " + defaultField + " siteContentType: " + siteContentType); Logger.error(HostAPIImpl.class, "Inode: " + siteContentType.inode()); Logger.error(HostAPIImpl.class, "Fields: " + fields); diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostFactoryImpl.java b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostFactoryImpl.java index 4151c3586917..691e090eaace 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostFactoryImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/contentlet/business/HostFactoryImpl.java @@ -647,7 +647,7 @@ public Optional findDefaultHost(final String contentTypeId, final String c inode = dotConnect.getString("working_inode"); } if (UtilMethods.isNotSet(inode)) { - Logger.debug(HostAPIImpl.class, "Unable to find default host inode from db using json ="+APILocator.getContentletJsonAPI().isPersistContentAsJson()); + Logger.error(HostFactoryImpl.class, "Unable to find default host inode from db using json ="+APILocator.getContentletJsonAPI().isPersistContentAsJson()); return Optional.empty(); } defaultHost = new Host(APILocator.getContentletAPI().find(inode, APILocator.systemUser(), false)); From a8f65f99db08df48ecb841b8928412e7b9b73c8c Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Mon, 6 Feb 2023 11:17:38 -0500 Subject: [PATCH 25/28] restore full suite --- .../java/com/dotcms/MainSuite.java | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/dotCMS/src/integration-test/java/com/dotcms/MainSuite.java b/dotCMS/src/integration-test/java/com/dotcms/MainSuite.java index 2f8c2caee095..82282cb92106 100644 --- a/dotCMS/src/integration-test/java/com/dotcms/MainSuite.java +++ b/dotCMS/src/integration-test/java/com/dotcms/MainSuite.java @@ -8,12 +8,14 @@ import com.dotcms.cache.lettuce.RedisClientTest; import com.dotcms.content.business.ObjectMapperTest; import com.dotcms.content.business.json.ContentletJsonAPITest; +import com.dotcms.content.elasticsearch.business.ESContentletAPIImplTest; import com.dotcms.content.elasticsearch.business.ESIndexAPITest; import com.dotcms.content.elasticsearch.business.ElasticsearchUtilTest; import com.dotcms.content.elasticsearch.util.ESMappingUtilHelperTest; import com.dotcms.content.model.hydration.MetadataDelegateTest; import com.dotcms.contenttype.business.ContentTypeInitializerTest; import com.dotcms.contenttype.business.DotAssetBaseTypeToContentTypeStrategyImplTest; +import com.dotcms.contenttype.business.SiteAndFolderResolverImplTest; import com.dotcms.contenttype.business.StoryBlockAPITest; import com.dotcms.contenttype.test.DotAssetAPITest; import com.dotcms.csspreproc.CSSCacheTest; @@ -23,13 +25,16 @@ import com.dotcms.ema.EMAWebInterceptorTest; import com.dotcms.enterprise.cluster.ClusterFactoryTest; import com.dotcms.enterprise.publishing.bundler.URLMapBundlerTest; +import com.dotcms.enterprise.publishing.remote.PushPublishBundleGeneratorTest; import com.dotcms.enterprise.publishing.remote.StaticPushPublishBundleGeneratorTest; import com.dotcms.enterprise.publishing.remote.bundler.ContainerBundlerTest; import com.dotcms.enterprise.publishing.remote.bundler.ContentBundlerTest; import com.dotcms.enterprise.publishing.remote.bundler.ContentTypeBundlerTest; +import com.dotcms.enterprise.publishing.remote.bundler.DependencyBundlerTest; import com.dotcms.enterprise.publishing.remote.bundler.FolderBundlerTest; import com.dotcms.enterprise.publishing.remote.bundler.HostBundlerTest; import com.dotcms.enterprise.publishing.remote.bundler.LinkBundlerTest; +import com.dotcms.enterprise.publishing.remote.bundler.RuleBundlerTest; import com.dotcms.enterprise.publishing.remote.bundler.TemplateBundlerTest; import com.dotcms.enterprise.publishing.remote.bundler.WorkflowBundlerTest; import com.dotcms.enterprise.publishing.remote.handler.ContentHandlerTest; @@ -37,7 +42,12 @@ import com.dotcms.enterprise.publishing.remote.handler.HandlerUtilTest; import com.dotcms.enterprise.publishing.staticpublishing.AWSS3PublisherTest; import com.dotcms.enterprise.publishing.staticpublishing.LanguageFolderTest; +import com.dotcms.enterprise.publishing.staticpublishing.StaticPublisherIntegrationTest; +import com.dotcms.enterprise.rules.RulesAPIImplIntegrationTest; +import com.dotcms.experiments.business.ExperimentAPIImpIT; +import com.dotcms.experiments.business.web.ExperimentWebAPIImplIT; import com.dotcms.filters.interceptor.meta.MetaWebInterceptorTest; +import com.dotcms.graphql.DotGraphQLHttpServletTest; import com.dotcms.integritycheckers.HostIntegrityCheckerTest; import com.dotcms.integritycheckers.IntegrityUtilTest; import com.dotcms.junit.MainBaseSuite; @@ -46,11 +56,14 @@ import com.dotcms.publisher.bundle.business.BundleAPITest; import com.dotcms.publisher.bundle.business.BundleFactoryTest; import com.dotcms.publisher.business.PublishAuditAPITest; +import com.dotcms.publisher.business.PublishQueueElementTransformerTest; import com.dotcms.publisher.receiver.BundlePublisherTest; import com.dotcms.publisher.util.DependencyManagerTest; +import com.dotcms.publisher.util.DependencyModDateUtilTest; import com.dotcms.publishing.BundlerUtilTest; import com.dotcms.publishing.PublisherFilterImplTest; import com.dotcms.publishing.PushPublishFiltersInitializerTest; +import com.dotcms.publishing.job.SiteSearchJobImplTest; import com.dotcms.publishing.manifest.CSVManifestBuilderTest; import com.dotcms.publishing.manifest.CSVManifestReaderTest; import com.dotcms.publishing.manifest.ManifestReaderFactoryTest; @@ -62,6 +75,7 @@ import com.dotcms.rendering.velocity.viewtools.JSONToolTest; import com.dotcms.rendering.velocity.viewtools.MessageToolTest; import com.dotcms.rendering.velocity.viewtools.XmlToolTest; +import com.dotcms.rendering.velocity.viewtools.XsltToolTest; import com.dotcms.rendering.velocity.viewtools.content.StoryBlockMapTest; import com.dotcms.rendering.velocity.viewtools.content.StoryBlockTest; import com.dotcms.rest.BundlePublisherResourceIntegrationTest; @@ -88,6 +102,7 @@ import com.dotcms.storage.StoragePersistenceAPITest; import com.dotcms.storage.repository.HashedLocalFileRepositoryManagerTest; import com.dotcms.translate.GoogleTranslationServiceIntegrationTest; +import com.dotcms.uuid.shorty.LegacyShortyIdApiTest; import com.dotcms.variant.VariantAPITest; import com.dotcms.variant.VariantFactoryTest; import com.dotcms.variant.business.VariantCacheTest; @@ -97,6 +112,7 @@ import com.dotmarketing.business.PermissionBitFactoryImplTest; import com.dotmarketing.business.VersionableFactoryImplTest; import com.dotmarketing.business.helper.PermissionHelperTest; +import com.dotmarketing.cache.FolderCacheImplIntegrationTest; import com.dotmarketing.common.db.DBTimeZoneCheckTest; import com.dotmarketing.filters.AutoLoginFilterTest; import com.dotmarketing.image.filter.ImageFilterAPIImplTest; @@ -106,7 +122,9 @@ import com.dotmarketing.portlets.categories.business.CategoryFactoryTest; import com.dotmarketing.portlets.cmsmaintenance.factories.CMSMaintenanceFactoryTest; import com.dotmarketing.portlets.containers.business.ContainerFactoryImplTest; +import com.dotmarketing.portlets.containers.business.ContainerStructureFinderStrategyResolverTest; import com.dotmarketing.portlets.contentlet.business.ContentletCacheImplTest; +import com.dotmarketing.portlets.contentlet.business.web.ContentletWebAPIImplIntegrationTest; import com.dotmarketing.portlets.contentlet.model.ContentletDependenciesTest; import com.dotmarketing.portlets.contentlet.model.IntegrationResourceLinkTest; import com.dotmarketing.portlets.fileassets.business.FileAssetAPIImplIntegrationTest; @@ -141,7 +159,6 @@ import com.dotmarketing.quartz.DotStatefulJobTest; import com.dotmarketing.quartz.job.CleanUpFieldReferencesJobTest; import com.dotmarketing.quartz.job.IntegrityDataGenerationJobTest; -import com.dotmarketing.quartz.job.StartEndScheduledExperimentsJobTest; import com.dotmarketing.startup.StartupTasksExecutorTest; import com.dotmarketing.startup.runalways.Task00050LoadAppsSecretsTest; import com.dotmarketing.startup.runonce.Task05195CreatesDestroyActionAndAssignDestroyDefaultActionsToTheSystemWorkflowTest; @@ -184,10 +201,12 @@ import com.dotmarketing.startup.runonce.Task220606UpdatePushNowActionletNameTest; import com.dotmarketing.startup.runonce.Task220822CreateVariantTableTest; import com.dotmarketing.startup.runonce.Task220824CreateDefaultVariantTest; +import com.dotmarketing.startup.runonce.Task220825CreateVariantFieldTest; import com.dotmarketing.startup.runonce.Task220825MakeSomeSystemFieldsRemovableTest; import com.dotmarketing.startup.runonce.Task220829CreateExperimentsTableTest; import com.dotmarketing.startup.runonce.Task220912UpdateCorrectShowOnMenuPropertyTest; import com.dotmarketing.startup.runonce.Task220928AddLookbackWindowColumnToExperimentTest; +import com.dotmarketing.startup.runonce.Task221007AddVariantIntoPrimaryKeyTest; import com.dotmarketing.startup.runonce.Task230110MakeSomeSystemFieldsRemovableByBaseTypeTest; import com.dotmarketing.util.ConfigTest; import com.dotmarketing.util.HashBuilderTest; @@ -207,21 +226,14 @@ @RunWith(MainBaseSuite.class) @SuiteClasses({ - /* - StartEndScheduledExperimentsJobTest.class, - RulesAPIImplIntegrationTest.class, Task220825CreateVariantFieldTest.class, - */ // AnalyticsAPIImplTest.class, - /* Task221007AddVariantIntoPrimaryKeyTest.class, ESContentletAPIImplTest.class, ExperimentAPIImpIT.class, ExperimentWebAPIImplIT.class, - */ // AccessTokenRenewJobTest.class, - /* ContentletWebAPIImplIntegrationTest.class, // moved to top because of failures on GHA DependencyBundlerTest.class, // moved to top because of failures on GHA SiteAndFolderResolverImplTest.class, //Moved up to avoid conflicts with CT deletion @@ -414,8 +426,7 @@ com.dotmarketing.common.reindex.ReindexAPITest.class, com.dotmarketing.common.db.DotDatabaseMetaDataTest.class, com.dotmarketing.common.db.ParamsSetterTest.class, - com.dotmarketing.cms.urlmap.URLMapAPIImplTest.class - ,*/ + com.dotmarketing.cms.urlmap.URLMapAPIImplTest.class, com.dotmarketing.factories.PublishFactoryTest.class, com.dotmarketing.factories.WebAssetFactoryTest.class, com.dotmarketing.factories.MultiTreeAPITest.class, @@ -444,7 +455,7 @@ Task05210CreateDefaultDotAssetTest.class, CleanUpFieldReferencesJobTest.class, CachedParameterDecoratorTest.class, - //ContainerFactoryImplTest.class, test remove looks like creating new default host + ContainerFactoryImplTest.class, TemplateFactoryImplTest.class, TestConfig.class, ConfigTest.class, From 7d50702045589211b5ea4b4bfe331d171cfd93aa Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Mon, 6 Feb 2023 18:26:01 -0500 Subject: [PATCH 26/28] await index queue empty --- .../business/ContentletIndexAPIImplTest.java | 5 +++- .../common/reindex/ReindexQueueAPI.java | 9 ++++++++ .../common/reindex/ReindexQueueAPIImpl.java | 23 +++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/dotCMS/src/integration-test/java/com/dotcms/content/elasticsearch/business/ContentletIndexAPIImplTest.java b/dotCMS/src/integration-test/java/com/dotcms/content/elasticsearch/business/ContentletIndexAPIImplTest.java index 981192f8d915..234436946622 100644 --- a/dotCMS/src/integration-test/java/com/dotcms/content/elasticsearch/business/ContentletIndexAPIImplTest.java +++ b/dotCMS/src/integration-test/java/com/dotcms/content/elasticsearch/business/ContentletIndexAPIImplTest.java @@ -56,6 +56,7 @@ import com.dotmarketing.portlets.structure.model.Structure; import com.dotmarketing.portlets.templates.model.Template; import com.dotmarketing.sitesearch.business.SiteSearchAPI; +import com.dotmarketing.util.Config; import com.dotmarketing.util.Logger; import com.dotmarketing.util.ThreadUtils; import com.dotmarketing.util.UUIDGenerator; @@ -849,7 +850,9 @@ public void test_that_live_and_working_content_makes_it_into_the_index() throws indexAPI.removeContentFromIndex(content); ReindexThread.startThread(); APILocator.getReindexQueueAPI().addContentletReindex(content); - ThreadUtils.sleep(10000); + int waitTime=60; + boolean queueEmpty = APILocator.getReindexQueueAPI().waitForEmptyQueue(waitTime); + assertTrue("Index queue not empty after "+waitTime,queueEmpty); liveSearch = contentletAPI.searchIndex("+identifier:" + content.getIdentifier() + " +live:true", 1, 0, "modDate", user, false); workingSearch = contentletAPI.searchIndex("+identifier:" + content.getIdentifier() + " +live:false", 1, 0, "modDate", user, false); assert(liveSearch.size()>0); diff --git a/dotCMS/src/main/java/com/dotmarketing/common/reindex/ReindexQueueAPI.java b/dotCMS/src/main/java/com/dotmarketing/common/reindex/ReindexQueueAPI.java index cddd473a49cd..6121199a02dd 100644 --- a/dotCMS/src/main/java/com/dotmarketing/common/reindex/ReindexQueueAPI.java +++ b/dotCMS/src/main/java/com/dotmarketing/common/reindex/ReindexQueueAPI.java @@ -77,6 +77,15 @@ public interface ReindexQueueAPI { */ public boolean areRecordsLeftToIndex() throws DotDataException; + /** + * Wait for the queue to be empty for the local server + * + * @param maxWaitSeconds - max number of seconds to wait for the queue to be empty + * @return true if the queue is empty, false if the maxWaitSeconds was reached + * @throws DotDataException + */ + public boolean waitForEmptyQueue(int maxWaitSeconds) throws DotDataException; + /** * @return the serverId */ diff --git a/dotCMS/src/main/java/com/dotmarketing/common/reindex/ReindexQueueAPIImpl.java b/dotCMS/src/main/java/com/dotmarketing/common/reindex/ReindexQueueAPIImpl.java index 3dba67df756f..2ca023e4053c 100644 --- a/dotCMS/src/main/java/com/dotmarketing/common/reindex/ReindexQueueAPIImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/common/reindex/ReindexQueueAPIImpl.java @@ -88,6 +88,29 @@ public boolean areRecordsLeftToIndex() throws DotDataException { public long recordsInQueue() throws DotDataException { return recordsInQueue(DbConnectionFactory.getConnection()); } + + @Override + public boolean waitForEmptyQueue(int maxWaitSeconds) throws DotDataException { + // Uses Busy Waiting with polling of DB every second to check if there are records in the queue + // This should be replaced with a more efficient solution that uses a notification mechanism + // but that would require a more complex implementation and this is required for some tests currently + try { + long recordsInQueue = recordsInQueue(); + while (recordsInQueue > 0) { + Logger.info(this, "Waiting for queue to be empty. Records in queue: " + recordsInQueue); + if (maxWaitSeconds <= 0) { + return false; + } + Thread.sleep(1000); + maxWaitSeconds--; + recordsInQueue = recordsInQueue(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); // restore interrupted status + Logger.error(this, e.getMessage(), e); + } + return true; + } @Override @CloseDBIfOpened From ef29177e9f9046735d3359c488c92ed8d9e531fc Mon Sep 17 00:00:00 2001 From: dotcmsbuild Date: Mon, 6 Feb 2023 18:26:26 -0500 Subject: [PATCH 27/28] Track config overrides between tests --- .../java/com/dotcms/junit/MainBaseSuite.java | 40 +++++++++++++++++++ .../java/com/dotmarketing/util/Config.java | 34 +++++++++++++++- 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/dotCMS/src/integration-test/java/com/dotcms/junit/MainBaseSuite.java b/dotCMS/src/integration-test/java/com/dotcms/junit/MainBaseSuite.java index dd7c57a061f4..42195ee3b690 100644 --- a/dotCMS/src/integration-test/java/com/dotcms/junit/MainBaseSuite.java +++ b/dotCMS/src/integration-test/java/com/dotcms/junit/MainBaseSuite.java @@ -1,12 +1,20 @@ package com.dotcms.junit; +import static org.junit.Assert.assertTrue; + +import com.dotcms.IntegrationTestBase; import com.dotcms.business.bytebuddy.ByteBuddyFactory; import com.dotcms.util.StdOutErrLog; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.common.reindex.ReindexQueueAPI; import com.dotmarketing.db.DbConnectionFactory; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.util.Config; import java.util.LinkedList; import java.util.List; import com.dotmarketing.util.Logger; +import java.util.Map; import org.junit.runner.Description; import org.junit.runner.Runner; import org.junit.runner.notification.RunNotifier; @@ -61,7 +69,32 @@ public Description getDescription() { @Override public void run(RunNotifier notifier) { + Config.getOverrides().forEach((k,v)->{ + Logger.warn(this.getClass(), ()->"Config overrides before test: " + k + " = " + v); + }); + + Logger.info(MainBaseSuite.class,"Checking indexer status"); + try { + int waitTime=60; + ReindexQueueAPI queueAPI = APILocator.getReindexQueueAPI(); + if (queueAPI.areRecordsLeftToIndex()) { + Logger.info(MainBaseSuite.class,"Indexer is not idle, waiting for it to finish"); + boolean queueEmpty = queueAPI.waitForEmptyQueue(waitTime); + if (queueEmpty) { + Logger.info(MainBaseSuite.class,"Indexer Completed"); + } else { + Logger.info(MainBaseSuite.class,"Indexer did not complete after "+waitTime+" seconds"); + } + } + } catch (DotDataException e) { + throw new RuntimeException(e); + } + + + Map overrideTracker = Config.getOverrides(); + try { + this.runner.run(notifier); } finally { if (DbConnectionFactory.inTransaction()) @@ -70,6 +103,13 @@ public void run(RunNotifier notifier) { if (DbConnectionFactory.connectionExists()) Logger.error(DotRunner.class,"Test "+this.getDescription()+" has open connection after"); DbConnectionFactory.closeSilently(); + + Map modifiedOverrides = Config.compareOverrides(overrideTracker); + if (!modifiedOverrides.isEmpty()) { + String mapContents = modifiedOverrides.entrySet().stream() + .map(entry -> entry.getKey() + "=" + entry.getValue()).reduce((a, b) -> a + ", " + b).orElse("empty"); + Logger.warn(IntegrationTestBase.class, "Modified Config overrides after:" + mapContents); + } } } } diff --git a/dotCMS/src/main/java/com/dotmarketing/util/Config.java b/dotCMS/src/main/java/com/dotmarketing/util/Config.java index 1a2b8b6717f7..b33c5a039656 100644 --- a/dotCMS/src/main/java/com/dotmarketing/util/Config.java +++ b/dotCMS/src/main/java/com/dotmarketing/util/Config.java @@ -1,5 +1,6 @@ package com.dotmarketing.util; +import com.dotcms.exception.ExceptionUtil; import com.google.common.collect.ImmutableList; import java.io.File; import java.io.IOException; @@ -9,9 +10,12 @@ import java.net.URL; import java.nio.file.Files; import java.util.Date; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.NoSuchElementException; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.PropertiesConfiguration; @@ -46,6 +50,8 @@ public class Config { public static final AtomicBoolean useWatcherMode = new AtomicBoolean(true); public static final AtomicBoolean isWatching = new AtomicBoolean(false); + public static final Map testOverrideTracker = new ConcurrentHashMap<>(); + /** * If this property is set in the dotmarketing-config, will try to use the interpolator for the properties @@ -554,11 +560,37 @@ public static boolean getBooleanProperty (String name, boolean defaultVal) { */ public static void setProperty(String key, Object value) { if(props!=null) { - Logger.info(Config.class, "Setting property: " + key + " = " + value); + if (value==null){ + Logger.warn(Config.class, "Setting null: for property: " + key + ExceptionUtil.getCurrentStackTraceAsString()); + } + if(value == null) { + testOverrideTracker.remove(key); + return; + } else { + testOverrideTracker.put(key, value.toString()); + } + Logger.info(Config.class, "Setting property: " + key + " to " + value); props.setProperty(key, value); } } + public static Map getOverrides() { + return Map.copyOf(testOverrideTracker); + } + + public static Map compareOverrides(Map before) { + Map after = getOverrides(); + Map diff = new HashMap(); + for(String key : after.keySet()) { + if(!before.containsKey(key) || !before.get(key).equals(after.get(key))) { + diff.put(key, after.get(key)); + } + } + return diff; + } + + + /** * * @return From aa5f8668409ae0d952382a4747d373f5af71c6f2 Mon Sep 17 00:00:00 2001 From: Victor Alfaro Date: Tue, 7 Feb 2023 09:58:47 -0600 Subject: [PATCH 28/28] #23631: docker copy fix --- .github/actions/run-postman-tests/dist/index.js | 2 +- .github/actions/run-postman-tests/src/postman.ts | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/actions/run-postman-tests/dist/index.js b/.github/actions/run-postman-tests/dist/index.js index dffb39640b24..c69d7e8ad4e3 100644 --- a/.github/actions/run-postman-tests/dist/index.js +++ b/.github/actions/run-postman-tests/dist/index.js @@ -186,7 +186,7 @@ exports.runTests = runTests; */ const copyOutputs = () => __awaiter(void 0, void 0, void 0, function* () { const logFile = path.join(dotCmsRoot, `${normalize(parallelCollection)}.log`); - yield execCmd(toCommand('docker', ['cp', 'docker_dotcms-app_1:/srv/dotserver/tomcat-9.0.60/logs/dotcms.log', logFile])); + yield execCmd(toCommand('docker', ['cp', 'dotcms-app_1:/srv/dotserver/tomcat-9.0.60/logs/dotcms.log', logFile])); }); const copyHeaderAndFooter = () => __awaiter(void 0, void 0, void 0, function* () { yield execCmd(toCommand('cp', [path.join(resourcesFolder, 'postman-results-header.html'), reportFolder])); diff --git a/.github/actions/run-postman-tests/src/postman.ts b/.github/actions/run-postman-tests/src/postman.ts index 4ffa4e279818..10569c833ad7 100644 --- a/.github/actions/run-postman-tests/src/postman.ts +++ b/.github/actions/run-postman-tests/src/postman.ts @@ -106,9 +106,7 @@ export const runTests = async (): Promise => { */ const copyOutputs = async () => { const logFile = path.join(dotCmsRoot, `${normalize(parallelCollection)}.log`) - await execCmd( - toCommand('docker', ['cp', 'docker_dotcms-app_1:/srv/dotserver/tomcat-9.0.60/logs/dotcms.log', logFile]) - ) + await execCmd(toCommand('docker', ['cp', 'dotcms-app_1:/srv/dotserver/tomcat-9.0.60/logs/dotcms.log', logFile])) } const copyHeaderAndFooter = async () => {