SOLR-17556: Add solr-home option for examples

This is important for testing, but is much better than modifying the server directory in solr.
Also remove test_utf8 from the smoke test, and the data for the test.

forward-ported from 9x commit: 9d67ad6

solr/bin/solr

@@ -718,6 +718,7 @@ if [ $# -gt 0 ]; then
             fi
 
             SOLR_HOME="$2"
+            PASS_TO_RUN_EXAMPLE+=("--solr-home" "$SOLR_HOME")
             shift 2
         ;;
         --data-home)
@@ -736,6 +737,14 @@ if [ $# -gt 0 ]; then
             EXAMPLE="$2"
             shift 2
         ;;
+        --example-dir)
+            if [[ -z "$2" || "${2:0:1}" == "-" ]]; then
+              print_usage "$SCRIPT_CMD" "Example directory is required when using the $1 option!"
+              exit 1
+            fi
+            PASS_TO_RUN_EXAMPLE+=("--example-dir" "$2")
+            shift 2
+        ;;
         -f|--foreground)
             FG="true"
             shift

solr/core/src/java/org/apache/solr/cli/RunExampleTool.java

@@ -48,6 +48,7 @@
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
 import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.EnvUtils;
 import org.noggit.CharArr;
 import org.noggit.JSONWriter;
 
@@ -116,6 +117,16 @@ public class RunExampleTool extends ToolBase {
               "Path to the Solr example directory; if not provided, ${serverDir}/../example is expected to exist.")
           .build();
 
+  private static final Option SOLR_HOME_OPTION =
+      Option.builder()
+          .longOpt("solr-home")
+          .hasArg()
+          .argName("SOLR_HOME_DIR")
+          .required(false)
+          .desc(
+              "Path to the Solr home directory; if not provided, ${serverDir}/solr is expected to exist.")
+          .build();
+
   private static final Option URL_SCHEME_OPTION =
       Option.builder()
           .longOpt("url-scheme")
@@ -165,6 +176,7 @@ public class RunExampleTool extends ToolBase {
   protected String script;
   protected Path serverDir;
   protected Path exampleDir;
+  protected Path solrHomeDir;
   protected String urlScheme;
 
   /** Default constructor used by the framework when running as a command-line application. */
@@ -190,6 +202,7 @@ public Options getOptions() {
         .addOption(EXAMPLE_OPTION)
         .addOption(SCRIPT_OPTION)
         .addOption(SERVER_DIR_OPTION)
+        .addOption(SOLR_HOME_OPTION)
         .addOption(FORCE_OPTION)
         .addOption(EXAMPLE_DIR_OPTION)
         .addOption(URL_SCHEME_OPTION)
@@ -204,6 +217,7 @@ public Options getOptions() {
   @Override
   public void runImpl(CommandLine cli) throws Exception {
     this.urlScheme = cli.getOptionValue(URL_SCHEME_OPTION, "http");
+    String exampleType = cli.getOptionValue(EXAMPLE_OPTION);
 
     serverDir = Path.of(cli.getOptionValue(SERVER_DIR_OPTION));
     if (!Files.isDirectory(serverDir))
@@ -242,15 +256,35 @@ public void runImpl(CommandLine cli) throws Exception {
               + exampleDir.toAbsolutePath()
               + " is not a directory!");
 
+    if (cli.hasOption(SOLR_HOME_OPTION)) {
+      solrHomeDir = Path.of(cli.getOptionValue(SOLR_HOME_OPTION));
+    } else {
+      String solrHomeProp = EnvUtils.getProperty("solr.home");
+      if (solrHomeProp != null && !solrHomeProp.isEmpty()) {
+        solrHomeDir = Path.of(solrHomeProp);
+      } else if ("cloud".equals(exampleType)) {
+        solrHomeDir = exampleDir.resolve("cloud");
+        if (!Files.isDirectory(solrHomeDir)) Files.createDirectory(solrHomeDir);
+      } else {
+        solrHomeDir = serverDir.resolve("solr");
+      }
+    }
+    if (!Files.isDirectory(solrHomeDir))
+      throw new IllegalArgumentException(
+          "Value of --solr-home option is invalid! "
+              + solrHomeDir.toAbsolutePath()
+              + " is not a directory!");
+
     echoIfVerbose(
         "Running with\nserverDir="
             + serverDir.toAbsolutePath()
             + ",\nexampleDir="
             + exampleDir.toAbsolutePath()
+            + ",\nsolrHomeDir="
+            + solrHomeDir.toAbsolutePath()
             + "\nscript="
             + script);
 
-    String exampleType = cli.getOptionValue(EXAMPLE_OPTION);
     if ("cloud".equals(exampleType)) {
       runCloudExample(cli);
     } else if ("techproducts".equals(exampleType)
@@ -275,8 +309,7 @@ protected void runExample(CommandLine cli, String exampleName) throws Exception
     int port =
         Integer.parseInt(
             cli.getOptionValue(PORT_OPTION, System.getenv().getOrDefault("SOLR_PORT", "8983")));
-    Map<String, Object> nodeStatus =
-        startSolr(serverDir.resolve("solr"), isCloudMode, cli, port, zkHost, 30);
+    Map<String, Object> nodeStatus = startSolr(solrHomeDir, isCloudMode, cli, port, zkHost, 30);
 
     String solrUrl = CLIUtils.normalizeSolrUrl((String) nodeStatus.get("baseUrl"));
 
@@ -494,8 +527,6 @@ protected void runCloudExample(CommandLine cli) throws Exception {
       // Override the old default port numbers if user has started the example overriding SOLR_PORT
       cloudPorts = new int[] {defaultPort, defaultPort + 1, defaultPort + 2, defaultPort + 3};
     }
-    Path cloudDir = exampleDir.resolve("cloud");
-    if (!Files.isDirectory(cloudDir)) Files.createDirectory(cloudDir);
 
     echo("\nWelcome to the SolrCloud example!\n");
 
@@ -541,9 +572,9 @@ protected void runCloudExample(CommandLine cli) throws Exception {
     }
 
     // setup a unique solr.solr.home directory for each node
-    Path node1Dir = setupExampleDir(serverDir, cloudDir, "node1");
+    Path node1Dir = setupSolrHomeDir(serverDir, solrHomeDir, "node1");
     for (int n = 2; n <= numNodes; n++) {
-      Path nodeNDir = cloudDir.resolve("node" + n);
+      Path nodeNDir = solrHomeDir.resolve("node" + n);
       if (!Files.isDirectory(nodeNDir)) {
         echo("Cloning " + node1Dir.toAbsolutePath() + " into\n   " + nodeNDir.toAbsolutePath());
         PathUtils.copyDirectory(node1Dir, nodeNDir, StandardCopyOption.REPLACE_EXISTING);
@@ -574,7 +605,7 @@ protected void runCloudExample(CommandLine cli) throws Exception {
       // start the other nodes
       for (int n = 1; n < numNodes; n++)
         startSolr(
-            cloudDir.resolve("node" + (n + 1)).resolve("solr"),
+            solrHomeDir.resolve("node" + (n + 1)).resolve("solr"),
             true,
             cli,
             cloudPorts[n],
@@ -668,7 +699,7 @@ protected Map<String, Object> startSolr(
     String callScript = (!isWindows && cwd.equals(binDir.getParent())) ? "bin/solr" : script;
 
     String cwdPath = cwd.toAbsolutePath().toString();
-    String solrHome = solrHomeDir.toAbsolutePath().toString();
+    String solrHome = solrHomeDir.toAbsolutePath().toRealPath().toString();
 
     // don't display a huge path for solr home if it is relative to the cwd
     if (!isWindows && cwdPath.length() > 1 && solrHome.startsWith(cwdPath))
@@ -682,11 +713,12 @@ protected Map<String, Object> startSolr(
     String startCmd =
         String.format(
             Locale.ROOT,
-            "\"%s\" start %s -p %d --solr-home \"%s\" %s %s %s %s %s %s %s %s",
+            "\"%s\" start %s -p %d --solr-home \"%s\" --server-dir \"%s\" %s %s %s %s %s %s %s %s",
             callScript,
             cloudModeArg,
             port,
             solrHome,
+            serverDir.toAbsolutePath(),
             hostArg,
             zkHostArg,
             memArg,
@@ -952,7 +984,7 @@ protected Map<String, Object> getNodeStatus(String solrUrl, String credentials,
     return nodeStatus;
   }
 
-  protected Path setupExampleDir(Path serverDir, Path exampleParentDir, String dirName)
+  protected Path setupSolrHomeDir(Path serverDir, Path solrHomeParentDir, String dirName)
       throws IOException {
     Path solrXml = serverDir.resolve("solr").resolve("solr.xml");
     if (!Files.isRegularFile(solrXml))
@@ -964,7 +996,7 @@ protected Path setupExampleDir(Path serverDir, Path exampleParentDir, String dir
       throw new IllegalArgumentException(
           "Value of --server-dir option is invalid! " + zooCfg.toAbsolutePath() + " not found!");
 
-    Path solrHomeDir = exampleParentDir.resolve(dirName).resolve("solr");
+    Path solrHomeDir = solrHomeParentDir.resolve(dirName).resolve("solr");
     if (!Files.isDirectory(solrHomeDir)) {
       echo("Creating Solr home directory " + solrHomeDir);
       Files.createDirectories(solrHomeDir);

solr/core/src/test/org/apache/solr/cli/TestSolrCLIRunExample.java

@@ -121,9 +121,12 @@ public int execute(org.apache.commons.exec.CommandLine cmd) throws IOException {
 
           String solrHomeDir = getArg("--solr-home", args);
           int port = Integer.parseInt(getArg("-p", args));
-          String solrxml =
-              Files.readString(
-                  Paths.get(solrHomeDir).resolve("solr.xml"), Charset.defaultCharset());
+          Path solrXmlPath = Paths.get(solrHomeDir).resolve("solr.xml");
+          if (!Files.exists(solrXmlPath)) {
+            String solrServerDir = getArg("--server-dir", args);
+            solrXmlPath = Paths.get(solrServerDir).resolve("solr").resolve("solr.xml");
+          }
+          String solrxml = Files.readString(solrXmlPath, Charset.defaultCharset());
 
           JettyConfig jettyConfig = JettyConfig.builder().setPort(port).build();
           try {
@@ -318,34 +321,31 @@ public void tearDown() throws Exception {
   }
 
   @Test
-  @LuceneTestCase.Nightly
   public void testTechproductsExample() throws Exception {
     testExample("techproducts");
   }
 
   @Test
-  @LuceneTestCase.Nightly
   public void testSchemalessExample() throws Exception {
     testExample("schemaless");
   }
 
   @Test
-  @LuceneTestCase.Nightly
   public void testFilmsExample() throws Exception {
     testExample("films");
   }
 
   protected void testExample(String exampleName) throws Exception {
     // Occasionally we want to test in User Managed mode, not the default SolrCloud mode.
     String testStandaloneMode = LuceneTestCase.rarely() ? "--user-managed" : "";
-    File solrHomeDir = new File(ExternalPaths.SERVER_HOME);
-    if (!solrHomeDir.isDirectory()) {
-      fail(solrHomeDir.getAbsolutePath() + " not found and is required to run this test!");
+    File defaultSolrHomeDir = new File(ExternalPaths.SERVER_HOME);
+    if (!defaultSolrHomeDir.isDirectory()) {
+      fail(defaultSolrHomeDir.getAbsolutePath() + " not found and is required to run this test!");
     }
 
     Path tmpDir = createTempDir();
-    File solrExampleDir = tmpDir.toFile();
-    File solrServerDir = solrHomeDir.getParentFile();
+    File testSolrHomeDir = tmpDir.toFile();
+    File solrServerDir = defaultSolrHomeDir.getParentFile();
 
     for (int pass = 0; pass < 2; pass++) {
       // need a port to start the example server on
@@ -362,8 +362,8 @@ protected void testExample(String exampleName) throws Exception {
             exampleName,
             "--server-dir",
             solrServerDir.getAbsolutePath(),
-            "--example-dir",
-            solrExampleDir.getAbsolutePath(),
+            "--solr-home",
+            testSolrHomeDir.getAbsolutePath(),
             "-p",
             String.valueOf(bindPort),
             testStandaloneMode
@@ -421,13 +421,13 @@ protected void testExample(String exampleName) throws Exception {
             numFound = solrClient.query(query).getResults().getNumFound();
           }
           assertEquals(
-              "expected 32 docs in the "
+              "expected 31 docs in the "
                   + exampleName
                   + " example but found "
                   + numFound
                   + ", output: "
                   + toolOutput,
-              32,
+              31,
               numFound);
         }
       }

solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java

@@ -129,7 +129,7 @@ public void testLoadDocsIntoGettingStartedCollection() throws Exception {
     CommandLine postCli = SolrCLI.processCommandLineArgs(postTool, argsForPost);
     postTool.runTool(postCli);
 
-    int expectedXmlDocCount = 32;
+    int expectedXmlDocCount = 31;
 
     int numFound = 0;
 

solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerAPI.java

@@ -239,7 +239,7 @@ public void testAddTechproductsProgressively() throws Exception {
     schemaDesignerAPI.query(req, rsp);
     assertNotNull(rsp.getResponseHeader());
     SolrDocumentList results = (SolrDocumentList) rsp.getResponse();
-    assertEquals(48, results.getNumFound());
+    assertEquals(47, results.getNumFound());
 
     // publish schema to a config set that can be used by real collections
     reqParams.clear();

solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java

@@ -323,7 +323,7 @@ public void testMultipleSearchResults() throws Exception {
     assertU(commit());
 
     assertJQ(
-        req("q", "id:myid*", "fl", "*"),
+        req("q", "id:myid*", "fl", "*", "sort", "id asc"),
         "/response/docs==["
             + "{'id':'myid1','test_is_dvo':[101,102,103]},"
             + "{'id':'myid2','test_is_dvo':[201,202]},"

solr/packaging/test/test_post.bats

@@ -120,8 +120,8 @@ teardown() {
   # We filter to xml,json,and csv as we don't want to invoke the Extract handler, and are running it as a dry run
   run solr post --dry-run --filetypes xml,json,csv --solr-url http://localhost:${SOLR_PORT} -c foobar --skip-commit ${SOLR_TIP}/example/exampledocs
 
-  assert_output --partial 'Dry run complete. 16 would have been indexed.'
-  refute_output --partial '16 files indexed.'
+  assert_output --partial 'Dry run complete. 15 would have been indexed.'
+  refute_output --partial '15 files indexed.'
   refute_output --partial 'ERROR'
 }
 
@@ -132,10 +132,10 @@ teardown() {
   # We filter to xml,json,and csv as we don't want to invoke the Extract handler.
   run solr post --filetypes xml,json,csv --solr-url http://localhost:${SOLR_PORT} -c mixed_content ${SOLR_TIP}/example/exampledocs
 
-  assert_output --partial '16 files indexed.'
+  assert_output --partial '15 files indexed.'
   refute_output --partial 'ERROR'
   run curl "http://localhost:${SOLR_PORT}/solr/mixed_content/select?q=*:*"
-  assert_output --partial '"numFound":46'
+  assert_output --partial '"numFound":45'
 }
 
 # this test doesn't complete due to issues in posting to the /extract handler

solr/solr-ref-guide/modules/getting-started/pages/tutorial-techproducts.adoc

@@ -195,9 +195,8 @@ POSTing file sample.html (text/html) to [base]/extract
 POSTing file sd500.xml (application/xml) to [base]
 POSTing file solr-word.pdf (application/pdf) to [base]/extract
 POSTing file solr.xml (application/xml) to [base]
-POSTing file utf8-example.xml (application/xml) to [base]
 POSTing file vidcard.xml (application/xml) to [base]
-20 files indexed.
+19 files indexed.
 COMMITting Solr index changes to http://localhost:8983/solr/techproducts/update...
 Time spent: 0:00:00.822
 ----