diff --git a/.asf.yaml b/.asf.yaml index 57999445552..9aa6153333c 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -29,6 +29,7 @@ github: branch_9_5: {} branch_9_6: {} branch_9_7: {} + branch_9_8: {} branch_9x: {} protected_tags: diff --git a/.github/labeler.yml b/.github/labeler.yml index 793957db16d..a3b7d5fe64c 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -2,7 +2,8 @@ dependencies: - changed-files: - any-glob-to-any-file: - - versions.props + - gradle/libs.versions.toml # Solr 10+ + - versions.props # Solr < v10 - versions.lock - solr/licenses/** @@ -122,11 +123,6 @@ module:gcs-repository: - any-glob-to-any-file: - solr/modules/gcs-repository/** -module:hadoop-auth: - - changed-files: - - any-glob-to-any-file: - - solr/modules/hadoop-auth/** - module:hdfs: - changed-files: - any-glob-to-any-file: diff --git a/build-tools/build-infra/build.gradle b/build-tools/build-infra/build.gradle index 0d78cce49bf..9b5ff387488 100644 --- a/build-tools/build-infra/build.gradle +++ b/build-tools/build-infra/build.gradle @@ -17,7 +17,7 @@ plugins { id "java-gradle-plugin" - id 'com.diffplug.spotless' version '6.5.2' apply false + alias(libs.plugins.diffplug.spotless) apply false } repositories { @@ -34,12 +34,9 @@ apply from: file('../../gradle/validation/check-environment.gradle') tasks.register("checkJdkInternalsExportedToGradle") {} apply from: file('../../gradle/validation/spotless.gradle') -// Load common script dependencies. -apply from: file("../scriptDepVersions.gradle") - java { - sourceCompatibility = scriptDepVersions['min-java-version'] - targetCompatibility = scriptDepVersions['min-java-version'] + sourceCompatibility = JavaVersion.toVersion(libs.versions.java.min.get()) + targetCompatibility = JavaVersion.toVersion(libs.versions.java.min.get()) } gradlePlugin { @@ -57,5 +54,5 @@ dependencies { implementation gradleApi() implementation localGroovy() - implementation "commons-codec:commons-codec:${scriptDepVersions['commons-codec']}" + implementation libs.commonscodec.commonscodec } diff --git a/build-tools/build-infra/settings.gradle b/build-tools/build-infra/settings.gradle index 350b781b7f4..7a55021b366 100644 --- a/build-tools/build-infra/settings.gradle +++ b/build-tools/build-infra/settings.gradle @@ -16,3 +16,12 @@ */ rootProject.name = 'build-infra' + +// Use project's version catalog for centralized dependency management +dependencyResolutionManagement { + versionCatalogs { + libs { + from(files("../../gradle/libs.versions.toml")) + } + } +} diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/WrapperDownloader.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/WrapperDownloader.java index 88d91028552..adb8f3eaf07 100644 --- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/WrapperDownloader.java +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/WrapperDownloader.java @@ -22,7 +22,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.net.HttpURLConnection; -import java.net.URL; +import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -54,7 +54,7 @@ public static void main(String[] args) { } public static void checkVersion() { - int major = Runtime.getRuntime().version().feature(); + int major = Runtime.version().feature(); if (major < 21 || major > 23) { throw new IllegalStateException( "java version must be between 21 and 23, your version: " + major); @@ -89,12 +89,12 @@ public void run(Path destination) throws IOException, NoSuchAlgorithmException { } } - URL url = - new URL( + URI uri = + URI.create( "https://raw.githubusercontent.com/gradle/gradle/v" + wrapperVersion + "/gradle/wrapper/gradle-wrapper.jar"); - System.err.println("Downloading gradle-wrapper.jar from " + url); + System.err.println("Downloading gradle-wrapper.jar from " + uri); // Zero-copy save the jar to a temp file Path temp = Files.createTempFile(destination.getParent(), ".gradle-wrapper", ".tmp"); @@ -103,7 +103,7 @@ public void run(Path destination) throws IOException, NoSuchAlgorithmException { int retryDelay = 30; HttpURLConnection connection; while (true) { - connection = (HttpURLConnection) url.openConnection(); + connection = (HttpURLConnection) uri.toURL().openConnection(); try { connection.connect(); } catch (IOException e) { diff --git a/build-tools/missing-doclet/build.gradle b/build-tools/missing-doclet/build.gradle index 2525f50d79b..11a7fc6e2a8 100644 --- a/build-tools/missing-doclet/build.gradle +++ b/build-tools/missing-doclet/build.gradle @@ -17,14 +17,13 @@ plugins { id 'java-library' - id 'com.diffplug.spotless' version '6.5.2' apply false + alias(libs.plugins.diffplug.spotless) apply false } repositories { mavenCentral() } -version = "1.0.0-SNAPSHOT" group = "org.apache.solr.tools" description = 'Doclet-based javadoc validation' @@ -36,12 +35,9 @@ apply from: file('../../gradle/validation/check-environment.gradle') tasks.register("checkJdkInternalsExportedToGradle") {} apply from: file('../../gradle/validation/spotless.gradle') -// Load common script dependencies. -apply from: file("../scriptDepVersions.gradle") - java { - sourceCompatibility = scriptDepVersions['min-java-version'] - targetCompatibility = scriptDepVersions['min-java-version'] + sourceCompatibility = JavaVersion.toVersion(libs.versions.java.min.get()) + targetCompatibility = JavaVersion.toVersion(libs.versions.java.min.get()) } tasks.withType(JavaCompile).configureEach { diff --git a/build-tools/missing-doclet/settings.gradle b/build-tools/missing-doclet/settings.gradle index 73d696331bc..c39219472be 100644 --- a/build-tools/missing-doclet/settings.gradle +++ b/build-tools/missing-doclet/settings.gradle @@ -16,3 +16,12 @@ */ rootProject.name = "missing-doclet" + +// Use project's version catalog for centralized dependency management +dependencyResolutionManagement { + versionCatalogs { + libs { + from(files("../../gradle/libs.versions.toml")) + } + } +} diff --git a/build-tools/missing-doclet/src/main/java/org/apache/lucene/missingdoclet/MissingDoclet.java b/build-tools/missing-doclet/src/main/java/org/apache/lucene/missingdoclet/MissingDoclet.java index 44c9c411a9e..8a3c252463c 100644 --- a/build-tools/missing-doclet/src/main/java/org/apache/lucene/missingdoclet/MissingDoclet.java +++ b/build-tools/missing-doclet/src/main/java/org/apache/lucene/missingdoclet/MissingDoclet.java @@ -341,8 +341,7 @@ private boolean hasInheritedJavadocs(Element element) { } // Check for methods up the types tree. - if (element instanceof ExecutableElement) { - ExecutableElement thisMethod = (ExecutableElement) element; + if (element instanceof ExecutableElement thisMethod) { Iterable superTypes = () -> superTypeForInheritDoc(thisMethod.getEnclosingElement()).iterator(); diff --git a/build.gradle b/build.gradle index 13065fcfc8f..9e40cfbcf88 100644 --- a/build.gradle +++ b/build.gradle @@ -21,21 +21,23 @@ import java.time.format.DateTimeFormatter plugins { id 'base' id 'solr.build-infra' - id 'com.palantir.consistent-versions' version '2.16.0' - id 'org.owasp.dependencycheck' version '9.0.8' - id 'ca.cutterslade.analyze' version '1.10.0' - id 'de.thetaphi.forbiddenapis' version '3.7' apply false - id 'de.undercouch.download' version '5.5.0' apply false - id 'net.ltgt.errorprone' version '3.1.0' apply false - id 'com.diffplug.spotless' version '6.5.2' apply false - id 'com.github.node-gradle.node' version '7.0.1' apply false -} -apply from: file('build-tools/scriptDepVersions.gradle') + alias(libs.plugins.carrotsearch.dependencychecks) + alias(libs.plugins.owasp.dependencycheck) + alias(libs.plugins.cutterslade.analyze) + alias(libs.plugins.benmanes.versions) + alias(libs.plugins.littlerobots.versioncatalogupdate) apply false + alias(libs.plugins.thetaphi.forbiddenapis) apply false + alias(libs.plugins.undercouch.download) apply false + alias(libs.plugins.ltgt.errorprone) apply false + alias(libs.plugins.diffplug.spotless) apply false + alias(libs.plugins.nodegradle.node) apply false + alias(libs.plugins.openapi.generator) apply false +} // Declare default Java versions for the entire project and for SolrJ separately -rootProject.ext.minJavaVersionDefault = JavaVersion.toVersion(scriptDepVersions['min-java-version']) -rootProject.ext.minJavaVersionSolrJ = JavaVersion.toVersion(scriptDepVersions['min-solrj-java-version']) +rootProject.ext.minJavaVersionDefault = JavaVersion.toVersion(libs.versions.java.min.get()) +rootProject.ext.minJavaVersionSolrJ = JavaVersion.toVersion(libs.versions.java.solrj.get()) apply from: file('gradle/globals.gradle') @@ -100,7 +102,7 @@ ext { } luceneBaseVersionProvider = project.provider { - def luceneVersion = getVersion('org.apache.lucene:lucene-core') + def luceneVersion = libs.versions.apache.lucene.get() def m = (luceneVersion =~ /^\d+\.\d+\.\d+\b/) if (!m) { throw GradleException("Can't strip base version from " + luceneVersion) @@ -150,7 +152,6 @@ apply from: file('gradle/validation/precommit.gradle') apply from: file('gradle/validation/forbidden-apis.gradle') apply from: file('gradle/validation/jar-checks.gradle') apply from: file('gradle/validation/git-status.gradle') -apply from: file('gradle/validation/versions-props-sorted.gradle') apply from: file('gradle/validation/validate-source-patterns.gradle') apply from: file('gradle/validation/rat-sources.gradle') apply from: file('gradle/validation/owasp-dependency-check.gradle') @@ -161,7 +162,7 @@ apply from: file('gradle/validation/validate-log-calls.gradle') apply from: file('gradle/validation/check-broken-links.gradle') apply from: file('gradle/validation/solr.config-file-sanity.gradle') - +apply from: file('gradle/validation/dependencies.gradle') apply from: file('gradle/validation/spotless.gradle') // Wire up included builds to some validation tasks. diff --git a/dev-docs/dependency-upgrades.adoc b/dev-docs/dependency-upgrades.adoc index 9f7372cc1bd..aa5cd93a2de 100644 --- a/dev-docs/dependency-upgrades.adoc +++ b/dev-docs/dependency-upgrades.adoc @@ -16,30 +16,57 @@ // specific language governing permissions and limitations // under the License. -Solr has lots of 3rd party dependencies, defined mainly in `versions.props`. +Solr has lots of 3rd party dependencies, defined in `gradle/libs.versions.toml`. Keeping them up-to-date is crucial for a number of reasons: * minimizing the risk of critical CVE vulnerabilities by staying on a recent and supported version * avoiding "dependency hell", that can arise from falling too far behind -Read the https://github.com/apache/solr/blob/main/help/dependencies.txt[help/dependencies.txt] file for an in-depth explanation of how gradle is deployed in Solr, using -https://github.com/palantir/gradle-consistent-versions[Gradle consistent-versions] plugin. +Read the https://github.com/apache/solr/blob/main/help/dependencies.txt[help/dependencies.txt] file for an in-depth +explanation of how dependencies are managed. == Manual dependency upgrades In order to upgrade a dependency, you need to run through a number of steps: 1. Identify the available versions from e.g. https://search.maven.org[Maven Central] -2. Update the version in `versions.props` file -3. Run `./gradlew --write-locks` to re-generate `versions.lock`. Note that this may cause a cascading effect where +2. Update the version in `gradle/libs.versions.toml` file +3. Run `./gradlew writeLocks` to re-generate `versions.lock`. Note that this may cause a cascading effect where the locked version of other dependencies also change. -4. Run `./gradlew updateLicenses` to re-generate SHA1 checksums of the new jar files. -5. Once in a while, a new version of a dependency will transitively bring in brand-new dependencies. +4. In case of a conflict, resolve the conflict according to `help/dependencies.txt` +5. Check if there are any constraints that are obsolete after the dependency update +6. Update the license and notice files of the changed dependencies. See `help/dependencies.txt` for + details. +7. Run `./gradlew updateLicenses` to re-generate SHA1 checksums of the new jar files. +8. Once in a while, a new version of a dependency will transitively bring in brand-new dependencies. You'll need to decide whether to keep or exclude them. See `help/dependencies.txt` for details. +=== Reviewing Constraints + +The constraints are defined in gradle/validation/dependencies.gradle. There, if the updated dependency is listed, +the constraint can be reviewed, updated or removed. + +The constraints fall into two "groups". In the first group there are dependency constraints from dependencies +that our project directly includes and require version alignment to sync the versions across all transitive +dependencies. In the second group are dependencies that are only present as transitive dependencies. +There, we try to follow the convention to provide additional information with "which dependencies use what version", +so that the next person reviewing the constraint does not have to look it up. However, this is quite time-consuming +to analyze the dependencies and therefore subject to change. + +In order to review a constraint, you have to check if the updated dependency is mentioned in any of the constraints, +either as a reason for another dependency constraint or as the constraint's dependency. Removing temporarily +a constraint, the task writeLocks will fail if the constraint is still required. + +This process and the constraints of dependencies.gradle are not optimal, as it is quite time-consuming and not obvious +by just looking at it. We just haven't found yet a more efficient way to maintain these constraints. + == Renovate bot Pull Requests + +The renovate bot may be replaced in the future with dependabot and this section may only be relevant for older +versions (<10.0). See https://lists.apache.org/thread/1sb9ttv3lp57z2yod1htx1fykp5sj73z for updates. + A member of the Solr community operates a Github bot running https://github.com/renovatebot/renovate[Renovate], which files Pull Requests to Solr with dependency upgrade proposals. The PRs are labeled `dependencies` and do include -changes resulting from `gradle --write-locks` and `updateLicenses`. +changes resulting from `./gradlew writeLocks` and `updateLicenses`. Community members and committers can then review, and if manual changes are needed, help bring the PR to completion. For many dependencies, a changelog is included in the PR text, which may help guide the upgrade decision. diff --git a/dev-docs/v2-api-conventions.adoc b/dev-docs/v2-api-conventions.adoc index d2159239694..74a760c2bfd 100644 --- a/dev-docs/v2-api-conventions.adoc +++ b/dev-docs/v2-api-conventions.adoc @@ -66,6 +66,14 @@ For use within the v2 API, the four "popular" HTTP methods have the following se * `PUT` - used for idempotent resource modifications. * `DELETE` - Used to delete or cleanup resource +== Errors + +v2 APIs should be consistent in how they report errors. Throwing a `SolrException` will convey +1.the error code as the HTTP response status code, as `responseHeader.status` and as `error.code`, and +1.the error message as `error.msg`. + +API calls that reference a specific resource (e.g. `specificCollName`, `specificAliasName`, `specificPropertyName` and others per the above list) that do not exist should return `SolrException.ErrorCode.NOT_FOUND` (HTTP 404). + == Exceptional Cases - "Command" APIs The pairing of semantic HTTP verbs and "resource"-based paths gives Solr an intuitive pattern for representing many operations, but not all. diff --git a/gradle/documentation/changes-to-html.gradle b/gradle/documentation/changes-to-html.gradle index 3b4ca69bf9d..af9d1b5fa9a 100644 --- a/gradle/documentation/changes-to-html.gradle +++ b/gradle/documentation/changes-to-html.gradle @@ -76,6 +76,13 @@ class ChangesToHtmlTask extends DefaultTask { def toHtml(File versionsFile) { def output = new ByteArrayOutputStream() + + // Check if the perl executable exists + if (!perlExists()) { + logger.warn("WARNING: Perl is not installed, skipping creating Changes.html") + return + } + def result = project.exec { executable project.externalTool("perl") standardInput changesFile.newInputStream() @@ -114,4 +121,14 @@ class ChangesToHtmlTask extends DefaultTask { throw new GradleException("Changes file ${changesFile} or Doap file ${changesDoapFile} not found.") } } + + def perlExists() { + try { + def process = "perl -v".execute() + process.waitFor() + return process.exitValue() == 0 + } catch (Exception e) { + return false + } + } } diff --git a/gradle/documentation/markdown.gradle b/gradle/documentation/markdown.gradle index 29d23d87c75..d9a890d72eb 100644 --- a/gradle/documentation/markdown.gradle +++ b/gradle/documentation/markdown.gradle @@ -33,10 +33,10 @@ buildscript { } dependencies { - classpath "com.vladsch.flexmark:flexmark:${scriptDepVersions['flexmark']}" - classpath "com.vladsch.flexmark:flexmark-ext-abbreviation:${scriptDepVersions['flexmark']}" - classpath "com.vladsch.flexmark:flexmark-ext-attributes:${scriptDepVersions['flexmark']}" - classpath "com.vladsch.flexmark:flexmark-ext-autolink:${scriptDepVersions['flexmark']}" + classpath libs.flexmark.flexmark + classpath libs.flexmark.extensions.abbreviation + classpath libs.flexmark.extensions.attributes + classpath libs.flexmark.extensions.autolink } } diff --git a/gradle/documentation/pull-lucene-javadocs.gradle b/gradle/documentation/pull-lucene-javadocs.gradle index 5fdc4a70040..17985d88f1b 100644 --- a/gradle/documentation/pull-lucene-javadocs.gradle +++ b/gradle/documentation/pull-lucene-javadocs.gradle @@ -45,11 +45,11 @@ configure(project(":solr:documentation")) { // from all Solr javadocs?) then perhaps we can find a way to build this list programatically? // - If these javadocs are (only every) consumed by the ref guide only, then these deps & associated tasks // should just be moved to the ref-guide build.gradle - javadocs group: 'org.apache.lucene', name: 'lucene-core', classifier: 'javadoc' - javadocs group: 'org.apache.lucene', name: 'lucene-analysis-common', classifier: 'javadoc' - javadocs group: 'org.apache.lucene', name: 'lucene-analysis-stempel', classifier: 'javadoc' - javadocs group: 'org.apache.lucene', name: 'lucene-queryparser', classifier: 'javadoc' - javadocs group: 'org.apache.lucene', name: 'lucene-spatial-extras', classifier: 'javadoc' + javadocs variantOf(libs.apache.lucene.core) { classifier 'javadoc' } + javadocs variantOf(libs.apache.lucene.analysis.common) { classifier 'javadoc' } + javadocs variantOf(libs.apache.lucene.analysis.stempel) { classifier 'javadoc' } + javadocs variantOf(libs.apache.lucene.queryparser) { classifier 'javadoc' } + javadocs variantOf(libs.apache.lucene.spatialextras) { classifier 'javadoc' } } diff --git a/gradle/generation/javacc.gradle b/gradle/generation/javacc.gradle index 54fc7e91359..0b70ba656ee 100644 --- a/gradle/generation/javacc.gradle +++ b/gradle/generation/javacc.gradle @@ -26,7 +26,7 @@ configure(rootProject) { } dependencies { - javacc "net.java.dev.javacc:javacc:${scriptDepVersions['javacc']}" + javacc libs.javacc.javacc } task javacc() { diff --git a/gradle/ide/eclipse.gradle b/gradle/ide/eclipse.gradle index a088c8b87a5..d7d453c39d9 100644 --- a/gradle/ide/eclipse.gradle +++ b/gradle/ide/eclipse.gradle @@ -21,65 +21,68 @@ import org.gradle.plugins.ide.eclipse.model.ClasspathEntry def resources = scriptResources(buildscript) configure(rootProject) { - apply plugin: "eclipse" + plugins.withType(JavaPlugin) { + apply plugin: "eclipse" - def relativize = { other -> rootProject.rootDir.relativePath(other).toString() } + def eclipseJavaVersion = propertyOrDefault("eclipse.javaVersion", libs.versions.java.min.get()) + def relativize = { other -> rootProject.rootDir.relativePath(other).toString() } - eclipse { - project { - name = "Apache Solr ${version}" - } + eclipse { + project { + name = "Apache Solr ${version}" + } - classpath { - downloadSources = true - downloadJavadoc = true - defaultOutputDir = file('build/eclipse') + classpath { + downloadSources = true + downloadJavadoc = true + defaultOutputDir = file('build/eclipse') - file { - beforeMerged { classpath -> classpath.entries.removeAll { it.kind == "src" } } + file { + beforeMerged { classpath -> classpath.entries.removeAll { it.kind == "src" } } - whenMerged { classpath -> - def projects = allprojects.findAll { prj -> - return prj.plugins.hasPlugin(JavaPlugin) && - prj.path != ":solr:solr-ref-guide" - } - - Set sources = [] - Set jars = [] - projects.each { prj -> - prj.sourceSets.each { sourceSet -> - sources += sourceSet.java.srcDirs.findAll { dir -> dir.exists() }.collect { dir -> relativize(dir) } - sources += sourceSet.resources.srcDirs.findAll { dir -> dir.exists() }.collect { dir -> relativize(dir) } + whenMerged { classpath -> + def projects = allprojects.findAll { prj -> + return prj.plugins.hasPlugin(JavaPlugin) && + prj.path != ":solr:solr-ref-guide" } - // This is hacky - we take the resolved compile classpath and just - // include JAR files from there. We should probably make it smarter - // by looking at real dependencies. But then: this Eclipse configuration - // doesn't really separate sources anyway so why bother. - jars += prj.configurations.compileClasspath.resolve() - jars += prj.configurations.testCompileClasspath.resolve() - } + Set sources = [] + Set jars = [] + projects.each { prj -> + prj.sourceSets.each { sourceSet -> + sources += sourceSet.java.srcDirs.findAll { dir -> dir.exists() }.collect { dir -> relativize(dir) } + sources += sourceSet.resources.srcDirs.findAll { dir -> dir.exists() }.collect { dir -> relativize(dir) } + } + + // This is hacky - we take the resolved compile classpath and just + // include JAR files from there. We should probably make it smarter + // by looking at real dependencies. But then: this Eclipse configuration + // doesn't really separate sources anyway so why bother. + jars += prj.configurations.compileClasspath.resolve() + jars += prj.configurations.testCompileClasspath.resolve() + } - classpath.entries += sources.sort().collect {name -> new SourceFolder(name, "build/eclipse/" + name) } - classpath.entries += jars.unique().findAll { location -> location.isFile() }.collect { location -> - new LibEntry(location.toString()) + classpath.entries += sources.sort().collect {name -> new SourceFolder(name, "build/eclipse/" + name) } + classpath.entries += jars.unique().findAll { location -> location.isFile() }.collect { location -> + new LibEntry(location.toString()) + } } } } } jdt { - sourceCompatibility = rootProject.minJavaVersionDefault - targetCompatibility = rootProject.minJavaVersionDefault - javaRuntimeName = "JavaSE-${rootProject.minJavaVersionDefault}" + sourceCompatibility = eclipseJavaVersion + targetCompatibility = eclipseJavaVersion + javaRuntimeName = "JavaSE-${eclipseJavaVersion}" } - } - eclipseJdt { - doLast { - project.sync { - from rootProject.file("${resources}/dot.settings") - into rootProject.file(".settings") + eclipseJdt { + doLast { + project.sync { + from rootProject.file("${resources}/dot.settings") + into rootProject.file(".settings") + } } } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml new file mode 100644 index 00000000000..dd7a5dfff0f --- /dev/null +++ b/gradle/libs.versions.toml @@ -0,0 +1,459 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +[versions] +adobe-testing-s3mock = "2.17.0" +amazon-awssdk = "2.26.19" +# @keep Antora version used in ref-guide +antora = "3.1.4" +# @keep Most recent commit as of 2022-06-24, this repo does not have tags +antora-default-ui = "51ad811622394027afb4e182c2fdabc235ae04dd" +# @keep Antora Lunr extensions version used in ref-guide +antora-lunr-extension = "1.0.0-alpha.8" +apache-calcite = "1.37.0" +apache-calcite-avatica = "1.25.0" +apache-commons-collections4 = "4.4" +apache-commons-compress = "1.26.1" +apache-commons-configuration2 = "2.11.0" +apache-commons-exec = "1.4.0" +apache-commons-lang3 = "3.15.0" +apache-commons-math3 = "3.6.1" +# @keep for version alignment +apache-commons-text = "1.12.0" +apache-curator = "5.7.1" +apache-hadoop = "3.4.0" +apache-hadoop-thirdparty = "1.2.0" +apache-httpcomponents-httpclient = "4.5.14" +apache-httpcomponents-httpcore = "4.4.16" +apache-httpcomponents-httpmime = "4.5.14" +apache-kafka = "3.7.1" +apache-log4j = "2.21.0" +apache-lucene = "9.11.1" +apache-opennlp = "1.9.4" +apache-poi = "5.2.2" +apache-rat = "0.15" +apache-tika = "1.28.5" +apache-tomcat = "6.0.53" +apache-zookeeper = "3.9.2" +# @keep for version alignment +apiguardian = "1.1.2" +aqute-bnd = "6.4.1" +# @keep Asciidoctor mathjax version used in ref-guide +asciidoctor-mathjax = "0.0.9" +# @keep Asciidoctor tabs version used in ref-guide +asciidoctor-tabs = "1.0.0-beta.6" +# @keep bats-assert (node) version used in packaging +bats-assert = "2.0.0" +# @keep bats-core (node) version used in packaging +bats-core = "1.8.2" +# @keep bats-file (node) version used in packaging +bats-file = "0.3.0" +bc-jose4j = "0.9.6" +benmanes-caffeine = "3.1.8" +benmanes-versions = "0.51.0" +bouncycastle = "1.78.1" +# @keep Browserify version used in ref-guide +browserify = "17.0.0" +carrot2-core = "4.5.1" +carrotsearch-dependencychecks = "0.0.9" +carrotsearch-hppc = "0.10.0" +carrotsearch-randomizedtesting = "2.8.1" +# @keep for version alignment +checkerframework = "3.44.0" +codehaus-woodstox = "4.2.2" +commons-cli = "1.9.0" +commons-codec = "1.17.1" +commons-collections = "3.2.2" +commons-io = "2.15.1" +cutterslade-analyze = "1.10.0" +cybozulabs-langdetect = "1.1-20120112" +diffplug-spotless = "6.5.2" +dropwizard-metrics = "4.2.26" +eclipse-ecj = "3.39.0" +eclipse-jetty = "10.0.22" +eclipse-jettytoolchain = "4.0.6" +# @keep jgit version used by git-status.gradle +eclipse-jgit = "6.7.0.202309050840-r" +fasterxml = "2.18.0" +fasterxml-woodstox = "7.0.0" +# @keep Flexmark used in classpath +flexmark = "0.64.8" +google-api-gax = "2.33.0" +# @keep for version alignment +google-api-grpc-proto = "2.41.0" +google-auth = "1.19.0" +# @keep for version alignment +google-autovalue = "1.10.4" +google-cloud-bom = "0.224.0" +google-cloud-core = "2.23.0" +google-cloud-nio = "0.127.3" +google-cloud-storage = "2.27.0" +google-errorprone = "2.31.0" +# @keep for version alignment +google-failureaccess = "1.0.2" +# @keep for version alignment +google-gson = "2.11.0" +google-guava = "33.1.0-jre" +# @keep for version alignment +google-j2objc = "3.0.0" +# @keep This is GJF version for spotless/ tidy. +google-javaformat = "1.18.1" +# @keep for version alignment +google-protobuf = "3.25.3" +google-re2j = "1.7" +# @keep Gradle version to run the build +gradle = "8.10" +grpc = "1.65.1" +# @keep Gulp version used in ref-guide +gulp-cli = "2.3.0" +hamcrest = "3.0" +hk2 = "3.1.1" +hsqldb = "2.7.2" +ibm-icu = "74.2" +immutables-valueannotations = "2.10.1" +j256-simplemagic = "1.17" +jakarta-annotation = "2.1.1" +jakarta-inject = "2.0.1" +jakarta-ws = "3.1.0" +# @keep This is the minimum required Java version for the project. +java-min = "21" +# @keep This is the minimum required Java version for SolrJ. +java-solrj = "17" +javacc = "7.0.12" +# @keep for version alignment +jaxb = "2.3.8" +jayway-jsonpath = "2.9.0" +jctools = "4.0.5" +jersey = "3.1.9" +# TODO Sync with jersey versions +jersey-containers = "2.39.1" +# @keep for version alignment +joda-time = "2.8.1" +junit = "4.13.2" +langchain4j = "0.35.0" +# @keep Link checker version used in ref-guide +link-checker = "1.4.2" +littlerobots-versioncatalogupdate = "0.8.4" +lmax-disruptor = "3.4.4" +ltgt-errorprone = "3.1.0" +mockito = "5.12.0" +morethan-jmhreport = "0.9.0" +navsecurity = "0.5.10" +netty = "4.1.114.Final" +# @keep for version alignment +netty-tcnative = "2.0.66.Final" +nimbusds-josejwt = "9.30.2" +nodegradle-node = "7.0.1" +# @keep Node JS version used in node.gradle (LTS) +nodejs = "16.20.2" +openapi = "7.6.0" +openjdk-jmh = "1.37" +opentelemetry = "1.40.0" +osgi-annotation = "8.1.0" +# @keep for version alignment +ow2-asm = "9.3" +owasp-dependencycheck = "9.0.8" +# @keep for version alignment +perfmark = "0.27.0" +prometheus-metrics = "1.1.0" +prometheus-simpleclient = "0.16.0" +quicktheories = "0.26" +semver4j = "5.3.0" +slf4j = "2.0.13" +spatial4j = "0.8" +spotbugs = "4.8.6" +squareup-okhttp3-mockwebserver = "4.11.0" +squareup-okhttp3-okhttp = "4.12.0" +stephenc-jcip = "1.0-1" +swagger3 = "2.2.22" +tdunning-tdigest = "3.3" +thetaphi-forbiddenapis = "3.7" +thisptr-jacksonjq = "0.0.13" +threeten-bp = "1.6.8" +undercouch-download = "5.5.0" +xerces = "2.12.2" +xerial-snappy = "1.1.10.5" + +[plugins] +benmanes-versions = { id = "com.github.ben-manes.versions", version.ref = "benmanes-versions" } +carrotsearch-dependencychecks = { id = "com.carrotsearch.gradle.dependencychecks", version.ref = "carrotsearch-dependencychecks" } +cutterslade-analyze = { id = "ca.cutterslade.analyze", version.ref = "cutterslade-analyze" } +diffplug-spotless = { id = "com.diffplug.spotless", version.ref = "diffplug-spotless" } +littlerobots-versioncatalogupdate = { id = "nl.littlerobots.version-catalog-update", version.ref = "littlerobots-versioncatalogupdate" } +ltgt-errorprone = { id = "net.ltgt.errorprone", version.ref = "ltgt-errorprone" } +morethan-jmhreport = { id = "io.morethan.jmhreport", version.ref = "morethan-jmhreport" } +nodegradle-node = { id = "com.github.node-gradle.node", version.ref = "nodegradle-node" } +openapi-generator = { id = "org.openapi.generator", version.ref = "openapi" } +owasp-dependencycheck = { id = "org.owasp.dependencycheck", version.ref = "owasp-dependencycheck" } +swagger3-core = { id = "io.swagger.core.v3.swagger-gradle-plugin", version.ref = "swagger3" } +thetaphi-forbiddenapis = { id = "de.thetaphi.forbiddenapis", version.ref = "thetaphi-forbiddenapis" } +undercouch-download = { id = "de.undercouch.download", version.ref = "undercouch-download" } + +[libraries] +adobe-testing-s3mock-junit4 = { module = "com.adobe.testing:s3mock-junit4", version.ref = "adobe-testing-s3mock" } +adobe-testing-s3mock-testsupportcommon = { module = "com.adobe.testing:s3mock-testsupport-common", version.ref = "adobe-testing-s3mock" } +amazon-awssdk-apacheclient = { module = "software.amazon.awssdk:apache-client", version.ref = "amazon-awssdk" } +amazon-awssdk-auth = { module = "software.amazon.awssdk:auth", version.ref = "amazon-awssdk" } +amazon-awssdk-awscore = { module = "software.amazon.awssdk:aws-core", version.ref = "amazon-awssdk" } +amazon-awssdk-bom = { module = "software.amazon.awssdk:bom", version.ref = "amazon-awssdk" } +amazon-awssdk-httpclient-spi = { module = "software.amazon.awssdk:http-client-spi", version.ref = "amazon-awssdk" } +amazon-awssdk-profiles = { module = "software.amazon.awssdk:profiles", version.ref = "amazon-awssdk" } +amazon-awssdk-regions = { module = "software.amazon.awssdk:regions", version.ref = "amazon-awssdk" } +amazon-awssdk-s3 = { module = "software.amazon.awssdk:s3", version.ref = "amazon-awssdk" } +amazon-awssdk-sdkcore = { module = "software.amazon.awssdk:sdk-core", version.ref = "amazon-awssdk" } +amazon-awssdk-sts = { module = "software.amazon.awssdk:sts", version.ref = "amazon-awssdk" } +apache-calcite-avatica-core = { module = "org.apache.calcite.avatica:avatica-core", version.ref = "apache-calcite-avatica" } +apache-calcite-core = { module = "org.apache.calcite:calcite-core", version.ref = "apache-calcite" } +apache-calcite-linq4j = { module = "org.apache.calcite:calcite-linq4j", version.ref = "apache-calcite" } +apache-commons-collections4 = { module = "org.apache.commons:commons-collections4", version.ref = "apache-commons-collections4" } +apache-commons-compress = { module = "org.apache.commons:commons-compress", version.ref = "apache-commons-compress" } +apache-commons-configuration2 = { module = "org.apache.commons:commons-configuration2", version.ref = "apache-commons-configuration2" } +apache-commons-exec = { module = "org.apache.commons:commons-exec", version.ref = "apache-commons-exec" } +apache-commons-lang3 = { module = "org.apache.commons:commons-lang3", version.ref = "apache-commons-lang3" } +apache-commons-math3 = { module = "org.apache.commons:commons-math3", version.ref = "apache-commons-math3" } +# @keep transitive dependency for version alignment +apache-commons-text = { module = "org.apache.commons:commons-text", version.ref = "apache-commons-text" } +apache-curator-client = { module = "org.apache.curator:curator-client", version.ref = "apache-curator" } +apache-curator-framework = { module = "org.apache.curator:curator-framework", version.ref = "apache-curator" } +apache-curator-recipes = { module = "org.apache.curator:curator-recipes", version.ref = "apache-curator" } +apache-curator-test = { module = "org.apache.curator:curator-test", version.ref = "apache-curator" } +apache-hadoop-client-api = { module = "org.apache.hadoop:hadoop-client-api", version.ref = "apache-hadoop" } +apache-hadoop-client-minicluster = { module = "org.apache.hadoop:hadoop-client-minicluster", version.ref = "apache-hadoop" } +apache-hadoop-client-runtime = { module = "org.apache.hadoop:hadoop-client-runtime", version.ref = "apache-hadoop" } +apache-hadoop-hdfs = { module = "org.apache.hadoop:hadoop-hdfs", version.ref = "apache-hadoop" } +apache-hadoop-thirdparty-shadedguava = { module = "org.apache.hadoop.thirdparty:hadoop-shaded-guava", version.ref = "apache-hadoop-thirdparty" } +apache-httpcomponents-httpclient = { module = "org.apache.httpcomponents:httpclient", version.ref = "apache-httpcomponents-httpclient" } +apache-httpcomponents-httpcore = { module = "org.apache.httpcomponents:httpcore", version.ref = "apache-httpcomponents-httpcore" } +apache-httpcomponents-httpmime = { module = "org.apache.httpcomponents:httpmime", version.ref = "apache-httpcomponents-httpmime" } +apache-kafka-clients = { module = "org.apache.kafka:kafka-clients", version.ref = "apache-kafka" } +apache-kafka-kafka213 = { module = "org.apache.kafka:kafka_2.13", version.ref = "apache-kafka" } +apache-kafka-server-common = { module = "org.apache.kafka:kafka-server-common", version.ref = "apache-kafka" } +apache-kafka-streams = { module = "org.apache.kafka:kafka-streams", version.ref = "apache-kafka" } +apache-log4j-api = { module = "org.apache.logging.log4j:log4j-api", version.ref = "apache-log4j" } +apache-log4j-core = { module = "org.apache.logging.log4j:log4j-core", version.ref = "apache-log4j" } +apache-log4j-jul = { module = "org.apache.logging.log4j:log4j-jul", version.ref = "apache-log4j" } +apache-log4j-layout-templatejson = { module = "org.apache.logging.log4j:log4j-layout-template-json", version.ref = "apache-log4j" } +apache-log4j-slf4j2impl = { module = "org.apache.logging.log4j:log4j-slf4j2-impl", version.ref = "apache-log4j" } +apache-log4j-web = { module = "org.apache.logging.log4j:log4j-web", version.ref = "apache-log4j" } +apache-log4j1-api = { module = "org.apache.logging.log4j:log4j-1.2-api", version.ref = "apache-log4j" } +apache-lucene-analysis-common = { module = "org.apache.lucene:lucene-analysis-common", version.ref = "apache-lucene" } +apache-lucene-analysis-icu = { module = "org.apache.lucene:lucene-analysis-icu", version.ref = "apache-lucene" } +apache-lucene-analysis-kuromoji = { module = "org.apache.lucene:lucene-analysis-kuromoji", version.ref = "apache-lucene" } +apache-lucene-analysis-morfologik = { module = "org.apache.lucene:lucene-analysis-morfologik", version.ref = "apache-lucene" } +apache-lucene-analysis-nori = { module = "org.apache.lucene:lucene-analysis-nori", version.ref = "apache-lucene" } +apache-lucene-analysis-opennlp = { module = "org.apache.lucene:lucene-analysis-opennlp", version.ref = "apache-lucene" } +apache-lucene-analysis-phonetic = { module = "org.apache.lucene:lucene-analysis-phonetic", version.ref = "apache-lucene" } +apache-lucene-analysis-smartcn = { module = "org.apache.lucene:lucene-analysis-smartcn", version.ref = "apache-lucene" } +apache-lucene-analysis-stempel = { module = "org.apache.lucene:lucene-analysis-stempel", version.ref = "apache-lucene" } +apache-lucene-backward-codecs = { module = "org.apache.lucene:lucene-backward-codecs", version.ref = "apache-lucene" } +apache-lucene-classification = { module = "org.apache.lucene:lucene-classification", version.ref = "apache-lucene" } +apache-lucene-codecs = { module = "org.apache.lucene:lucene-codecs", version.ref = "apache-lucene" } +apache-lucene-core = { module = "org.apache.lucene:lucene-core", version.ref = "apache-lucene" } +apache-lucene-expressions = { module = "org.apache.lucene:lucene-expressions", version.ref = "apache-lucene" } +apache-lucene-grouping = { module = "org.apache.lucene:lucene-grouping", version.ref = "apache-lucene" } +apache-lucene-highlighter = { module = "org.apache.lucene:lucene-highlighter", version.ref = "apache-lucene" } +apache-lucene-join = { module = "org.apache.lucene:lucene-join", version.ref = "apache-lucene" } +apache-lucene-misc = { module = "org.apache.lucene:lucene-misc", version.ref = "apache-lucene" } +apache-lucene-queries = { module = "org.apache.lucene:lucene-queries", version.ref = "apache-lucene" } +apache-lucene-queryparser = { module = "org.apache.lucene:lucene-queryparser", version.ref = "apache-lucene" } +apache-lucene-spatialextras = { module = "org.apache.lucene:lucene-spatial-extras", version.ref = "apache-lucene" } +apache-lucene-suggest = { module = "org.apache.lucene:lucene-suggest", version.ref = "apache-lucene" } +apache-lucene-testframework = { module = "org.apache.lucene:lucene-test-framework", version.ref = "apache-lucene" } +apache-opennlp-tools = { module = "org.apache.opennlp:opennlp-tools", version.ref = "apache-opennlp" } +apache-poi-ooxml = { module = "org.apache.poi:poi-ooxml", version.ref = "apache-poi" } +apache-poi-poi = { module = "org.apache.poi:poi", version.ref = "apache-poi" } +apache-rat-rat = { module = "org.apache.rat:apache-rat", version.ref = "apache-rat" } +apache-tika-core = { module = "org.apache.tika:tika-core", version.ref = "apache-tika" } +apache-tika-parsers = { module = "org.apache.tika:tika-parsers", version.ref = "apache-tika" } +apache-tomcat-annotationsapi = { module = "org.apache.tomcat:annotations-api", version.ref = "apache-tomcat" } +apache-zookeeper-jute = { module = "org.apache.zookeeper:zookeeper-jute", version.ref = "apache-zookeeper" } +apache-zookeeper-zookeeper = { module = "org.apache.zookeeper:zookeeper", version.ref = "apache-zookeeper" } +# @keep transitive dependency for version alignment +apiguardian-api = { module = "org.apiguardian:apiguardian-api", version.ref = "apiguardian" } +aqute-bnd-annotation = { module = "biz.aQute.bnd:biz.aQute.bnd.annotation", version.ref = "aqute-bnd" } +bc-jose4j = { module = "org.bitbucket.b_c:jose4j", version.ref = "bc-jose4j" } +benmanes-caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version.ref = "benmanes-caffeine" } +bouncycastle-bcpkix = { module = "org.bouncycastle:bcpkix-jdk18on", version.ref = "bouncycastle" } +bouncycastle-bcprov = { module = "org.bouncycastle:bcprov-jdk18on", version.ref = "bouncycastle" } +carrot2-core = { module = "org.carrot2:carrot2-core", version.ref = "carrot2-core" } +carrotsearch-hppc = { module = "com.carrotsearch:hppc", version.ref = "carrotsearch-hppc" } +carrotsearch-randomizedtesting-runner = { module = "com.carrotsearch.randomizedtesting:randomizedtesting-runner", version.ref = "carrotsearch-randomizedtesting" } +# @keep transitive dependency for version alignment +checkerframework-qual = { module = "org.checkerframework:checker-qual", version.ref = "checkerframework" } +codehaus-woodstox-stax2api = { module = "org.codehaus.woodstox:stax2-api", version.ref = "codehaus-woodstox" } +commonscli-commonscli = { module = "commons-cli:commons-cli", version.ref = "commons-cli" } +commonscodec-commonscodec = { module = "commons-codec:commons-codec", version.ref = "commons-codec" } +commonscollections-commonscollections = { module = "commons-collections:commons-collections", version.ref = "commons-collections" } +commonsio-commonsio = { module = "commons-io:commons-io", version.ref = "commons-io" } +cybozulabs-langdetect = { module = "com.cybozu.labs:langdetect", version.ref = "cybozulabs-langdetect" } +dropwizard-metrics-core = { module = "io.dropwizard.metrics:metrics-core", version.ref = "dropwizard-metrics" } +dropwizard-metrics-graphite = { module = "io.dropwizard.metrics:metrics-graphite", version.ref = "dropwizard-metrics" } +dropwizard-metrics-jetty10 = { module = "io.dropwizard.metrics:metrics-jetty10", version.ref = "dropwizard-metrics" } +dropwizard-metrics-jmx = { module = "io.dropwizard.metrics:metrics-jmx", version.ref = "dropwizard-metrics" } +dropwizard-metrics-jvm = { module = "io.dropwizard.metrics:metrics-jvm", version.ref = "dropwizard-metrics" } +dropwizard-metrics-servlets = { module = "io.dropwizard.metrics:metrics-servlets", version.ref = "dropwizard-metrics" } +eclipse-jdt-ecj = { module = "org.eclipse.jdt:ecj", version.ref = "eclipse-ecj" } +eclipse-jetty-alpnjavaclient = { module = "org.eclipse.jetty:jetty-alpn-java-client", version.ref = "eclipse-jetty" } +eclipse-jetty-alpnjavaserver = { module = "org.eclipse.jetty:jetty-alpn-java-server", version.ref = "eclipse-jetty" } +eclipse-jetty-alpnserver = { module = "org.eclipse.jetty:jetty-alpn-server", version.ref = "eclipse-jetty" } +eclipse-jetty-client = { module = "org.eclipse.jetty:jetty-client", version.ref = "eclipse-jetty" } +eclipse-jetty-deploy = { module = "org.eclipse.jetty:jetty-deploy", version.ref = "eclipse-jetty" } +eclipse-jetty-http = { module = "org.eclipse.jetty:jetty-http", version.ref = "eclipse-jetty" } +eclipse-jetty-http2-client = { module = "org.eclipse.jetty.http2:http2-client", version.ref = "eclipse-jetty" } +eclipse-jetty-http2-common = { module = "org.eclipse.jetty.http2:http2-common", version.ref = "eclipse-jetty" } +eclipse-jetty-http2-hpack = { module = "org.eclipse.jetty.http2:http2-hpack", version.ref = "eclipse-jetty" } +eclipse-jetty-http2-httpclienttransport = { module = "org.eclipse.jetty.http2:http2-http-client-transport", version.ref = "eclipse-jetty" } +eclipse-jetty-http2-server = { module = "org.eclipse.jetty.http2:http2-server", version.ref = "eclipse-jetty" } +eclipse-jetty-io = { module = "org.eclipse.jetty:jetty-io", version.ref = "eclipse-jetty" } +eclipse-jetty-jmx = { module = "org.eclipse.jetty:jetty-jmx", version.ref = "eclipse-jetty" } +eclipse-jetty-rewrite = { module = "org.eclipse.jetty:jetty-rewrite", version.ref = "eclipse-jetty" } +eclipse-jetty-security = { module = "org.eclipse.jetty:jetty-security", version.ref = "eclipse-jetty" } +eclipse-jetty-server = { module = "org.eclipse.jetty:jetty-server", version.ref = "eclipse-jetty" } +eclipse-jetty-servlet = { module = "org.eclipse.jetty:jetty-servlet", version.ref = "eclipse-jetty" } +eclipse-jetty-servlets = { module = "org.eclipse.jetty:jetty-servlets", version.ref = "eclipse-jetty" } +eclipse-jetty-start = { module = "org.eclipse.jetty:jetty-start", version.ref = "eclipse-jetty" } +eclipse-jetty-toolchain-servletapi = { module = "org.eclipse.jetty.toolchain:jetty-servlet-api", version.ref = "eclipse-jettytoolchain" } +eclipse-jetty-util = { module = "org.eclipse.jetty:jetty-util", version.ref = "eclipse-jetty" } +eclipse-jetty-webapp = { module = "org.eclipse.jetty:jetty-webapp", version.ref = "eclipse-jetty" } +eclipse-jetty-xml = { module = "org.eclipse.jetty:jetty-xml", version.ref = "eclipse-jetty" } +eclipse-jgit-jgit = { module = "org.eclipse.jgit:org.eclipse.jgit", version.ref = "eclipse-jgit" } +fasterxml-jackson-bom = { module = "com.fasterxml.jackson:jackson-bom", version.ref = "fasterxml" } +fasterxml-jackson-core-annotations = { module = "com.fasterxml.jackson.core:jackson-annotations", version.ref = "fasterxml" } +fasterxml-jackson-core-core = { module = "com.fasterxml.jackson.core:jackson-core", version.ref = "fasterxml" } +fasterxml-jackson-core-databind = { module = "com.fasterxml.jackson.core:jackson-databind", version.ref = "fasterxml" } +fasterxml-jackson-dataformat-cbor = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor", version.ref = "fasterxml" } +fasterxml-jackson-dataformat-smile = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-smile", version.ref = "fasterxml" } +fasterxml-woodstox-core = { module = "com.fasterxml.woodstox:woodstox-core", version.ref = "fasterxml-woodstox" } +flexmark-extensions-abbreviation = { module = "com.vladsch.flexmark:flexmark-ext-abbreviation", version.ref = "flexmark" } +flexmark-extensions-attributes = { module = "com.vladsch.flexmark:flexmark-ext-attributes", version.ref = "flexmark" } +flexmark-extensions-autolink = { module = "com.vladsch.flexmark:flexmark-ext-autolink", version.ref = "flexmark" } +flexmark-flexmark = { module = "com.vladsch.flexmark:flexmark", version.ref = "flexmark" } +google-api-gax = { module = "com.google.api:gax", version.ref = "google-api-gax" } +# @keep transitive dependency for version alignment +google-api-grpc-proto = { module = "com.google.api.grpc:proto-google-common-protos", version.ref = "google-api-grpc-proto" } +google-auth-credentials = { module = "com.google.auth:google-auth-library-credentials", version.ref = "google-auth" } +google-auth-oauth2http = { module = "com.google.auth:google-auth-library-oauth2-http", version.ref = "google-auth" } +# @keep transitive dependency for version alignment +google-autovalue-annotations = { module = "com.google.auto.value:auto-value-annotations", version.ref = "google-autovalue" } +google-cloud-bom = { module = "com.google.cloud:google-cloud-bom", version.ref = "google-cloud-bom" } +google-cloud-core = { module = "com.google.cloud:google-cloud-core", version.ref = "google-cloud-core" } +google-cloud-corehttp = { module = "com.google.cloud:google-cloud-core-http", version.ref = "google-cloud-core" } +google-cloud-nio = { module = "com.google.cloud:google-cloud-nio", version.ref = "google-cloud-nio" } +google-cloud-storage = { module = "com.google.cloud:google-cloud-storage", version.ref = "google-cloud-storage" } +# @keep transitive dependency for version alignment +google-errorprone-annotations = { module = "com.google.errorprone:error_prone_annotations", version.ref = "google-errorprone" } +google-errorprone-core = { module = "com.google.errorprone:error_prone_core", version.ref = "google-errorprone" } +# @keep transitive dependency for version alignment +google-gson = { module = "com.google.code.gson:gson", version.ref = "google-gson" } +google-guava = { module = "com.google.guava:guava", version.ref = "google-guava" } +# @keep transitive dependency for version alignment +google-j2objc-annotations = { module = "com.google.j2objc:j2objc-annotations", version.ref = "google-j2objc" } +# @keep transitive dependency for version alignment +google-protobuf-java = { module = "com.google.protobuf:protobuf-java", version.ref = "google-protobuf" } +google-protobuf-javautils = { module = "com.google.protobuf:protobuf-java-util", version.ref = "google-protobuf" } +google-re2j = { module = "com.google.re2j:re2j", version.ref = "google-re2j" } +# @keep transitive dependency for version alignment +grpc-api = { module = "io.grpc:grpc-api", version.ref = "grpc" } +# @keep transitive dependency for version alignment +grpc-bom = { module = "io.grpc:grpc-bom", version.ref = "grpc" } +grpc-context = { module = "io.grpc:grpc-context", version.ref = "grpc" } +# @keep transitive dependency for version alignment +grpc-core = { module = "io.grpc:grpc-core", version.ref = "grpc" } +grpc-netty = { module = "io.grpc:grpc-netty", version.ref = "grpc" } +grpc-protobuf = { module = "io.grpc:grpc-protobuf", version.ref = "grpc" } +# @keep transitive dependency for version alignment +grpc-protobuf-lite = { module = "io.grpc:grpc-protobuf-lite", version.ref = "grpc" } +grpc-stub = { module = "io.grpc:grpc-stub", version.ref = "grpc" } +# @keep transitive dependency for version alignment +grpc-util = { module = "io.grpc:grpc-util", version.ref = "grpc" } +hamcrest-hamcrest = { module = "org.hamcrest:hamcrest", version.ref = "hamcrest" } +hk2-api = { module = "org.glassfish.hk2:hk2-api", version.ref = "hk2" } +# @keep transitive dependency for version alignment +hk2-locator = { module = "org.glassfish.hk2:hk2-locator", version.ref = "hk2" } +hsqldb-hsqldb = { module = "org.hsqldb:hsqldb", version.ref = "hsqldb" } +ibm-icu-icu4j = { module = "com.ibm.icu:icu4j", version.ref = "ibm-icu" } +immutables-valueannotations = { module = "org.immutables:value-annotations", version.ref = "immutables-valueannotations" } +j256-simplemagic = { module = "com.j256.simplemagic:simplemagic", version.ref = "j256-simplemagic" } +jakarta-annotation-api = { module = "jakarta.annotation:jakarta.annotation-api", version.ref = "jakarta-annotation" } +jakarta-inject-api = { module = "jakarta.inject:jakarta.inject-api", version.ref = "jakarta-inject" } +jakarta-ws-rsapi = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version.ref = "jakarta-ws" } +javacc-javacc = { module = "net.java.dev.javacc:javacc", version.ref = "javacc" } +# @keep transitive dependency for version alignment +jaxb-runtime = { module = "org.glassfish.jaxb:jaxb-runtime", version.ref = "jaxb" } +jayway-jsonpath = { module = "com.jayway.jsonpath:json-path", version.ref = "jayway-jsonpath" } +jctools-core = { module = "org.jctools:jctools-core", version.ref = "jctools" } +jersey-containers-jettyhttp = { module = "org.glassfish.jersey.containers:jersey-container-jetty-http", version.ref = "jersey-containers" } +jersey-core-common = { module = "org.glassfish.jersey.core:jersey-common", version.ref = "jersey" } +jersey-core-server = { module = "org.glassfish.jersey.core:jersey-server", version.ref = "jersey" } +jersey-inject-hk2 = { module = "org.glassfish.jersey.inject:jersey-hk2", version.ref = "jersey" } +jersey-media-jsonjackson = { module = "org.glassfish.jersey.media:jersey-media-json-jackson", version.ref = "jersey" } +# @keep transitive dependency for version alignment +jodatime-jodatime = { module = "joda-time:joda-time", version.ref = "joda-time" } +junit-junit = { module = "junit:junit", version.ref = "junit" } +langchain4j-cohere = { module = "dev.langchain4j:langchain4j-cohere", version.ref = "langchain4j" } +langchain4j-core = { module = "dev.langchain4j:langchain4j-core", version.ref = "langchain4j" } +langchain4j-hugging-face = { module = "dev.langchain4j:langchain4j-hugging-face", version.ref = "langchain4j" } +langchain4j-mistral-ai = { module = "dev.langchain4j:langchain4j-mistral-ai", version.ref = "langchain4j" } +langchain4j-open-ai = { module = "dev.langchain4j:langchain4j-open-ai", version.ref = "langchain4j" } +lmax-disruptor = { module = "com.lmax:disruptor", version.ref = "lmax-disruptor" } +locationtech-spatial4j = { module = "org.locationtech.spatial4j:spatial4j", version.ref = "spatial4j" } +mockito-core = { module = "org.mockito:mockito-core", version.ref = "mockito" } +mockito-subclass = { module = "org.mockito:mockito-subclass", version.ref = "mockito" } +navsecurity-mockoauth2server = { module = "no.nav.security:mock-oauth2-server", version.ref = "navsecurity" } +netty-bom = { module = "io.netty:netty-bom", version.ref = "netty" } +netty-codechttp = { module = "io.netty:netty-codec-http", version.ref = "netty" } +# @keep transitive dependency for version alignment +netty-handler = { module = "io.netty:netty-handler", version.ref = "netty" } +# @keep transitive dependency for version alignment +netty-tcnative-boringssl = { module = "io.netty:netty-tcnative-boringssl-static", version.ref = "netty-tcnative" } +netty-tcnative-classes = { module = "io.netty:netty-tcnative-classes", version.ref = "netty-tcnative" } +# @keep transitive dependency for version alignment +netty-transport-classes-epoll = { module = "io.netty:netty-transport-classes-epoll", version.ref = "netty" } +netty-transport-native-epoll = { module = "io.netty:netty-transport-native-epoll", version.ref = "netty" } +nimbusds-josejwt = { module = "com.nimbusds:nimbus-jose-jwt", version.ref = "nimbusds-josejwt" } +openjdk-jmh-core = { module = "org.openjdk.jmh:jmh-core", version.ref = "openjdk-jmh" } +openjdk-jmh-generatorannprocess = { module = "org.openjdk.jmh:jmh-generator-annprocess", version.ref = "openjdk-jmh" } +opentelemetry-api = { module = "io.opentelemetry:opentelemetry-api", version.ref = "opentelemetry" } +opentelemetry-bom = { module = "io.opentelemetry:opentelemetry-bom", version.ref = "opentelemetry" } +opentelemetry-context = { module = "io.opentelemetry:opentelemetry-context", version.ref = "opentelemetry" } +opentelemetry-exporter-otlp = { module = "io.opentelemetry:opentelemetry-exporter-otlp", version.ref = "opentelemetry" } +opentelemetry-sdk = { module = "io.opentelemetry:opentelemetry-sdk", version.ref = "opentelemetry" } +opentelemetry-sdkextension-autoconfigure = { module = "io.opentelemetry:opentelemetry-sdk-extension-autoconfigure", version.ref = "opentelemetry" } +opentelemetry-sdktesting = { module = "io.opentelemetry:opentelemetry-sdk-testing", version.ref = "opentelemetry" } +opentelemetry-sdktrace = { module = "io.opentelemetry:opentelemetry-sdk-trace", version.ref = "opentelemetry" } +osgi-annotation = { module = "org.osgi:osgi.annotation", version.ref = "osgi-annotation" } +# @keep transitive dependency for version alignment +ow2-asm = { module = "org.ow2.asm:asm", version.ref = "ow2-asm" } +# @keep transitive dependency for version alignment +perfmark-api = { module = "io.perfmark:perfmark-api", version.ref = "perfmark" } +prometheus-metrics-expositionformats = { module = "io.prometheus:prometheus-metrics-exposition-formats", version.ref = "prometheus-metrics" } +prometheus-metrics-model = { module = "io.prometheus:prometheus-metrics-model", version.ref = "prometheus-metrics" } +prometheus-simpleclient = { module = "io.prometheus:simpleclient", version.ref = "prometheus-simpleclient" } +prometheus-simpleclient-httpserver = { module = "io.prometheus:simpleclient_httpserver", version.ref = "prometheus-simpleclient" } +quicktheories-quicktheories = { module = "org.quicktheories:quicktheories", version.ref = "quicktheories" } +semver4j-semver4j = { module = "org.semver4j:semver4j", version.ref = "semver4j" } +slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" } +slf4j-jcloverslf4j = { module = "org.slf4j:jcl-over-slf4j", version.ref = "slf4j" } +slf4j-jultoslf4j = { module = "org.slf4j:jul-to-slf4j", version.ref = "slf4j" } +spotbugs-annotations = { module = "com.github.spotbugs:spotbugs-annotations", version.ref = "spotbugs" } +squareup-okhttp3-mockwebserver = { module = "com.squareup.okhttp3:mockwebserver", version.ref = "squareup-okhttp3-mockwebserver" } +squareup-okhttp3-okhttp = { module = "com.squareup.okhttp3:okhttp", version.ref = "squareup-okhttp3-okhttp" } +stephenc-jcip-annotations = { module = "com.github.stephenc.jcip:jcip-annotations", version.ref = "stephenc-jcip" } +swagger3-annotations-jakarta = { module = "io.swagger.core.v3:swagger-annotations-jakarta", version.ref = "swagger3" } +swagger3-jaxrs2-jakarta = { module = "io.swagger.core.v3:swagger-jaxrs2-jakarta", version.ref = "swagger3" } +tdunning-tdigest = { module = "com.tdunning:t-digest", version.ref = "tdunning-tdigest" } +thisptr-jacksonjq = { module = "net.thisptr:jackson-jq", version.ref = "thisptr-jacksonjq" } +threeten-bp = { module = "org.threeten:threetenbp", version.ref = "threeten-bp" } +xerces-impl = { module = "xerces:xercesImpl", version.ref = "xerces" } +xerial-snappy-java = { module = "org.xerial.snappy:snappy-java", version.ref = "xerial-snappy" } diff --git a/gradle/lucene-dev/lucene-dev-repo-composite.gradle b/gradle/lucene-dev/lucene-dev-repo-composite.gradle index d612b29fe70..62274b32e6c 100644 --- a/gradle/lucene-dev/lucene-dev-repo-composite.gradle +++ b/gradle/lucene-dev/lucene-dev-repo-composite.gradle @@ -104,7 +104,7 @@ if (luceneDevRepo != null) { // We substitute the exact version of Lucene we currently have in versions.props across all the dependencies. // We can't just substitute all references without looking at the versoin because // plugin dependencies then also get substituted and everything crashes. - String luceneVersion = (file("${rootDir}/versions.props").getText("UTF-8") =~ /org.apache.lucene:\*=(.+)/)[0][1] + String luceneVersion = libs.versions.apache.lucene.get() logger.lifecycle("Local Lucene development repository will be used substituting ${luceneVersion}: ${luceneDevRepo}") // Include Lucene repository as a composite and substitute module names. diff --git a/gradle/maven/defaults-maven.gradle b/gradle/maven/defaults-maven.gradle index 49a28dfa8d6..3239c04acfc 100644 --- a/gradle/maven/defaults-maven.gradle +++ b/gradle/maven/defaults-maven.gradle @@ -151,17 +151,6 @@ configure(subprojects.findAll { it.path in rootProject.published }) { prj -> artifact javadocJar pom(configurePom) - - pom({ - // LUCENE-9561: - // Remove dependencyManagement section created by a combination of - // Palantir and the publishing plugin. - // - // https://github.com/palantir/gradle-consistent-versions/issues/550 - withXml { - asNode().dependencyManagement.replaceNode {} - } - }) } } } diff --git a/gradle/node.gradle b/gradle/node.gradle index ff8404c2db2..d585ab5f871 100644 --- a/gradle/node.gradle +++ b/gradle/node.gradle @@ -16,7 +16,7 @@ */ configure([project(":solr:packaging"), project(":solr:solr-ref-guide"), project(":solr:webapp")]) { - apply plugin: "com.github.node-gradle.node" + apply plugin: libs.plugins.nodegradle.node.get().pluginId def npmRegistry = "${ -> propertyOrEnvOrDefault("solr.npm.registry", "SOLR_NPM_REGISTRY", '') }" if (!npmRegistry.isEmpty()) { @@ -41,7 +41,7 @@ configure([project(":solr:packaging"), project(":solr:solr-ref-guide"), project( node { download = true - version = "16.20.2" // LTS + version = libs.versions.nodejs.get() def nodeDistUrl = "${ -> propertyOrEnvOrDefault("solr.node.distUrl", "SOLR_NODE_DIST_URL", '') }" if (!nodeDistUrl.isEmpty()) { diff --git a/gradle/testing/alternative-jdk-support.gradle b/gradle/testing/alternative-jdk-support.gradle index a1ff1b4b014..97e5311ee09 100644 --- a/gradle/testing/alternative-jdk-support.gradle +++ b/gradle/testing/alternative-jdk-support.gradle @@ -50,7 +50,7 @@ if (jvmGradle != jvmCurrent) { doFirst { def jvmInfo = { JavaInfo javaInfo -> - JvmInstallationMetadata jvmMetadata = jvmDetector.getMetadata(new InstallationLocation(javaInfo.javaHome, "specific path")) + JvmInstallationMetadata jvmMetadata = jvmDetector.getMetadata(InstallationLocation.userDefined(javaInfo.javaHome, "specific path")) return "${jvmMetadata.languageVersion} (${jvmMetadata.displayName} ${jvmMetadata.runtimeVersion}, home at: ${jvmMetadata.javaHome})" } diff --git a/gradle/testing/randomization.gradle b/gradle/testing/randomization.gradle index 183943a39e7..d3ae962c144 100644 --- a/gradle/testing/randomization.gradle +++ b/gradle/testing/randomization.gradle @@ -30,7 +30,7 @@ buildscript { } dependencies { - classpath 'com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.7.9' + classpath libs.carrotsearch.randomizedtesting.runner } } diff --git a/gradle/testing/randomization/policies/solr-tests.policy b/gradle/testing/randomization/policies/solr-tests.policy index 61df0871a35..4d61f7985c0 100644 --- a/gradle/testing/randomization/policies/solr-tests.policy +++ b/gradle/testing/randomization/policies/solr-tests.policy @@ -58,6 +58,9 @@ grant { // needed by randomizedtesting runner to identify test methods. permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; + permission java.lang.reflect.ReflectPermission "newProxyInPackage.dev.langchain4j.model.cohere"; + permission java.lang.reflect.ReflectPermission "newProxyInPackage.dev.ai4j.openai4j"; + permission java.lang.reflect.ReflectPermission "newProxyInPackage.dev.langchain4j.model.huggingface"; permission java.lang.RuntimePermission "accessDeclaredMembers"; // needed by certain tests to redirect sysout/syserr: permission java.lang.RuntimePermission "setIO"; @@ -100,7 +103,7 @@ grant { permission java.lang.RuntimePermission "closeClassLoader"; // needed by HttpSolrClient permission java.lang.RuntimePermission "getFileSystemAttributes"; - // needed by hadoop auth (TODO: there is a cleaner way to handle this) + // needed by hadoop hdfs (TODO: there is a cleaner way to handle this) permission java.lang.RuntimePermission "loadLibrary.jaas"; permission java.lang.RuntimePermission "loadLibrary.jaas_unix"; permission java.lang.RuntimePermission "loadLibrary.jaas_nt"; @@ -135,17 +138,19 @@ grant { permission javax.management.MBeanServerPermission "findMBeanServer"; permission javax.management.MBeanServerPermission "releaseMBeanServer"; permission javax.management.MBeanTrustPermission "register"; - - // needed by hadoop auth + + // needed by hadoop hdfs permission javax.security.auth.AuthPermission "getSubject"; permission javax.security.auth.AuthPermission "modifyPrincipals"; permission javax.security.auth.AuthPermission "doAs"; - permission javax.security.auth.AuthPermission "getLoginConfiguration"; - permission javax.security.auth.AuthPermission "setLoginConfiguration"; permission javax.security.auth.AuthPermission "modifyPrivateCredentials"; permission javax.security.auth.AuthPermission "modifyPublicCredentials"; permission javax.security.auth.PrivateCredentialPermission "org.apache.hadoop.security.Credentials * \"*\"", "read"; + // needed by crossdc + permission javax.security.auth.AuthPermission "getLoginConfiguration"; + permission javax.security.auth.AuthPermission "setLoginConfiguration"; + // needed by hadoop security permission java.security.SecurityPermission "putProviderProperty.SaslPlainServer"; permission java.security.SecurityPermission "insertProvider"; diff --git a/gradle/validation/check-environment.gradle b/gradle/validation/check-environment.gradle index 6e5c01ef19c..f56e9fa4e78 100644 --- a/gradle/validation/check-environment.gradle +++ b/gradle/validation/check-environment.gradle @@ -22,7 +22,7 @@ import org.gradle.util.GradleVersion configure(rootProject) { ext { - expectedGradleVersion = '8.10' + expectedGradleVersion = libs.versions.gradle.get() } wrapper { @@ -31,7 +31,7 @@ configure(rootProject) { } def currentJavaVersion = JavaVersion.current() - def minJavaVersion = rootProject.sourceCompatibility// JavaVersion.VERSION_11 // TODO Write out java version + def minJavaVersion = JavaVersion.toVersion(libs.versions.java.min.get()) if (currentJavaVersion < minJavaVersion) { throw new GradleException("At least Java ${minJavaVersion} is required, you are running Java ${currentJavaVersion} " + "[${System.getProperty('java.vm.name')} ${System.getProperty('java.vm.version')}]") diff --git a/gradle/validation/dependencies.gradle b/gradle/validation/dependencies.gradle new file mode 100644 index 00000000000..cfb78ee15a2 --- /dev/null +++ b/gradle/validation/dependencies.gradle @@ -0,0 +1,346 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Helper function for specifying stable versions for dependency updates +// https://github.com/ben-manes/gradle-versions-plugin +def isNonStable = { String version -> + def stableKeyword = ['RELEASE', 'FINAL', 'GA'].any { it -> version.toUpperCase().contains(it) } + def regex = /^[0-9,.v-]+(-r)?$/ + return !stableKeyword && !(version ==~ regex) +} + +// Configure sanity check for conflicting dependencies across certain configurations +allprojects { + apply plugin: libs.plugins.carrotsearch.dependencychecks.get().pluginId + + def consolidatedConfigurations = project.configurations.matching { + it.name in [ + "annotationProcessor", + "compileClasspath", + "libExt", + "packaging", + "runtimeClasspath", + "runtimeLibs", + "server", + "serverLib", + "solrCore", + "solrFullTgz", + "solrPlatformLibs", + "solrSlimTgz", + "testCompileClasspath", + "testRuntimeClasspath", + ] + } + + dependencyVersionChecks { + lockFileComment = "An inventory of resolved dependency versions. Do not edit this file directly." + + configurationGroups { + // consolidated_dependencies is a configuration group that is used + // to check for conflicting versions of the included configurations + consolidated_dependencies { + include consolidatedConfigurations + } + } + } + + dependencies { + modules { + module("org.hamcrest:hamcrest-core") { + replacedBy("org.hamcrest:hamcrest", "hamcrest-core was renamed to hamcrest") + } + } + + constraints { handler -> + consolidatedConfigurations.configureEach { Configuration conf -> + // Add BOMs as they resolve many dependency conflicts + handler.add(conf.name, libs.amazon.awssdk.bom, { + because 'version alignment with known BOM for consistency across project' + }) + handler.add(conf.name, libs.google.cloud.bom, { + because 'version alignment with known BOM for consistency across project' + }) + handler.add(conf.name, libs.fasterxml.jackson.bom, { + because 'version alignment with known BOM for consistency across project' + }) + handler.add(conf.name, libs.opentelemetry.bom, { + because 'version alignment with known BOM for consistency across project' + }) + handler.add(conf.name, libs.grpc.bom, { + because 'version alignment with known BOM for consistency across project' + }) + handler.add(conf.name, libs.netty.bom, { + because 'version alignment with known BOM for consistency across project' + }) + + // Add known dependencies that have multiple versions as constraints + // to align versions + handler.add(conf.name, libs.google.guava, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.google.errorprone.annotations, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.apache.commons.exec, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.xerial.snappy.java, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.grpc.context, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.commonscli.commonscli, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.commonscodec.commonscodec, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.commonsio.commonsio, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.junit.junit, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.grpc.core, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.grpc.protobuf, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.jakarta.annotation.api, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.apache.commons.lang3, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.grpc.stub, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.eclipse.jetty.server, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.fasterxml.woodstox.core, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.slf4j.api, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.slf4j.jultoslf4j, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.apache.commons.compress, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.benmanes.caffeine, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.carrotsearch.hppc, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.apache.log4j.api, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.grpc.api, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.slf4j.jcloverslf4j, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.netty.codechttp, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.bc.jose4j, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.lmax.disruptor, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.apache.httpcomponents.httpclient, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.apache.httpcomponents.httpcore, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.apache.httpcomponents.httpmime, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.apache.zookeeper.zookeeper, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.apache.zookeeper.jute, { + because 'version alignment for consistency across project' + }) + handler.add(conf.name, libs.hamcrest.hamcrest, { + because 'version alignment for consistency across project' + }) + + // Add transitive dependencies as constraints to align versions + handler.add(conf.name, libs.checkerframework.qual, { + because "transitive version alignment for consistency across project" + + "\n- ${getFullName(libs.google.guava)} uses 3.42.0" + + "\n- ${getFullName(libs.benmanes.caffeine)} uses 3.37.0" + + "\n- ${getFullName(libs.google.cloud.storage)} uses 3.44.0" + }) + handler.add(conf.name, libs.ow2.asm, { + because "transitive version alignment for consistency across project" + + "\n- ${getFullName(libs.apache.lucene.expressions)} uses 7.2" + + "\n- ${getFullName(libs.apache.tika.parsers)} uses 9.3" + }) + handler.add(conf.name, libs.google.protobuf.java, { + because "transitive version alignment for consistency across project" + + "\n- ${getFullName(libs.google.errorprone.core)} uses 3.19.6" + + "\n- ${getFullName(libs.apache.tika.parsers)} uses 3.21.5" + + "\n- ${getFullName(libs.apache.calcite.avatica.core)} uses 3.21.9" + + "\n- ${getFullName(libs.google.cloud.storage)} uses 3.25.3" + + "\n- ${getFullName(libs.google.cloud.core)} uses 3.25.3" + }) + handler.add(conf.name, libs.google.gson, { + because "transitive version alignment for consistency across project" + + "\n- ${getFullName(libs.grpc.core)} uses 2.10.1" + + "\n- ${getFullName(libs.apache.tika.parsers)} uses 2.9.1" + + "\n- ${getFullName(libs.google.cloud.storage)} uses 2.11.0 and 2.10.1" + + "\n- ${getFullName(libs.google.protobuf.java)} uses 2.8.9" + + "\n- ${getFullName(libs.google.cloud.core)} uses 2.8.9 and 2.10.1" + + "\n- ${getFullName(libs.google.auth.oauth2http)} uses 2.10.1" + }) + handler.add(conf.name, libs.google.autovalue.annotations, { + because "transitive version alignment for consistency across project" + + "\n- ${getFullName(libs.google.cloud.storage)} uses 1.10.4" + + "\n- ${getFullName(libs.google.cloud.corehttp)} uses 1.10.4" + + "\n- ${getFullName(libs.google.cloud.core)} uses 1.10.4" + + "\n- ${getFullName(libs.google.api.gax)} uses 1.10.4" + + "\n- ${getFullName(libs.google.auth.oauth2http)} uses 1.10.4" + + "\n- ${getFullName(libs.google.cloud.bom)} uses 1.10.4" + + "\n- ${getFullName(libs.google.errorprone.core)} uses 1.9" + }) + handler.add(conf.name, libs.apache.commons.text, { + because "transitive version alignment for consistency across project" + + "\n- ${getFullName(libs.apache.calcite.core)} uses 1.11.0" + + "\n- ${getFullName(libs.apache.commons.configuration2)} uses 1.12.0" + }) + handler.add(conf.name, libs.grpc.util, { + because "transitive version alignment for consistency across project" + + "\n- ${getFullName(libs.google.cloud.storage)} uses 1.62.2" + + "\n- ${getFullName(libs.grpc.netty)} uses 1.65.1" + }) + handler.add(conf.name, libs.jodatime.jodatime, { + because "transitive version alignment for consistency across project" + + "\n- ${getFullName(libs.amazon.awssdk.sdkcore)} uses 2.8.1" + + "\n- ${getFullName(libs.apache.tika.parsers)} uses 2.2" + }) + handler.add(conf.name, libs.google.api.grpc.proto, { + because "transitive version alignment for consistency across project" + + "\n- ${getFullName(libs.grpc.protobuf.asProvider())} uses 2.29.0" + + "\n- ${getFullName(libs.google.api.gax)} uses 2.41.0" + + "\n- ${getFullName(libs.google.api.grpc.proto)} uses 2.41.0" + + "\n- ${getFullName(libs.google.cloud.core)} uses 2.41.0" + + "\n- ${getFullName(libs.google.cloud.storage)} uses 2.41.0" + }) + handler.add(conf.name, libs.netty.handler, { + because "transitive version alignment for consistency across project" + + "\n- ${getFullName(libs.apache.zookeeper.zookeeper)} uses 4.1.105.Final" + + "\n- ${getFullName(libs.netty.codechttp)} uses 4.1.112.Final" + }) + handler.add(conf.name, libs.grpc.protobuf.lite, { + because "transitive version alignment for consistency across project" + + "\n- ${getFullName(libs.google.cloud.storage)} uses 1.62.2" + + "\n- ${getFullName(libs.grpc.protobuf.asProvider())} uses 1.65.1" + }) + handler.add(conf.name, libs.jaxb.runtime, { + because "transitive version alignment for consistency across project" + + "\n- ${getFullName(libs.apache.tika.parsers)} uses 2.3.5" + + "\n- ${getFullName(libs.adobe.testing.s3mock.testsupportcommon)} uses 2.3.8" + }) + handler.add(conf.name, libs.perfmark.api, { + because "transitive version alignment for consistency across project" + + "\n- ${getFullName(libs.grpc.core)} uses 0.26.0" + + "\n- ${getFullName(libs.grpc.netty)} uses 0.26.0" + + "\n- ${getFullName(libs.google.cloud.storage)} uses 0.27.0" + }) + handler.add(conf.name, libs.netty.tcnative.boringssl, { + because "transitive version alignment for consistency across project" + + "\n- ${getFullName(libs.netty.bom)} uses 2.0.66.Final" + + "\n- ${getFullName(libs.apache.zookeeper.zookeeper)} uses 2.0.61.Final" + }) + handler.add(conf.name, libs.netty.transport.classes.epoll, { + because "transitive version alignment for consistency across project" + + "\n- ${getFullName(libs.netty.bom)} uses 4.1.114.Final" + + "\n- ${getFullName(libs.apache.zookeeper.zookeeper)} uses 4.1.105.Final" + }) + handler.add(conf.name, libs.netty.transport.native.epoll, { + because "transitive version alignment for consistency across project" + + "\n- ${getFullName(libs.netty.bom)} uses 4.1.114.Final" + + "\n- ${getFullName(libs.apache.zookeeper.zookeeper)} uses 4.1.105.Final" + }) + handler.add(conf.name, libs.google.j2objc.annotations, { + because "transitive version alignment for consistency across project" + + "\n- ${getFullName(libs.google.guava)} uses 3.0.0" + + "\n- ${getFullName(libs.google.protobuf.javautils)} uses 2.8" + }) + handler.add(conf.name, libs.apiguardian.api, { + because "transitive version alignment for consistency across project" + + "\n- ${getFullName(libs.apache.calcite.core)} uses 1.1.2" + + "\n- ${getFullName(libs.junit.junit)} (api) uses 1.1.0" + }) + handler.add(conf.name, libs.hk2.locator, { + because "transitive version alignment for consistency across project" + + "\n- ${getFullName(libs.jersey.inject.hk2)} uses 3.0.6" + + "\n- ${getFullName(libs.hk2.api)} uses 3.1.1" + }) + } + } + } +} + +static def getFullName(Provider dependency) { + def resolvedDep = dependency.get() + return "${resolvedDep.module.group}:${resolvedDep.module.name}" +} + +// Configure version catalog cleanups plugin. +configure(rootProject) { + apply plugin: libs.plugins.littlerobots.versioncatalogupdate.get().pluginId + + versionCatalogUpdate { + sortByKey = true + } + + tasks.matching { it.name == "tidy" }.configureEach { + it.dependsOn(":versionCatalogFormat") + } + + tasks.matching { + it.path in [":versionCatalogUpdate"] + }.configureEach { + it.interactive = true + } + + tasks.register("updateLibs", { + dependsOn ":versionCatalogUpdate" + }) + + // on dependencyUpdates get only stable versions recommended if current version is stable + // https://github.com/ben-manes/gradle-versions-plugin + tasks.named("dependencyUpdates").configure { + checkConstraints = true + checkBuildEnvironmentConstraints = true + rejectVersionIf { + isNonStable(it.candidate.version) && !isNonStable(it.currentVersion) + } + } +} diff --git a/gradle/validation/dependency-analyze.gradle b/gradle/validation/dependency-analyze.gradle index 1f35012ecf2..92125aba11c 100644 --- a/gradle/validation/dependency-analyze.gradle +++ b/gradle/validation/dependency-analyze.gradle @@ -20,7 +20,7 @@ allprojects { prj -> plugins.withId("java", { - prj.apply plugin: 'ca.cutterslade.analyze' + prj.apply plugin: libs.plugins.cutterslade.analyze.get().pluginId analyzeClassesDependencies { warnUsedUndeclared = false // means fail build if UsedUndeclared found diff --git a/gradle/validation/ecj-lint.gradle b/gradle/validation/ecj-lint.gradle index f47f70587a1..86f30cd5f1f 100644 --- a/gradle/validation/ecj-lint.gradle +++ b/gradle/validation/ecj-lint.gradle @@ -23,7 +23,7 @@ configure(rootProject) { } dependencies { - ecjDeps "org.eclipse.jdt:ecj:${scriptDepVersions['ecj']}" + ecjDeps libs.eclipse.jdt.ecj } } diff --git a/gradle/validation/error-prone.gradle b/gradle/validation/error-prone.gradle index 00e14ed0eab..b7242b566c0 100644 --- a/gradle/validation/error-prone.gradle +++ b/gradle/validation/error-prone.gradle @@ -37,24 +37,25 @@ if (skipReason) { allprojects { prj -> plugins.withType(JavaPlugin) { - // LUCENE-9650: Errorprone on master/gradle does not work when running as plugin - // inside a forked Javac process. Javac running inside Gradle works, because we have - // additional module system opens in place. - // This is a hack to keep the dependency (so that palantir's version check doesn't complain) - // but don't include the plugin (which fails on JDK16+). + // LUCENE-9650: Errorprone does not work when running as a plugin inside a forked Javac process. + // Javac running inside Gradle works, because we have additional module system opens in place. if (skipReason) { tasks.withType(JavaCompile) { task -> task.dependsOn ":errorProneSkipped" } + + // Error prone plugin adds error prone to test classpath. We need to add it here too + // (manually) so that versions.lock is consistent with or without error prone. configurations { errorprone } dependencies { - errorprone("com.google.errorprone:error_prone_core") + errorprone libs.google.errorprone.core } + configurations.annotationProcessor.extendsFrom(configurations.errorprone) } else { - prj.apply plugin: 'net.ltgt.errorprone' + prj.apply plugin: libs.plugins.ltgt.errorprone.get().pluginId dependencies { - errorprone("com.google.errorprone:error_prone_core") + errorprone libs.google.errorprone.core } tasks.withType(JavaCompile) { task -> @@ -179,6 +180,7 @@ allprojects { prj -> '-Xep:MathRoundIntLong:ERROR', // '-Xep:MislabeledAndroidString:OFF', // we don't use android '-Xep:MisplacedScopeAnnotations:ERROR', + // '-Xep:MissingRuntimeRetention:ERROR', // todo check if useful or comment why not // '-Xep:MissingSuperCall:OFF', // we don't use this annotation // '-Xep:MissingTestCall:OFF', // we don't use this annotation '-Xep:MisusedDayOfYear:ERROR', @@ -218,12 +220,15 @@ allprojects { prj -> '-Xep:RandomCast:ERROR', '-Xep:RandomModInteger:ERROR', // '-Xep:RectIntersectReturnValueIgnored:OFF', // we don't use android + // '-Xep:RedundantSetterCall:ERROR', // todo check if useful or comment why not // '-Xep:RequiredModifiers:OFF', // we don't use this annotation // '-Xep:RestrictedApiChecker:OFF', // we don't use this annotation // '-Xep:ReturnValueIgnored:OFF', // todo there are problems that should be fixed + // '-Xep:SelfAssertion:ERROR', // todo check if useful or comment why not '-Xep:SelfAssignment:ERROR', '-Xep:SelfComparison:ERROR', '-Xep:SelfEquals:ERROR', + // '-Xep:SetUnrecognized:ERROR', // todo check if useful or comment why not // '-Xep:ShouldHaveEvenArgs:OFF', // we don't use truth '-Xep:SizeGreaterThanOrEqualsZero:ERROR', '-Xep:StreamToString:ERROR', @@ -236,7 +241,6 @@ allprojects { prj -> // '-Xep:ThrowIfUncheckedKnownChecked:OFF', // we don't use this annotation '-Xep:ThrowNull:ERROR', '-Xep:TreeToString:ERROR', - // '-Xep:TruthSelfEquals:OFF', // we don't use truth '-Xep:TryFailThrowable:ERROR', '-Xep:TypeParameterQualifier:ERROR', '-Xep:UnicodeDirectionalityCharacters:ERROR', @@ -265,6 +269,7 @@ allprojects { prj -> '-Xep:AssertionFailureIgnored:WARN', '-Xep:AssistedInjectAndInjectOnSameConstructor:WARN', '-Xep:AttemptedNegativeZero:WARN', + // '-Xep:AutoValueBoxedValues:WARN', // todo check if useful or comment why not // '-Xep:AutoValueFinalMethods:OFF', // we don't use autovalue // '-Xep:AutoValueImmutableFields:OFF', // we don't use autovalue // '-Xep:AutoValueSubclassLeaked:OFF', // we don't use autovalue @@ -285,6 +290,7 @@ allprojects { prj -> '-Xep:ChainedAssertionLosesContext:WARN', '-Xep:CharacterGetNumericValue:WARN', '-Xep:ClassCanBeStatic:WARN', + // '-Xep:ClassInitializationDeadlock:WARN', // todo check if useful or comment why not '-Xep:ClassNewInstance:WARN', // '-Xep:CloseableProvides:OFF', // we don't use this annotation '-Xep:ClosingStandardOutputStreams:WARN', @@ -296,6 +302,8 @@ allprojects { prj -> '-Xep:DateChecker:WARN', '-Xep:DateFormatConstant:WARN', // '-Xep:DefaultCharset:OFF', // we have forbiddenapis for that + //'-Xep:DeeplyNested:WARN', // todo check if useful or comment why not + //'-Xep:DefaultLocale:WARN', // todo check if useful or comment why not '-Xep:DefaultPackage:WARN', '-Xep:DeprecatedVariable:WARN', '-Xep:DirectInvocationOnMock:WARN', @@ -309,6 +317,7 @@ allprojects { prj -> '-Xep:EmptyBlockTag:WARN', // '-Xep:EmptyCatch:OFF', // todo check if useful or comment why not - might be handled by ECJ? // '-Xep:EmptySetMultibindingContributions:OFF', // we don't use this annotation + // '-Xep:EnumOrdinal:WARN', // todo check if useful or comment why not '-Xep:EqualsGetClass:WARN', '-Xep:EqualsIncompatibleType:WARN', '-Xep:EqualsUnsafeCast:WARN', @@ -330,6 +339,7 @@ allprojects { prj -> // '-Xep:FragmentNotInstantiable:OFF', // we don't use android // '-Xep:FutureReturnValueIgnored:OFF', // todo there are problems that should be fixed '-Xep:GetClassOnEnum:WARN', + // '-Xep:GuiceNestedCombine:WARN', // todo check if useful or comment why not '-Xep:HidingField:WARN', '-Xep:ICCProfileGetInstance:WARN', '-Xep:IdentityHashMapUsage:WARN', @@ -383,6 +393,7 @@ allprojects { prj -> '-Xep:JodaPlusMinusLong:WARN', '-Xep:JodaTimeConverterManager:WARN', '-Xep:JodaWithDurationAddedLong:WARN', + // '-Xep:JUnitIncompatibleType:WARN', // todo check if useful or comment why not // '-Xep:LabelledBreakTarget:OFF', // stylistic '-Xep:LiteEnumValueOf:WARN', '-Xep:LiteProtoToString:WARN', @@ -403,10 +414,12 @@ allprojects { prj -> // '-Xep:MissingSummary:OFF', // style preference that we don't want to enforce // '-Xep:MixedMutabilityReturnType:OFF', // todo check if useful or comment why not '-Xep:MockNotUsedInProduction:WARN', + // '-Xep:MockitoDoSetup:WARN', // todo check if useful or comment why not '-Xep:ModifiedButNotUsed:WARN', '-Xep:ModifyCollectionInEnhancedForLoop:WARN', '-Xep:ModifySourceCollectionInStream:WARN', '-Xep:MultimapKeys:WARN', + // '-Xep:MultipleNullnessAnnotations:WARN', // todo check if useful or comment why not '-Xep:MultipleParallelOrSequentialCalls:WARN', '-Xep:MultipleUnaryOperatorsInMethodCall:WARN', // '-Xep:MutableGuiceModule:OFF', // we don't use guice @@ -428,7 +441,9 @@ allprojects { prj -> '-Xep:NullableOptional:WARN', // '-Xep:NullablePrimitive:OFF', // we don't use this annotation // '-Xep:NullablePrimitiveArray:OFF', // we don't use this annotation + // '-Xep:NullableTypeParameter:WARN', // todo check if useful or comment why not // '-Xep:NullableVoid:OFF', // we don't use this annotation + // '-Xep:NullableWildcard:WARN', // todo check if useful or comment why not '-Xep:ObjectEqualsForPrimitives:WARN', // '-Xep:ObjectToString:OFF', // todo check if useful or comment why not '-Xep:ObjectsHashCodePrimitive:WARN', @@ -442,6 +457,7 @@ allprojects { prj -> '-Xep:Overrides:WARN', // '-Xep:OverridesGuiceInjectableMethod:OFF', // we don't use guice '-Xep:ParameterName:WARN', + '-Xep:PatternMatchingInstanceof:WARN', '-Xep:PreconditionsCheckNotNullRepeated:WARN', '-Xep:PrimitiveAtomicReference:WARN', '-Xep:ProtectedMembersInFinalClass:WARN', @@ -459,6 +475,7 @@ allprojects { prj -> // '-Xep:SameNameButDifferent:OFF', // todo check if useful or comment why not '-Xep:SelfAlwaysReturnsThis:WARN', // '-Xep:ShortCircuitBoolean:OFF', // todo check if useful or comment why not + // '-Xep:StatementSwitchToExpressionSwitch:WARN', // todo check if useful or comment why not // '-Xep:StaticAssignmentInConstructor:OFF', // we assign SolrTestCaseJ4.configString in many tests, difficult to untangle '-Xep:StaticAssignmentOfThrowable:WARN', // '-Xep:StaticGuardedByInstance:OFF', // todo check if useful or comment why not @@ -469,9 +486,12 @@ allprojects { prj -> '-Xep:StringCharset:WARN', '-Xep:StringFormatWithLiteral:WARN', // '-Xep:StringSplitter:OFF', // todo check if useful or comment why not - might be able to use forbidden-apis for this? + // '-Xep:SunApi:WARN', // todo check if useful or comment why not + // '-Xep:SuperCallToObjectMethod:WARN', // todo check if useful or comment why not '-Xep:SuperEqualsIsObjectEquals:WARN', // '-Xep:SwigMemoryLeak:OFF', // we don't use swig // '-Xep:SynchronizeOnNonFinalField:OFF', // todo check if useful or comment why not + // '-Xep:SystemConsoleNull:WARN', // todo check if useful or comment why not // '-Xep:ThreadJoinLoop:OFF', // todo check if useful or comment why not // '-Xep:ThreadLocalUsage:OFF', // todo check if useful or comment why not // '-Xep:ThreadPriorityCheck:OFF', // todo check if useful or comment why not @@ -493,6 +513,7 @@ allprojects { prj -> // '-Xep:UnicodeEscape:OFF', // can't enable since Lucene/Solr tests use unicode a bunch // '-Xep:UnnecessaryAssignment:OFF', // we don't use these annotations '-Xep:UnnecessaryAsync:WARN', + // '-Xep:UnnecessaryBreakInSwitch:WARN', // todo check if useful or comment why not '-Xep:UnnecessaryLambda:WARN', '-Xep:UnnecessaryLongToIntConversion:WARN', '-Xep:UnnecessaryMethodInvocationMatcher:WARN', @@ -513,6 +534,7 @@ allprojects { prj -> // '-Xep:UseBinds:OFF', // we don't use this annotation // '-Xep:UseCorrectAssertInTests:OFF', // we inherit from LuceneTestCase which extends Assert '-Xep:VariableNameSameAsType:WARN', + // '-Xep:VoidUsed:WARN', // todo check if useful or comment why not // '-Xep:WaitNotInLoop:OFF', // todo check if useful or comment why not // '-Xep:WakelockReleasedDangerously:OFF', // we don't use android // '-Xep:WithSignatureDiscouraged:OFF', // we aren't using this error-prone internal api diff --git a/gradle/validation/forbidden-apis.gradle b/gradle/validation/forbidden-apis.gradle index 0ffb3a3a272..2a09ff49dda 100644 --- a/gradle/validation/forbidden-apis.gradle +++ b/gradle/validation/forbidden-apis.gradle @@ -23,7 +23,7 @@ def resources = scriptResources(buildscript) // Only apply forbidden-apis to java projects. allprojects { prj -> plugins.withId("java", { - prj.apply plugin: 'de.thetaphi.forbiddenapis' + prj.apply plugin: libs.plugins.thetaphi.forbiddenapis.get().pluginId // This helper method appends signature files based on a set of true // dependencies from a given configuration. diff --git a/gradle/validation/forbidden-apis/commons-cli.commons-cli.all.txt b/gradle/validation/forbidden-apis/commons-cli.commons-cli.all.txt new file mode 100644 index 00000000000..469fef8238f --- /dev/null +++ b/gradle/validation/forbidden-apis/commons-cli.commons-cli.all.txt @@ -0,0 +1,16 @@ +@defaultMessage Use a org.apache.commons.cli.Option instead of a String value +org.apache.commons.cli.CommandLine#hasOption(java.lang.String) +org.apache.commons.cli.CommandLine#getOptionValue(java.lang.String) +org.apache.commons.cli.CommandLine#getOptionValue(java.lang.String, java.lang.String) +org.apache.commons.cli.CommandLine#getParsedOptionValue(java.lang.String, java.lang.Object) +org.apache.commons.cli.CommandLine#hasOption(char) +org.apache.commons.cli.CommandLine#getOptionValue(char) +org.apache.commons.cli.CommandLine#getOptionValue(char, java.lang.String) +#org.apache.commons.cli.CommandLine#getOptionValue(char, Supplier) +org.apache.commons.cli.CommandLine#getOptionValues(char) +org.apache.commons.cli.CommandLine#getOptionValues(java.lang.String) +org.apache.commons.cli.CommandLine#getParsedOptionValue(char) +# org.apache.commons.cli.CommandLine#getParsedOptionValue(char, Supplier) +org.apache.commons.cli.CommandLine#getParsedOptionValue(char, java.lang.Object) +org.apache.commons.cli.CommandLine#getParsedOptionValue(java.lang.String) +# org.apache.commons.cli.CommandLine#getParsedOptionValue(String, Supplier) diff --git a/gradle/validation/git-status.gradle b/gradle/validation/git-status.gradle index b34cf831ef7..8a43e7c7b3d 100644 --- a/gradle/validation/git-status.gradle +++ b/gradle/validation/git-status.gradle @@ -33,7 +33,7 @@ buildscript { } dependencies { - classpath "org.eclipse.jgit:org.eclipse.jgit:${scriptDepVersions['jgit']}" + classpath libs.eclipse.jgit.jgit } } diff --git a/gradle/validation/jar-checks.gradle b/gradle/validation/jar-checks.gradle index 7547214299d..650a3b3337b 100644 --- a/gradle/validation/jar-checks.gradle +++ b/gradle/validation/jar-checks.gradle @@ -36,7 +36,7 @@ buildscript { } dependencies { - classpath "commons-codec:commons-codec:${scriptDepVersions['commons-codec']}" + classpath libs.commonscodec.commonscodec } } diff --git a/gradle/validation/precommit.gradle b/gradle/validation/precommit.gradle index 8c2fe6cfd98..cc298b8771f 100644 --- a/gradle/validation/precommit.gradle +++ b/gradle/validation/precommit.gradle @@ -23,8 +23,6 @@ configure(rootProject) { description = "All precommit checks" // Root-level validation tasks. - dependsOn ":verifyLocks" - dependsOn ":versionsPropsAreSorted" dependsOn ":checkWorkingCopyClean" // Solr validation tasks. diff --git a/gradle/validation/rat-sources.gradle b/gradle/validation/rat-sources.gradle index 577529a9c16..5b600f956c0 100644 --- a/gradle/validation/rat-sources.gradle +++ b/gradle/validation/rat-sources.gradle @@ -24,7 +24,7 @@ configure(rootProject) { } dependencies { - ratDeps "org.apache.rat:apache-rat:${scriptDepVersions['apache-rat']}" + ratDeps libs.apache.rat.rat } } @@ -106,10 +106,6 @@ allprojects { exclude "src/test-files/META-INF/services/*" break - case ":solr:modules:hadoop-auth": - exclude "src/test-files/**/*.conf" - break - case ":solr:modules:hdfs": exclude "src/test-files/**/*.aff" exclude "src/test-files/**/*.dic" diff --git a/gradle/validation/spotless.gradle b/gradle/validation/spotless.gradle index d4444a87671..32234c93a4a 100644 --- a/gradle/validation/spotless.gradle +++ b/gradle/validation/spotless.gradle @@ -22,7 +22,7 @@ configure(allprojects) { prj -> plugins.withType(JavaPlugin) { - prj.apply plugin: 'com.diffplug.spotless' + prj.apply plugin: libs.plugins.diffplug.spotless.get().pluginId project.ext { spotlessJavaSetup = (Action){ @@ -34,7 +34,7 @@ configure(allprojects) { prj -> // it.licenseHeaderFile(file("${resources}/asl-header.txt"), '^(\\s*package)') it.setLineEndings(Enum.valueOf(rootProject.buildscript.classLoader.loadClass("com.diffplug.spotless.LineEnding"), "UNIX")) it.endWithNewline() - it.googleJavaFormat('1.18.1') + it.googleJavaFormat(libs.versions.google.javaformat.get()) it.custom('Refuse wildcard imports', { line -> // Wildcard imports can't be resolved by spotless itself. diff --git a/gradle/validation/validate-source-patterns.gradle b/gradle/validation/validate-source-patterns.gradle index ec44c804a9b..9c4c93353e0 100644 --- a/gradle/validation/validate-source-patterns.gradle +++ b/gradle/validation/validate-source-patterns.gradle @@ -29,7 +29,7 @@ buildscript { } dependencies { - classpath "org.apache.rat:apache-rat:${scriptDepVersions['apache-rat']}" + classpath libs.apache.rat.rat } } diff --git a/help/dependencies.txt b/help/dependencies.txt index c1c81c560b0..fd1bc68b711 100644 --- a/help/dependencies.txt +++ b/help/dependencies.txt @@ -7,7 +7,7 @@ and each configuration can have dependencies attached to it. There are some standard conventions so, for example, the Java plugin adds standard configurations such as "api", "implementation", "testImplementation" and others. These configurations can also inherit -from each other; more about this typic can be found here: +from each other; more about this topic can be found here: https://docs.gradle.org/current/userguide/dependency_management_for_java_projects.html#dependency_management_for_java_projects https://docs.gradle.org/current/userguide/java_library_plugin.html#sec:java_library_separation @@ -29,60 +29,126 @@ testImplementation - makes a dependency only available for test classes. Adding a library dependency --------------------------- -Let's say we wish to add a dependency on library "foo.bar:baz" in +Let's say we wish to add a new dependency on library "foo.bar:baz" in version 1.2 to :solr:core. Let's assume this library is only -used internally by the project. The :solr:core project is configured -by solr/core/build.gradle and we would add (or modify) the dependency -block as follows: +used internally by the project. For new dependencies, we would add +the dependency and its version to gradle/libs.versions.toml first: + +[versions] +... +foo-bar-baz = "1.2" +... + +[libraries] +... +foo-bar-baz = { module = "foo.bar:baz", version.ref = "foo-bar-baz" } + +Note that the used names separated by dashes are later referenced with dots +instead of dashes, but more on that later. + +The chosen name for the module should more or less reflect the module's +group name and module id in a way that it groups related dependencies under +the same "prefix" (see below). There is no specific convention here and +group prefixes for domain names like "com" and "io" are avoided, as they +do not add any value and increase the size of the reference / alias name. + +The :solr:core project is configured by solr/core/build.gradle, and we would +add (or modify) the dependency block as follows: dependencies { - implementation "foo.bar:baz" + implementation libs.foo.bar.baz } +In the project we use the default name "libs" that is used to reference +the version catalog gradle/libs.versions.toml. + The "implementation" here is a named configuration; we don't need to declare it because it is declared for us by the java-library plugin. -In "normal" gradle the version of the dependency would be present -directly inside the declaration but we use a plugin -(palantir-consistent-versions) to manage all dependency versions -from the top-level (so that conflicts can be resolved globally). +In case the IDE does not auto-completes the reference, you may have +to sync your project so that the newly added library is found. -If this is the first time "foo.bar:baz" is added to the project, we'd have -to add its version to "versions.props" file at the top level of the -checkout: +As mentioned before, we can use the dashes to group related libraries +together. So let's assume we have another dependency on "foo.bar:biz", +which is part of the same project as "foo.bar:baz" and therefore share +the same version. -foo.bar:baz=1.2 +In this case we would want to use the same version for both libraries +and add them as follows to the version catalog gradle/libs.versions.toml: -and then regenerate the "versions.lock" file using the following -command: +[versions] +... +foo-bar = "1.2" # Use a shared name for both libraries +... -gradlew --write-locks +[libraries] +... +foo-bar-biz = { module = "foo.bar:biz", version.ref = "foo-bar" } +foo-bar-baz = { module = "foo.bar:baz", version.ref = "foo-bar" } -IMPORTANT: The versions.lock file will contain the actual version -of the dependency picked based on other project dependencies and -their transitive dependencies. This selected version may be -different from what each of these actually requires (the highest -version number will be typically selected). To see which dependencies -require which version of the library use: +This way, both libraries use the same version reference and updates +would affect both. -gradlew why --hash=... +Adding new libraries requires additional actions. The first you want +to do is to run versionCatalogFormat to sort the version catalog. -where the hash code comes from versions.lock file. For example, at -the time of writing, jackson-databind has the following entry: +This command does also remove unused libraries. You can use "# @keep" +with a reason why the library should not be removed. This is sometimes +necessary if the usage of a library is not identified by the plugin, +like when using it with "classpath [dependency]". -com.fasterxml.jackson.core:jackson-databind:2.10.0 (3 constraints: 931a7796) +The next you want to regenerate the "versions.lock" file using the +following command: -and "gradlew why --hash=931a7796" prints: +gradlew writeLocks -com.fasterxml.jackson.core:jackson-databind:2.10.0 - projects -> 2.10.0 - net.thisptr:jackson-jq -> 2.7.0 - org.carrot2:carrot2-mini -> 2.9.9.3 +Since we are responsible to provide and maintain the versions of +libraries, the lock file will reflect the versions of the version +catalog. -Once the dependency is added it always makes sense to see the -tree of all module dependencies and maybe exclude transitive -dependencies of foo.bar:baz that we won't need. +The locking will fail if multiple versions of the same dependency are found. +This may be the case if libraries have a used library as transitive +dependency with a different version. If that is the case, you have to add +a constraint to the modules in gradle/dependencies.gradle with a reason +why the constraint is applied. The below example adds a constraint for +"foo.bar:baz" with the given version from the version catalog, enforcing +the version to all transitive dependencies as well: +dependencies { + ... + constraints { handler -> + consolidatedConfigurations.configureEach { Configuration conf -> + ... + handler.add(conf.name, libs.foo.bar.baz, { + because 'version alignment for consistency across project' + }) + } + } +} + +Because the constraints have to be maintained by the contributors and cleaned +up manually, you should always provide additional information why the constraint +is needed. This information may include the current version of the libraries +that add transitively the dependency of the constraint. This helps others later +find and cleanup constraints if they update dependencies that use newer versions +of the conflicting dependency. For example: + +handler.add(conf.name, libs.ow2.asm, { + because "transitive version alignment for consistency across project" + + "\n- ${getFullName(libs.apache.lucene.expressions)} uses 7.2" + + "\n- ${getFullName(libs.apache.tika.parsers)} uses 9.3" + }) + +In this case, the module org.ow2.asm:asm is used by both org.apache.lucene:lucene-expressions +and org.apache.tika:tika-parsers, but with completely different versions. +The constraint syncs the transitive dependency and lets the maintainers know +which dependencies use it. So if at some point someone else updates the +lucene-expressions to a newer version that uses asm version 9.3, the constraint +would be obsolete and could be removed. + +The hashes from the versions.lock file can be used to look up +which modules use a specific library. Simply look up the hash in the +versions.lock and you will find a group of modules that use it. Update Lucene prerelease ------------------------ @@ -100,12 +166,12 @@ If you want to upgrade Lucene to a newer build proceed like the following: queued) - remember the build number of Jenkins (left side, first build in list, prefixed by '#') -- Edit ./versions.props and change Lucene's version to '9.0.0-prereleaseX', +- Edit gradle/libs.versions.toml and change Lucene's version to '9.0.0-prereleaseX', with 'X' is the jenkins build number - Edit ./gradle/globals.gradle and change jenkins build number, too (this directs the repository to the one created by latest build): def lucenePrereleaseBuild = 'X' -- Run: gradlew --write-locks (as described before) +- Run: gradlew writeLocks (as described before) Lucene local dependency substitution @@ -173,7 +239,7 @@ crucial for the functioning of "foo.bar:baz". We can exclude it by adding an exclusion block to the original declaration: dependencies { - implementation("foo.bar:baz", { + implementation(libs.foo.bar.biz, { exclude group: "foo.bar", module: "irrelevant" }) } @@ -194,3 +260,31 @@ gradlew licenses To update JAR checksums (sha1) for licenses use: gradlew updateLicenses + +Note that this Gradle task does only update the checksums and does not make +any changes to license or notice files. + +Whenever the library's LICENSE or NOTICE file changes, these changes should +be reflected in our copies under solr/licenses. Currently it is necessary to +manually review and update these files. This process is subject to change, +progress can be tracked in SOLR-15929. + +License and notice files may be picked from the libraries' repositories. +When looking up the libraries in Maven Central (https://search.maven.org) +almost all projects have a direct reference to the source code (right side), +usually a GitHub repository, where you can find the License and Notice file +in the root directory. + +Remember to check out the correct tag / release before copying any license +or notice file. Some multi-module projects that publish multiple artifacts +may have subdirectories for each artifact. These directories sometimes +hold a different license for that specific artifact, so make sure to copy +the right license file. Other multi-module projects may have only a single +license and notice file for all modules, like netty, so multiple dependencies +fromt he same group may reference the same license and notice file. + +Other places where you may find a license and notice file are in the pom.xml +file as a URL under a tag if there is no reference to a repository +in Maven Central, or in the artifact downloaded by maven when the library +is added as a dependency (in IntelliJ IDEA the libraries can be found +in the project view under External Libraries at the bottom). diff --git a/settings-gradle.lockfile b/settings-gradle.lockfile new file mode 100644 index 00000000000..709a43f74f8 --- /dev/null +++ b/settings-gradle.lockfile @@ -0,0 +1,4 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +empty=incomingCatalogForLibs0 diff --git a/settings.gradle b/settings.gradle index 7035b75bfdf..1cd64f9a04a 100644 --- a/settings.gradle +++ b/settings.gradle @@ -48,10 +48,10 @@ include "solr:modules:cross-dc" include "solr:modules:opentelemetry" include "solr:modules:extraction" include "solr:modules:gcs-repository" -include "solr:modules:hadoop-auth" include "solr:modules:hdfs" include "solr:modules:jwt-auth" include "solr:modules:langid" +include "solr:modules:llm" include "solr:modules:ltr" include "solr:modules:s3-repository" include "solr:modules:scripting" diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 907827a6fd0..a135ecaa539 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -12,6 +12,8 @@ New Features * SOLR-17467: Solr CLI bin/solr start defaults to starting Solr in Cloud mode, use --user-managed switch for User Managed (aka Standalone) mode. (Eric Pugh) +* SOLR-14673: Solr CLI now has bin/solr stream tool that executates streaming expressions via command line, either locally or on solr cluster. (Eric Pugh) + Improvements --------------------- @@ -28,9 +30,18 @@ Improvements * SOLR-17544: Solr CLI will now stop when you combine mutually exclusive options. Combining -s and -z options is a common example. (Eric Pugh, Christos Malliaridis) +* SOLR-17495: Change Solr CLI delete command to not delete configs by default. Decouple lifecycle of collections from configsets. (Eric Pugh) + +* SOLR-17516: `LBHttp2SolrClient` is now generic, adding support for `HttpJdkSolrClient`. (James Dyer) + +* SOLR-17541: `LBHttp2SolrClient` now maintains a separate internal/delegate client per Solr Base URL. Both `LBHttp2SolrClient` and `CloudHttp2SolrClient` + always create and manage these internal clients. The ability for callers to provide a pre-built client is removed. Callers may specify the internal client + details by providing an instance of either `Http2SolrClient.Builder` or `HttpJdkSolrClient.Builder`. (James Dyer) + Optimizations --------------------- -(No changes) +* SOLR-17568: The CLI bin/solr export tool now contacts the appropriate nodes directly for data instead of proxying through one. + (David Smiley) Bug Fixes --------------------- @@ -67,7 +78,7 @@ Deprecation Removals in WordBreakSolrSpellChecker (Andrey Bozhko via Eric Pugh) * SOLR-14763: Remove deprecated asynchronous request methods from `Http2SolrClient`, `HttpJdkSolrClient` and `LBHttp2SolrClient` - in favor of the new CompletableFuture based methods. Remove the related deprecated interfaces `AsyncListener` and ``Cancellable` + in favor of the new CompletableFuture based methods. Remove the related deprecated interfaces `AsyncListener` and `Cancellable` (James Dyer) * SOLR-14115: Remove deprecated zkcli script in favour of equivalent bin/solr sub commmands. (Eric Pugh) @@ -82,6 +93,24 @@ Deprecation Removals * SOLR-17352: Remove deprecated Solr CLI options. Run bin/solr yourcommand -h to see current options. (Eric Pugh, Christos Malliardis) +* SOLR-17256: Previously deprecated `SolrRequest` methods `setBasePath` and `getBasePath` have been removed. SolrJ users + wishing to temporarily override an HTTP client's base URL may use `Http2SolrClient.requestWithBaseUrl` instead. (Jason Gerlowski) + +* SOLR-17564: Remove code in Assign used for backwards compatibility with Collections created prior to 7.0 (Paul McArthur) + +* SOLR-17576: Remove deprecated master/slave option language from ReplicationHandler. (Eric Pugh) + +* SOLR-16781: Support for `` directives (used in solrconfig.xml to add JARs on a core-by-core basis) has been removed. Users + looking for similar functionality can use Solr's package manager. Users that don't need to vary JAR access on a per-core basis + have many options, including the `` tag and directly modifying Solr's classpath prior to JVM startup. (Jason Gerlowski) + +* SOLR-17540: Removed the Hadoop Auth module, and thus Kerberos authentication and other exotic options. (Eric Pugh) + +* SOLR-17541: Removed `CloudHttp2SolrClient.Builder#withHttpClient` in favor of `CloudHttp2SolrClient.Builder#withInternalClientBuilder`. + The constructor on `LBHttp2SolrClient.Builder` that took an instance of `HttpSolrClientBase` is updated to instead take an instance of + `HttpSolrClientBuilderBase`. Renamed `LBHttp2SolrClient.Builder#withListenerFactory` to `LBHttp2SolrClient.Builder#withListenerFactories` + (James Dyer) + Dependency Upgrades --------------------- (No changes) @@ -117,6 +146,41 @@ Other Changes * SOLR-17321: Minimum Java version for Apache Solr is now 21, and for SolrJ, it is 17. (Sanjay Dutt, David Smiley) +* SOLR-16903: Update CLI tools to use java.nio.file.Path instead of java.io.File (Andrey Bozhko) + +* SOLR-17568: SolrCloud no longer reroutes/proxies a core request to another node if not found locally. (David Smiley) + +* SOLR-17548: Switch all public Java APIs from File to Path. (Matthew Biscocho via Eric Pugh) + +================== 9.9.0 ================== +New Features +--------------------- +(No changes) + +Improvements +--------------------- +* SOLR-17541: Deprecate `CloudHttp2SolrClient.Builder#withHttpClient` in favor of + `CloudHttp2SolrClient.Builder#withInternalClientBuilder`. + Deprecate `LBHttp2SolrClient.Builder#withListenerFactory` in favor of + `LBHttp2SolrClient.Builder#withListenerFactories` (James Dyer) + +Optimizations +--------------------- +* SOLR-17578: Remove ZkController internal core supplier, for slightly faster reconnection after Zookeeper session loss. (Pierre Salagnac) + +Bug Fixes +--------------------- +(No changes) + +Dependency Upgrades +--------------------- +(No changes) + +Other Changes +--------------------- +* SOLR-17579: Remove unused code and other refactorings in ReplicationHandler and tests. Removed unused public + LOCAL_ACTIVITY_DURING_REPLICATION variable. (Eric Pugh) + ================== 9.8.0 ================== New Features --------------------- @@ -126,6 +190,8 @@ New Features * SOLR-17150: Implement `memAllowed` parameter to limit per-thread memory allocations during request processing. (Andrzej Bialecki, Gus Heck) +* SOLR-17525: Added knn_text_to_vector query parser to encode text to vector at query time through external LLM services. (Alessandro Benedetti) + Improvements --------------------- * SOLR-17158: Users using query limits (timeAllowed, cpuTimeAllowed) for whom partial results are uninteresting @@ -158,6 +224,17 @@ Improvements * SOLR-17390: EmbeddedSolrServer now considers the ResponseParser (David Smiley) +* SOLR-16390: v2 "cluster prop" APIs have been updated to be more REST-ful. Cluster prop creation/update are now available + at `PUT /api/cluster/properties/somePropName`. Deletion is now available at `DELETE /api/cluster/properties/somePropName`. + New APIs for listing-all and fetching-single cluster props are also now available at `GET /api/cluster/properties` and + `GET /api/cluster/properties/somePropName`, respectively. (Carlos Ugarte via Jason Gerlowski) + +* SOLR-16470: Replication "fetch file" API now has a v2 equivalent, available at `GET /api/cores/coreName/replication/files/fileName` + (Matthew Biscocho via Jason Gerlowski) + +* SOLR-17554: Suppress printing out of password to console when using auth CLI command. (Christos Malliaridis via Eric Pugh) + + Optimizations --------------------- * SOLR-14985: Solrj CloudSolrClient with Solr URLs had serious performance regressions (since the @@ -168,7 +245,9 @@ Optimizations that which consumes almost no memory, saving 1MB of memory per SolrCore. (David Smiley) * SOLR-17381: Make CLUSTERSTATUS request configurable to improve performance by allowing retrieval of specific information, - reducing unnecessary data fetching. (Aparna Suresh, David Smiley) + reducing unnecessary data fetching. Enhanced CloudSolrClient's HTTP ClusterStateProvider to use it, and to scale to + more collections better as well. + (Aparna Suresh, David Smiley) * SOLR-17396: Reduce thread contention in ZkStateReader.getCollectionProperties(). (Aparna Suresh, David Smiley, Paul McArthur) @@ -179,7 +258,8 @@ Optimizations * SOLR-17441: Improve system metrics collection by skipping unreadable MXBean properties, making /admin/info/system calls faster (Haythem Khiri) -* SOLR-16503: Switched from HTTP1 to HTTP2 in SolrClientCloudManager by replacing CloudLegacySolrClient with CloudHttp2SolrClient. (Sanjay Dutt, David Smiley) +* SOLR-17592: Switched from HTTP1 to HTTP2 in SolrCloudManager via HttpClient change from Apache to Jetty. + (Sanjay Dutt, David Smiley) * SOLR-17453: Leverage waitForState() instead of busy waiting in CREATE, MIGRATE, REINDEXCOLLECTION, MOVEREPLICA commands, and in some tests. (Pierre Salagnac) @@ -205,6 +285,15 @@ Bug Fixes * SOLR-16976: Remove log4j-jul jar and use slf4j bridge for JUL to prevent exception from being logged when remote JMX is enabled (Shawn Heisey, Stephen Zhou, Eric Pugh, Christine Poerschke, David Smiley) +* SOLR-17575: Fixed broken backwards compatibility with the legacy "langid.whitelist" config in Solr Langid. (Jan Høydahl, Alexander Zagniotov) + +* SOLR-17574: Fix AllowListUrlChecker when liveNodes changes. Remove ClusterState.getHostAllowList (Bruno Roustant, David Smiley) + +* SOLR-17595: Fix two issues in Solr CLI that prevent Solr from starting with the techproducts example and from + correctly parsing arguments on Windows that start with -D and have multiple values separated by "," or spaces. (Christos Malliaridis) + +* SOLR-17306: fix replication problem on follower restart (Martin Anzinger and Peter Kroiss via Eric Pugh) + Dependency Upgrades --------------------- (No changes) @@ -225,8 +314,21 @@ led to the suppression of exceptions. (Andrey Bozhko) * SOLR-17534: Introduce ClusterState.getCollectionNames, a convenience method (David Smiley) +* SOLR-17535: Introduce ClusterState.collectionStream to replace getCollectionStates, getCollectionsMap, + and forEachCollection, which are now deprecated. (David Smiley) + * SOLR-17545: Upgrade to Gradle 8.10 (Houston Putman) +* SOLR-17504: CoreContainer calls UpdateHandler.commit when closing a read-only core (Bruno Roustant) + +* SOLR-17556: "home" and "data" directories used by Solr examples have been updated to align with documented best practices. (Eric Pugh, Houston Putman) + +* SOLR-17577: Remove "solr.indexfetcher.sotimeout" system property that was for optimizing replication tests. It was disabled, but not removed. (Eric Pugh) + +* SOLR-14680: NamedList: deprecating methods: forEachEntry, forEachKey, abortableForEachKey, abortableForEach, + asMap (no-arg only), get(key, default). Added getOrDefault. Deprecated the SimpleMap interface as well as the + entirety of the SolrJ package org.apache.solr.cluster.api, which wasn't used except for SimpleMap. (David Smiley) + ================== 9.7.1 ================== Bug Fixes --------------------- @@ -383,7 +485,7 @@ Other Changes * GITHUB#2454: Refactor preparePutOrPost method in HttpJdkSolrClient (Andy Webb) -* SOLR-16503: Use Jetty HTTP2 for SyncStrategy and PeerSyncWithLeader for "recovery" operations (Sanjay Dutt, David Smiley) +* SOLR-17290: Use Jetty HTTP2 for SyncStrategy and PeerSyncWithLeader for "recovery" operations (Sanjay Dutt, David Smiley) * SOLR-16796: Include cyclonedx SBOMs with maven artifacts (Arnout Engelen, Houston Putman, Kevin Risden) diff --git a/solr/api/build.gradle b/solr/api/build.gradle index 1f8118116c6..adad0302602 100644 --- a/solr/api/build.gradle +++ b/solr/api/build.gradle @@ -16,8 +16,8 @@ */ plugins { - id 'io.swagger.core.v3.swagger-gradle-plugin' version '2.2.2' - id "org.openapi.generator" version "6.0.1" + alias(libs.plugins.swagger3.core) + alias(libs.plugins.openapi.generator) } apply plugin: 'java-library' @@ -58,18 +58,18 @@ resolve { } dependencies { - runtimeOnly 'org.slf4j:slf4j-api' + runtimeOnly libs.slf4j.api - implementation 'jakarta.ws.rs:jakarta.ws.rs-api' - implementation 'com.fasterxml.jackson.core:jackson-annotations' - api 'io.swagger.core.v3:swagger-annotations-jakarta' - implementation 'org.semver4j:semver4j' + implementation libs.jakarta.ws.rsapi + implementation libs.fasterxml.jackson.core.annotations + api libs.swagger3.annotations.jakarta + implementation libs.semver4j.semver4j testImplementation project(':solr:test-framework') testImplementation project(':solr:api') - testImplementation 'org.apache.lucene:lucene-test-framework' + testImplementation libs.apache.lucene.testframework - swaggerBuild 'io.swagger.core.v3:swagger-jaxrs2-jakarta' + swaggerBuild libs.swagger3.jaxrs2.jakarta } // Non-Java client generation tasks below: diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterPropertyApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterPropertyApis.java new file mode 100644 index 00000000000..5c75eec0c60 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/ClusterPropertyApis.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.endpoint; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import java.util.Map; +import org.apache.solr.client.api.model.ListClusterPropertiesResponse; +import org.apache.solr.client.api.model.SetClusterPropertyRequestBody; +import org.apache.solr.client.api.model.SolrJerseyResponse; + +/** Definitions for v2 JAX-RS cluster properties APIs. */ +@Path("/cluster/properties") +public interface ClusterPropertyApis { + @GET + @Operation( + summary = "List all cluster properties in this Solr cluster.", + tags = {"cluster-properties"}) + ListClusterPropertiesResponse listClusterProperties(); + + @GET + @Path("/{propertyName}") + @Operation( + summary = "Get a cluster property in this Solr cluster.", + tags = {"cluster-properties"}) + SolrJerseyResponse getClusterProperty( + @Parameter(description = "The name of the property being retrieved.", required = true) + @PathParam("propertyName") + String propertyName); + + @PUT + @Path("/{propertyName}") + @Operation( + summary = "Set a single new or existing cluster property in this Solr cluster.", + tags = {"cluster-properties"}) + SolrJerseyResponse createOrUpdateClusterProperty( + @Parameter(description = "The name of the property being set.", required = true) + @PathParam("propertyName") + String propertyName, + @RequestBody(description = "Value to set for the property", required = true) + SetClusterPropertyRequestBody requestBody) + throws Exception; + + @PUT + @Operation( + summary = "Set nested cluster properties in this Solr cluster.", + tags = {"cluster-properties"}) + SolrJerseyResponse createOrUpdateNestedClusterProperty( + @RequestBody(description = "Property/ies to be set", required = true) + Map propertyValuesByName) + throws Exception; + + @DELETE + @Path("/{propertyName}") + @Operation( + summary = "Delete a cluster property in this Solr cluster.", + tags = {"cluster-properties"}) + SolrJerseyResponse deleteClusterProperty( + @Parameter(description = "The name of the property being deleted.", required = true) + @PathParam("propertyName") + String propertyName); +} diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/CollectionBackupApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/CollectionBackupApi.java new file mode 100644 index 00000000000..5c6c9d3bbbe --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/CollectionBackupApi.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.endpoint; + +import io.swagger.v3.oas.annotations.Operation; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import org.apache.solr.client.api.model.CreateCollectionBackupRequestBody; +import org.apache.solr.client.api.model.RestoreCollectionRequestBody; +import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; + +/** + * V2 API definition for creating a new "backup" of a specified collection + * + *

This API is analogous to the v1 /admin/collections?action=BACKUP command. + */ +public interface CollectionBackupApi { + + @Path("/collections/{collectionName}/backups/{backupName}/versions") + interface Create { + @POST + @Operation( + summary = "Creates a new backup point for a collection", + tags = {"collection-backups"}) + SolrJerseyResponse createCollectionBackup( + @PathParam("collectionName") String collectionName, + @PathParam("backupName") String backupName, + CreateCollectionBackupRequestBody requestBody) + throws Exception; + } + + @Path("/backups/{backupName}/restore") + interface Restore { + @POST + @Operation( + summary = "Restores an existing backup point to a (potentially new) collection.", + tags = {"collection-backups"}) + SubResponseAccumulatingJerseyResponse restoreCollection( + @PathParam("backupName") String backupName, RestoreCollectionRequestBody requestBody) + throws Exception; + } +} diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/CollectionSnapshotApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/CollectionSnapshotApis.java new file mode 100644 index 00000000000..21c1dc44224 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/CollectionSnapshotApis.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.endpoint; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.DefaultValue; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.QueryParam; +import org.apache.solr.client.api.model.CreateCollectionSnapshotRequestBody; +import org.apache.solr.client.api.model.CreateCollectionSnapshotResponse; +import org.apache.solr.client.api.model.DeleteCollectionSnapshotResponse; +import org.apache.solr.client.api.model.ListCollectionSnapshotsResponse; + +/** V2 API definitions for creating, accessing, and deleting collection-level snapshots. */ +public interface CollectionSnapshotApis { + + @Path("/collections/{collName}/snapshots") + interface Create { + @POST + @Path("/{snapshotName}") + @Operation( + summary = "Creates a new snapshot of the specified collection.", + tags = {"collection-snapshots"}) + CreateCollectionSnapshotResponse createCollectionSnapshot( + @Parameter(description = "The name of the collection.", required = true) + @PathParam("collName") + String collName, + @Parameter(description = "The name of the snapshot to be created.", required = true) + @PathParam("snapshotName") + String snapshotName, + @RequestBody(description = "Contains user provided parameters", required = true) + CreateCollectionSnapshotRequestBody requestBody) + throws Exception; + } + + @Path("/collections/{collName}/snapshots/{snapshotName}") + interface Delete { + @DELETE + @Operation( + summary = "Delete an existing collection-snapshot by name.", + tags = {"collection-snapshots"}) + DeleteCollectionSnapshotResponse deleteCollectionSnapshot( + @Parameter(description = "The name of the collection.", required = true) + @PathParam("collName") + String collName, + @Parameter(description = "The name of the snapshot to be deleted.", required = true) + @PathParam("snapshotName") + String snapshotName, + @Parameter(description = "A flag that treats the collName parameter as a collection alias.") + @DefaultValue("false") + @QueryParam("followAliases") + boolean followAliases, + @QueryParam("async") String asyncId) + throws Exception; + } + + @Path("/collections/{collName}/snapshots") + interface List { + @GET + @Operation( + summary = "List the snapshots available for a specified collection.", + tags = {"collection-snapshots"}) + ListCollectionSnapshotsResponse listSnapshots( + @Parameter(description = "The name of the collection.", required = true) + @PathParam("collName") + String collName) + throws Exception; + } +} diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/CreateAliasApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/CreateAliasApi.java new file mode 100644 index 00000000000..78b9b4376c0 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/CreateAliasApi.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.endpoint; + +import io.swagger.v3.oas.annotations.Operation; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import org.apache.solr.client.api.model.CreateAliasRequestBody; +import org.apache.solr.client.api.model.SolrJerseyResponse; + +@Path("/aliases") +public interface CreateAliasApi { + @POST + @Operation( + summary = "Create a traditional or 'routed' alias", + tags = {"aliases"}) + SolrJerseyResponse createAlias(CreateAliasRequestBody requestBody) throws Exception; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/CreateCollectionSnapshotApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/CreateCollectionSnapshotApi.java deleted file mode 100644 index 70240a02e65..00000000000 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/CreateCollectionSnapshotApi.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.client.api.endpoint; - -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.Parameter; -import io.swagger.v3.oas.annotations.parameters.RequestBody; -import jakarta.ws.rs.POST; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.PathParam; -import org.apache.solr.client.api.model.CreateCollectionSnapshotRequestBody; -import org.apache.solr.client.api.model.CreateCollectionSnapshotResponse; - -/** V2 API definition for creating a collection-level snapshot. */ -@Path("/collections/{collName}/snapshots") -public interface CreateCollectionSnapshotApi { - @POST - @Path("/{snapshotName}") - @Operation( - summary = "Creates a new snapshot of the specified collection.", - tags = {"collection-snapshots"}) - CreateCollectionSnapshotResponse createCollectionSnapshot( - @Parameter(description = "The name of the collection.", required = true) - @PathParam("collName") - String collName, - @Parameter(description = "The name of the snapshot to be created.", required = true) - @PathParam("snapshotName") - String snapshotName, - @RequestBody(description = "Contains user provided parameters", required = true) - CreateCollectionSnapshotRequestBody requestBody) - throws Exception; -} diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteCollectionSnapshotApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteCollectionSnapshotApi.java deleted file mode 100644 index 8fed7eb4ff9..00000000000 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/DeleteCollectionSnapshotApi.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.client.api.endpoint; - -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.Parameter; -import jakarta.ws.rs.DELETE; -import jakarta.ws.rs.DefaultValue; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.PathParam; -import jakarta.ws.rs.QueryParam; -import org.apache.solr.client.api.model.DeleteCollectionSnapshotResponse; - -@Path("/collections/{collName}/snapshots/{snapshotName}") -public interface DeleteCollectionSnapshotApi { - - /** This API is analogous to V1's (POST /solr/admin/collections?action=DELETESNAPSHOT) */ - @DELETE - @Operation( - summary = "Delete an existing collection-snapshot by name.", - tags = {"collection-snapshots"}) - DeleteCollectionSnapshotResponse deleteCollectionSnapshot( - @Parameter(description = "The name of the collection.", required = true) - @PathParam("collName") - String collName, - @Parameter(description = "The name of the snapshot to be deleted.", required = true) - @PathParam("snapshotName") - String snapshotName, - @Parameter(description = "A flag that treats the collName parameter as a collection alias.") - @DefaultValue("false") - @QueryParam("followAliases") - boolean followAliases, - @QueryParam("async") String asyncId) - throws Exception; -} diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/GetSchemaApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/GetSchemaApi.java index 8cec1d02f7a..119bf9d19ce 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/GetSchemaApi.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/GetSchemaApi.java @@ -22,8 +22,16 @@ import jakarta.ws.rs.DefaultValue; import jakarta.ws.rs.GET; import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; import jakarta.ws.rs.QueryParam; +import org.apache.solr.client.api.model.SchemaGetDynamicFieldInfoResponse; +import org.apache.solr.client.api.model.SchemaGetFieldInfoResponse; +import org.apache.solr.client.api.model.SchemaGetFieldTypeInfoResponse; import org.apache.solr.client.api.model.SchemaInfoResponse; +import org.apache.solr.client.api.model.SchemaListCopyFieldsResponse; +import org.apache.solr.client.api.model.SchemaListDynamicFieldsResponse; +import org.apache.solr.client.api.model.SchemaListFieldTypesResponse; +import org.apache.solr.client.api.model.SchemaListFieldsResponse; import org.apache.solr.client.api.model.SchemaNameResponse; import org.apache.solr.client.api.model.SchemaSimilarityResponse; import org.apache.solr.client.api.model.SchemaUniqueKeyResponse; @@ -34,6 +42,67 @@ @Path(INDEX_PATH_PREFIX + "/schema") public interface GetSchemaApi { + @Path(INDEX_PATH_PREFIX + "/schema") + interface Fields { + + @GET + @Path("/fields") + @StoreApiParameters + @Operation( + summary = "List all non-dynamic fields in the schema of the specified core or collection", + tags = {"schema"}) + SchemaListFieldsResponse listSchemaFields(); + + @GET + @Path("/fields/{fieldName}") + @StoreApiParameters + @Operation( + summary = "Get detailed info about a single non-dynamic field", + tags = {"schema"}) + SchemaGetFieldInfoResponse getFieldInfo(@PathParam("fieldName") String fieldName); + + @GET + @Path("/copyfields") + @StoreApiParameters + @Operation( + summary = "List all copy-fields in the schema of the specified core or collection", + tags = {"schema"}) + SchemaListCopyFieldsResponse listCopyFields(); + + @GET + @Path("/dynamicfields") + @StoreApiParameters + @Operation( + summary = "List all dynamic-fields in the schema of the specified core or collection", + tags = {"schema"}) + SchemaListDynamicFieldsResponse listDynamicFields(); + + @GET + @Path("/dynamicfields/{fieldName}") + @StoreApiParameters + @Operation( + summary = "Get detailed info about a single dynamic field", + tags = {"schema"}) + SchemaGetDynamicFieldInfoResponse getDynamicFieldInfo(@PathParam("fieldName") String fieldName); + + @GET + @Path("/fieldtypes") + @StoreApiParameters + @Operation( + summary = "List all field types in the schema used by the specified core or collection", + tags = {"schema"}) + SchemaListFieldTypesResponse listSchemaFieldTypes(); + + @GET + @Path("/fieldtypes/{fieldTypeName}") + @StoreApiParameters + @Operation( + summary = "Get detailed info about a single field type", + tags = {"schema"}) + SchemaGetFieldTypeInfoResponse getFieldTypeInfo( + @PathParam("fieldTypeName") String fieldTypeName); + } + @GET @StoreApiParameters @Operation( diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeLoggingApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeLoggingApis.java new file mode 100644 index 00000000000..c5b5cd95807 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/NodeLoggingApis.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.endpoint; + +import io.swagger.v3.oas.annotations.Operation; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.QueryParam; +import java.util.List; +import org.apache.solr.client.api.model.ListLevelsResponse; +import org.apache.solr.client.api.model.LogLevelChange; +import org.apache.solr.client.api.model.LogMessagesResponse; +import org.apache.solr.client.api.model.LoggingResponse; +import org.apache.solr.client.api.model.SetThresholdRequestBody; + +@Path("/node/logging") +public interface NodeLoggingApis { + + @GET + @Path("/levels") + @Operation( + summary = "List all log-levels for the target node.", + tags = {"logging"}) + ListLevelsResponse listAllLoggersAndLevels(); + + @PUT + @Path("/levels") + @Operation( + summary = "Set one or more logger levels on the target node.", + tags = {"logging"}) + LoggingResponse modifyLocalLogLevel(List requestBody); + + @GET + @Path("/messages") + @Operation( + summary = "Fetch recent log messages on the targeted node.", + tags = {"logging"}) + LogMessagesResponse fetchLocalLogMessages(@QueryParam("since") Long boundingTimeMillis); + + @PUT + @Path("/messages/threshold") + @Operation( + summary = "Set a threshold level for the targeted node's log message watcher.", + tags = {"logging"}) + LoggingResponse setMessageThreshold(SetThresholdRequestBody requestBody); +} diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/ReplicationApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/ReplicationApis.java new file mode 100644 index 00000000000..3fe5ac14f45 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/ReplicationApis.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.endpoint; + +import static org.apache.solr.client.api.util.Constants.OMIT_FROM_CODEGEN_PROPERTY; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.extensions.Extension; +import io.swagger.v3.oas.annotations.extensions.ExtensionProperty; +import jakarta.ws.rs.DefaultValue; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.QueryParam; +import jakarta.ws.rs.core.StreamingOutput; +import java.io.IOException; +import org.apache.solr.client.api.model.FileListResponse; +import org.apache.solr.client.api.model.IndexVersionResponse; +import org.apache.solr.client.api.util.CoreApiParameters; + +@Path("/cores/{coreName}/replication") +public interface ReplicationApis { + + @GET + @CoreApiParameters + @Path("/indexversion") + @Operation( + summary = "Return the index version of the specified core.", + tags = {"replication"}) + IndexVersionResponse fetchIndexVersion() throws IOException; + + @GET + @CoreApiParameters + @Path("/files") + @Operation( + summary = "Return the list of index file that make up the specified core.", + tags = {"replication"}) + FileListResponse fetchFileList( + @Parameter(description = "The generation number of the index", required = true) + @QueryParam("generation") + long gen); + + @GET + @CoreApiParameters + @Operation( + summary = "Get a stream of a specific file path of a core", + tags = {"core-replication"}, + extensions = { // TODO Remove as a part of SOLR-17562 + @Extension( + properties = {@ExtensionProperty(name = OMIT_FROM_CODEGEN_PROPERTY, value = "true")}) + }) + @Path("/files/{filePath}") + StreamingOutput fetchFile( + @PathParam("filePath") String filePath, + @Parameter( + description = + "Directory type for specific filePath (cf or tlogFile). Defaults to Lucene index (file) directory if empty", + required = true) + @QueryParam("dirType") + String dirType, + @Parameter(description = "Output stream read/write offset", required = false) + @QueryParam("offset") + String offset, + @Parameter(required = false) @QueryParam("len") String len, + @Parameter(description = "Compress file output", required = false) + @QueryParam("compression") + @DefaultValue("false") + Boolean compression, + @Parameter(description = "Write checksum with output stream", required = false) + @QueryParam("checksum") + @DefaultValue("false") + Boolean checksum, + @Parameter( + description = "Limit data write per seconds. Defaults to no throttling", + required = false) + @QueryParam("maxWriteMBPerSec") + double maxWriteMBPerSec, + @Parameter(description = "The generation number of the index", required = false) + @QueryParam("generation") + Long gen); +} diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/CreateCollectionBackupApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/ReplicationBackupApis.java similarity index 56% rename from solr/api/src/java/org/apache/solr/client/api/endpoint/CreateCollectionBackupApi.java rename to solr/api/src/java/org/apache/solr/client/api/endpoint/ReplicationBackupApis.java index 3e08bad6d45..ef28f902ecc 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/CreateCollectionBackupApi.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/ReplicationBackupApis.java @@ -17,27 +17,27 @@ package org.apache.solr.client.api.endpoint; import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.parameters.RequestBody; import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; -import jakarta.ws.rs.PathParam; -import org.apache.solr.client.api.model.CreateCollectionBackupRequestBody; -import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.model.ReplicationBackupRequestBody; +import org.apache.solr.client.api.model.ReplicationBackupResponse; +import org.apache.solr.client.api.util.CoreApiParameters; /** - * V2 API definition for creating a new "backup" of a specified collection + * V2 endpoint for Backup API used for User-Managed clusters and Single-Node Installation. * - *

This API is analogous to the v1 /admin/collections?action=BACKUP command. + * @see ReplicationApis */ -@Path("/collections/{collectionName}/backups/{backupName}/versions") -public interface CreateCollectionBackupApi { +@Path("/cores/{coreName}/replication") +public interface ReplicationBackupApis { @POST + @CoreApiParameters + @Path("/backups") @Operation( - summary = "Creates a new backup point for a collection", - tags = {"collection-backups"}) - SolrJerseyResponse createCollectionBackup( - @PathParam("collectionName") String collectionName, - @PathParam("backupName") String backupName, - CreateCollectionBackupRequestBody requestBody) - throws Exception; + summary = "Create a backup of a single core using Solr's 'Replication Handler'", + tags = {"replication-backups"}) + ReplicationBackupResponse createBackup( + @RequestBody ReplicationBackupRequestBody backupReplicationPayload); } diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/ZooKeeperReadApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/ZooKeeperReadApis.java new file mode 100644 index 00000000000..f41e8de3d63 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/ZooKeeperReadApis.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.endpoint; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.QueryParam; +import jakarta.ws.rs.core.MediaType; +import org.apache.solr.client.api.model.ZooKeeperFileResponse; +import org.apache.solr.client.api.model.ZooKeeperListChildrenResponse; + +/** V2 API definitions for Solr's ZooKeeper ready-proxy endpoint */ +@Path("/cluster/zookeeper/") +public interface ZooKeeperReadApis { + + @GET + @Path("/data{zkPath:.+}") + @Operation( + summary = "Return the data stored in a specified ZooKeeper node", + tags = {"zookeeper-read"}) + @Produces({"application/vnd.apache.solr.raw", MediaType.APPLICATION_JSON}) + ZooKeeperFileResponse readNode( + @Parameter(description = "The path of the node to read from ZooKeeper") @PathParam("zkPath") + String zkPath); + + // The 'Operation' annotation is omitted intentionally here to ensure this API isn't picked up in + // the OpenAPI spec and consequent code-generation. The server side needs this method to be + // different from 'readNode' above for security reasons (more privileges are needed to access + // security.json), but it's the same logical API expressed by the 'readNode' signature above. + @GET + @Path("/data/security.json") + @Produces({"application/vnd.apache.solr.raw", MediaType.APPLICATION_JSON}) + ZooKeeperFileResponse readSecurityJsonNode(); + + @GET + @Path("/children{zkPath:.*}") + @Produces({"application/json", "application/javabin"}) + @Operation( + summary = "List and stat all children of a specified ZooKeeper node", + tags = {"zookeeper-read"}) + ZooKeeperListChildrenResponse listNodes( + @Parameter(description = "The path of the ZooKeeper node to stat and list children of") + @PathParam("zkPath") + String zkPath, + @Parameter( + description = + "Controls whether stat information for child nodes is included in the response. 'true' by default.") + @QueryParam("children") + Boolean includeChildren) + throws Exception; +} diff --git a/build-tools/scriptDepVersions.gradle b/solr/api/src/java/org/apache/solr/client/api/model/CategoryRoutedAliasProperties.java similarity index 63% rename from build-tools/scriptDepVersions.gradle rename to solr/api/src/java/org/apache/solr/client/api/model/CategoryRoutedAliasProperties.java index 329905331a9..c3882e0b8a3 100644 --- a/build-tools/scriptDepVersions.gradle +++ b/solr/api/src/java/org/apache/solr/client/api/model/CategoryRoutedAliasProperties.java @@ -14,20 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package org.apache.solr.client.api.model; -// Declare script dependency versions outside of palantir's -// version unification control. These are not our main dependencies -// but are reused in buildSrc and across applied scripts. +import com.fasterxml.jackson.annotation.JsonProperty; -ext { - scriptDepVersions = [ - "min-java-version": "21", - "min-solrj-java-version" : "17", - "apache-rat": "0.15", - "commons-codec": "1.16.0", - "ecj": "3.39.0", - "javacc": "7.0.12", - "jgit": "6.7.0.202309050840-r", - "flexmark": "0.64.8", - ] +public class CategoryRoutedAliasProperties extends RoutedAliasProperties { + @JsonProperty("maxCardinality") + public Long maxCardinality; + + @JsonProperty("mustMatch") + public String mustMatch; } diff --git a/solr/api/src/java/org/apache/solr/client/api/model/ClusterPropertyDetails.java b/solr/api/src/java/org/apache/solr/client/api/model/ClusterPropertyDetails.java new file mode 100644 index 00000000000..9619e96ac1e --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/ClusterPropertyDetails.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; + +public class ClusterPropertyDetails { + @JsonProperty("name") + @Schema(description = "The name of the cluster property.") + public String name; + + @JsonProperty("value") + @Schema(description = "The value of the cluster property.") + public Object value; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/CreateAliasRequestBody.java b/solr/api/src/java/org/apache/solr/client/api/model/CreateAliasRequestBody.java new file mode 100644 index 00000000000..f4fee3a2d39 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/CreateAliasRequestBody.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import java.util.List; + +public class CreateAliasRequestBody { + @JsonProperty(required = true) + public String name; + + @JsonProperty("collections") + public List collections; + + @JsonProperty("async") + public String async; + + @JsonProperty("routers") + public List routers; + + @Schema( + description = + "Parameters to be used for any collections created by this alias. Only used for 'routed' aliases", + name = "collCreationParameters") + @JsonProperty("create-collection") + public CreateCollectionRequestBody collCreationParameters; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/DeleteCollectionSnapshotResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/DeleteCollectionSnapshotResponse.java index 569d4fbd096..905b0937f1d 100644 --- a/solr/api/src/java/org/apache/solr/client/api/model/DeleteCollectionSnapshotResponse.java +++ b/solr/api/src/java/org/apache/solr/client/api/model/DeleteCollectionSnapshotResponse.java @@ -21,12 +21,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.v3.oas.annotations.media.Schema; -import org.apache.solr.client.api.endpoint.DeleteCollectionSnapshotApi; -/** - * The Response for {@link DeleteCollectionSnapshotApi#deleteCollectionSnapshot(String, String, - * boolean, String)} - */ +/** The Response for {@link org.apache.solr.client.api.endpoint.CollectionSnapshotApis.Delete} */ public class DeleteCollectionSnapshotResponse extends AsyncJerseyResponse { @Schema(description = "The name of the collection.") @JsonProperty(COLLECTION) diff --git a/solr/core/src/java/org/apache/solr/jersey/ExperimentalResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/ExperimentalResponse.java similarity index 92% rename from solr/core/src/java/org/apache/solr/jersey/ExperimentalResponse.java rename to solr/api/src/java/org/apache/solr/client/api/model/ExperimentalResponse.java index 7824b7bc314..c460537e3fb 100644 --- a/solr/core/src/java/org/apache/solr/jersey/ExperimentalResponse.java +++ b/solr/api/src/java/org/apache/solr/client/api/model/ExperimentalResponse.java @@ -15,10 +15,9 @@ * limitations under the License. */ -package org.apache.solr.jersey; +package org.apache.solr.client.api.model; import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.solr.client.api.model.SolrJerseyResponse; /** * {@link SolrJerseyResponse} implementation with a warning field indicating that the format may diff --git a/solr/api/src/java/org/apache/solr/client/api/model/FileListResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/FileListResponse.java new file mode 100644 index 00000000000..d42260449a4 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/FileListResponse.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; + +/** Response body for the `GET /api/cores/coreName/replication/files` API */ +public class FileListResponse extends SolrJerseyResponse { + @JsonProperty("filelist") + public List fileList; + + @JsonProperty("confFiles") + public List confFiles; + + @JsonProperty("status") + public String status; + + @JsonProperty("message") + public String message; + + @JsonProperty("exception") + public Exception exception; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/FileMetaData.java b/solr/api/src/java/org/apache/solr/client/api/model/FileMetaData.java new file mode 100644 index 00000000000..79f4d659021 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/FileMetaData.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class FileMetaData { + + @JsonProperty("size") + public long size; + + @JsonProperty("name") + public String name; + + @JsonProperty("checksum") + public long checksum; + + @JsonProperty("alias") + public String alias; + + public FileMetaData() {} + + public FileMetaData(long size, String name, long checksum) { + this.size = size; + this.name = name; + this.checksum = checksum; + } +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/GetClusterPropertyResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/GetClusterPropertyResponse.java new file mode 100644 index 00000000000..3ebdd74ef60 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/GetClusterPropertyResponse.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; + +public class GetClusterPropertyResponse extends SolrJerseyResponse { + @JsonProperty("clusterProperty") + @Schema(description = "The requested cluster property.") + public ClusterPropertyDetails clusterProperty; +} diff --git a/gradle/validation/versions-props-sorted.gradle b/solr/api/src/java/org/apache/solr/client/api/model/IndexVersionResponse.java similarity index 56% rename from gradle/validation/versions-props-sorted.gradle rename to solr/api/src/java/org/apache/solr/client/api/model/IndexVersionResponse.java index 3282faf8391..0d5633ca6e8 100644 --- a/gradle/validation/versions-props-sorted.gradle +++ b/solr/api/src/java/org/apache/solr/client/api/model/IndexVersionResponse.java @@ -14,22 +14,27 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package org.apache.solr.client.api.model; -// This ensures 'versions.props' file is sorted lexicographically. +import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.stream.Collectors +/** Response body for the `GET /api/cores/coreName/replication/indexversion` API */ +public class IndexVersionResponse extends SolrJerseyResponse { -configure(rootProject) { - task versionsPropsAreSorted() { - doFirst { - def versionsProps = file('versions.props') - // remove # commented lines and blank lines - def lines = versionsProps.readLines("UTF-8").stream().filter(l -> !l.matches(/^(#.*|\s*)$/)).collect(Collectors.toList()) - def sorted = lines.toSorted() + @JsonProperty("indexversion") + public Long indexVersion; - if (!Objects.equals(lines, sorted)) { - throw new GradleException("${versionsProps} file is not sorted lexicographically.") - } - } + @JsonProperty("generation") + public Long generation; + + @JsonProperty("status") + public String status; + + public IndexVersionResponse() {} + + public IndexVersionResponse(Long indexVersion, Long generation, String status) { + this.indexVersion = indexVersion; + this.generation = generation; + this.status = status; } } diff --git a/solr/api/src/java/org/apache/solr/client/api/model/ListClusterPropertiesResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/ListClusterPropertiesResponse.java new file mode 100644 index 00000000000..46504fb23f1 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/ListClusterPropertiesResponse.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import java.util.List; + +public class ListClusterPropertiesResponse extends SolrJerseyResponse { + @JsonProperty("clusterProperties") + @Schema(description = "The list of cluster properties.") + public List clusterProperties; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/ListCollectionSnapshotsResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/ListCollectionSnapshotsResponse.java new file mode 100644 index 00000000000..962cfda7014 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/ListCollectionSnapshotsResponse.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import java.util.Map; + +/** The Response for the v2 "list collection snapshots" API */ +public class ListCollectionSnapshotsResponse extends AsyncJerseyResponse { + + // TODO In practice, map values are of the CollectionSnapshotMetaData type, but that cannot be + // used here until the class is made into more of a POJO and can join the 'api' module here + @Schema(description = "The snapshots for the collection.") + @JsonProperty("snapshots") + public Map snapshots; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/ListLevelsResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/ListLevelsResponse.java new file mode 100644 index 00000000000..b7ad63ac4c1 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/ListLevelsResponse.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; + +/** Response format for the 'GET /api/node/logging/levels' API. */ +public class ListLevelsResponse extends LoggingResponse { + @JsonProperty public List levels; + @JsonProperty public List loggers; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/LogLevelChange.java b/solr/api/src/java/org/apache/solr/client/api/model/LogLevelChange.java new file mode 100644 index 00000000000..31443a5dc8d --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/LogLevelChange.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** A user-requested modification in the level that a specified logger reports at. */ +public class LogLevelChange { + public LogLevelChange() {} + + public LogLevelChange(String logger, String level) { + this.logger = logger; + this.level = level; + } + + @JsonProperty public String logger; + @JsonProperty public String level; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/LogLevelInfo.java b/solr/api/src/java/org/apache/solr/client/api/model/LogLevelInfo.java new file mode 100644 index 00000000000..9bde4b1657a --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/LogLevelInfo.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** Representation of a single logger and its current state. */ +public class LogLevelInfo { + public LogLevelInfo() {} + + public LogLevelInfo(String name, String level, boolean set) { + this.name = name; + this.level = level; + this.set = set; + } + + @JsonProperty("name") + public String name; + + @JsonProperty("level") + public String level; + + @JsonProperty("set") + public boolean set; +} diff --git a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/HadoopAuthFakeGroupMapping.java b/solr/api/src/java/org/apache/solr/client/api/model/LogMessageInfo.java similarity index 61% rename from solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/HadoopAuthFakeGroupMapping.java rename to solr/api/src/java/org/apache/solr/client/api/model/LogMessageInfo.java index 053f3e55061..7596cc28f68 100644 --- a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/HadoopAuthFakeGroupMapping.java +++ b/solr/api/src/java/org/apache/solr/client/api/model/LogMessageInfo.java @@ -14,22 +14,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.security.hadoop; +package org.apache.solr.client.api.model; -import java.util.Collections; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; -import org.apache.hadoop.security.GroupMappingServiceProvider; -/** Fake mapping for Hadoop to prevent falling back to Shell group provider */ -public class HadoopAuthFakeGroupMapping implements GroupMappingServiceProvider { - @Override - public List getGroups(String user) { - return Collections.singletonList("supergroup"); - } +/** Metadata about the log messages returned by the 'GET /api/node/logging/messages' API */ +public class LogMessageInfo { + @JsonProperty("since") + public Long boundingTimeMillis; - @Override - public void cacheGroupsRefresh() {} + @JsonProperty public Boolean found; + @JsonProperty public List levels; - @Override - public void cacheGroupsAdd(List groups) {} + @JsonProperty("last") + public long lastRecordTimestampMillis; + + @JsonProperty public int buffer; + @JsonProperty public String threshold; } diff --git a/solr/api/src/java/org/apache/solr/client/api/model/LogMessagesResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/LogMessagesResponse.java new file mode 100644 index 00000000000..fb979afdfeb --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/LogMessagesResponse.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** Response format for the 'GET /api/node/logging/messages' API. */ +public class LogMessagesResponse extends LoggingResponse { + @JsonProperty public LogMessageInfo info; + + // TODO Make this declaration more specific. Value on the server side is currently a + // SolrDocumentList, which cannot live in 'api' + @JsonProperty("history") + public Object docs; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/LoggingResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/LoggingResponse.java new file mode 100644 index 00000000000..1ba4be2c0d3 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/LoggingResponse.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** Generic logging response that includes the name of the log watcher (e.g. "Log4j2") */ +public class LoggingResponse extends SolrJerseyResponse { + @JsonProperty("watcher") + public String watcherName; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/ReplicationBackupRequestBody.java b/solr/api/src/java/org/apache/solr/client/api/model/ReplicationBackupRequestBody.java new file mode 100644 index 00000000000..65b93a1d829 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/ReplicationBackupRequestBody.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; + +public class ReplicationBackupRequestBody { + + public ReplicationBackupRequestBody() {} + + public ReplicationBackupRequestBody( + String location, String name, int numberToKeep, String repository, String commitName) { + this.location = location; + this.name = name; + this.numberToKeep = numberToKeep; + this.repository = repository; + this.commitName = commitName; + } + + @Schema(description = "The path where the backup will be created") + @JsonProperty + public String location; + + @Schema(description = "The backup will be created in a directory called snapshot.") + @JsonProperty + public String name; + + @Schema(description = "The number of backups to keep.") + @JsonProperty + public int numberToKeep; + + @Schema(description = "The name of the repository to be used for e backup.") + @JsonProperty + public String repository; + + @Schema( + description = + "The name of the commit which was used while taking a snapshot using the CREATESNAPSHOT command.") + @JsonProperty + public String commitName; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/ReplicationBackupResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/ReplicationBackupResponse.java new file mode 100644 index 00000000000..15581fa734e --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/ReplicationBackupResponse.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** Response for the v2 "replication backup" API */ +public class ReplicationBackupResponse extends SolrJerseyResponse { + + @JsonProperty("result") + public Object result; + + @JsonProperty("status") + public String status; + + @JsonProperty("message") + public String message; + + @JsonProperty("exception") + public Exception exception; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/RestoreCollectionRequestBody.java b/solr/api/src/java/org/apache/solr/client/api/model/RestoreCollectionRequestBody.java new file mode 100644 index 00000000000..9a592d41e3b --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/RestoreCollectionRequestBody.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; + +/** Request body for the v2 "restore collection" API. */ +public class RestoreCollectionRequestBody { + + @JsonProperty(required = true) + public String collection; + + @JsonProperty public String location; + @JsonProperty public String repository; + @JsonProperty public Integer backupId; + + @Schema( + description = + "Parameters to be used for any collections created by this restore. Only used if the collection specified by the 'collection' property does not exist.", + name = "createCollectionParams") + @JsonProperty("create-collection") + public CreateCollectionRequestBody createCollectionParams; + + @JsonProperty public String async; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/RoutedAliasProperties.java b/solr/api/src/java/org/apache/solr/client/api/model/RoutedAliasProperties.java new file mode 100644 index 00000000000..8355937e70b --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/RoutedAliasProperties.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = TimeRoutedAliasProperties.class, name = "time"), + @JsonSubTypes.Type(value = CategoryRoutedAliasProperties.class, name = "category") +}) +public abstract class RoutedAliasProperties { + @JsonProperty(required = true) + public String field; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/SchemaGetDynamicFieldInfoResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/SchemaGetDynamicFieldInfoResponse.java new file mode 100644 index 00000000000..626828c42cd --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/SchemaGetDynamicFieldInfoResponse.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class SchemaGetDynamicFieldInfoResponse extends SolrJerseyResponse { + + // TODO Server code sets this field as 'SimpleOrderedMap'; make this type declaration more + // specific once SOLR-12959 is completed + @JsonProperty("dynamicField") + public Object dynamicFieldInfo; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/SchemaGetFieldInfoResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/SchemaGetFieldInfoResponse.java new file mode 100644 index 00000000000..9b94dbda428 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/SchemaGetFieldInfoResponse.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class SchemaGetFieldInfoResponse extends SolrJerseyResponse { + + // TODO Server code sets this field as 'SimpleOrderedMap'; make this type declaration more + // specific once SOLR-12959 is completed + @JsonProperty("field") + public Object fieldInfo; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/SchemaGetFieldTypeInfoResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/SchemaGetFieldTypeInfoResponse.java new file mode 100644 index 00000000000..8e243cc111c --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/SchemaGetFieldTypeInfoResponse.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class SchemaGetFieldTypeInfoResponse extends SolrJerseyResponse { + + // TODO Server code sets this field as 'SimpleOrderedMap'; make this type declaration more + // specific once SOLR-12959 is completed + @JsonProperty("fieldType") + public Object fieldTypeInfo; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/SchemaListCopyFieldsResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/SchemaListCopyFieldsResponse.java new file mode 100644 index 00000000000..ca18e9fa711 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/SchemaListCopyFieldsResponse.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; + +public class SchemaListCopyFieldsResponse extends SolrJerseyResponse { + @JsonProperty("copyFields") + public List copyFields; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/SchemaListDynamicFieldsResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/SchemaListDynamicFieldsResponse.java new file mode 100644 index 00000000000..7c43baf6020 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/SchemaListDynamicFieldsResponse.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; + +public class SchemaListDynamicFieldsResponse extends SolrJerseyResponse { + @JsonProperty("dynamicFields") + public List dynamicFields; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/SchemaListFieldTypesResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/SchemaListFieldTypesResponse.java new file mode 100644 index 00000000000..1b8033352a5 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/SchemaListFieldTypesResponse.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; + +public class SchemaListFieldTypesResponse extends SolrJerseyResponse { + @JsonProperty("fieldTypes") + public List fieldTypes; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/SchemaListFieldsResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/SchemaListFieldsResponse.java new file mode 100644 index 00000000000..4ab6f737c2d --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/SchemaListFieldsResponse.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; + +public class SchemaListFieldsResponse extends SolrJerseyResponse { + @JsonProperty("fields") + public List fields; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/SetClusterPropertyRequestBody.java b/solr/api/src/java/org/apache/solr/client/api/model/SetClusterPropertyRequestBody.java new file mode 100644 index 00000000000..057f4bcb1d5 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/SetClusterPropertyRequestBody.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; + +public class SetClusterPropertyRequestBody { + @Schema(description = "The value to assign to the property.") + @JsonProperty("value") + public String value; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/SetThresholdRequestBody.java b/solr/api/src/java/org/apache/solr/client/api/model/SetThresholdRequestBody.java new file mode 100644 index 00000000000..3dd5b070798 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/SetThresholdRequestBody.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** The request body for the 'PUT /api/node/logging/messages/threshold' API. */ +public class SetThresholdRequestBody { + public SetThresholdRequestBody() {} + + public SetThresholdRequestBody(String level) { + this.level = level; + } + + @JsonProperty(required = true) + public String level; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/TimeRoutedAliasProperties.java b/solr/api/src/java/org/apache/solr/client/api/model/TimeRoutedAliasProperties.java new file mode 100644 index 00000000000..8a953cfaf94 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/TimeRoutedAliasProperties.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class TimeRoutedAliasProperties extends RoutedAliasProperties { + // Expected to be a date/time in ISO format, or 'NOW' + @JsonProperty(required = true) + public String start; + + // TODO Change this to 'timezone' or something less abbreviated + @JsonProperty("tz") + public String tz; + + @JsonProperty(required = true) + public String interval; + + @JsonProperty("maxFutureMs") + public Long maxFutureMs; + + @JsonProperty("preemptiveCreateMath") + public String preemptiveCreateMath; + + @JsonProperty("autoDeleteAge") + public String autoDeleteAge; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/ZooKeeperFileResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/ZooKeeperFileResponse.java new file mode 100644 index 00000000000..d09302fa168 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/ZooKeeperFileResponse.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class ZooKeeperFileResponse extends SolrJerseyResponse { + // TODO Should be switched over to using StreamingOutput as a part of SOLR-17562 + @JsonProperty("content") // A flag value that RawResponseWriter handles specially + public Object output; + + @JsonProperty("zkData") + public String zkData; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/ZooKeeperListChildrenResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/ZooKeeperListChildrenResponse.java new file mode 100644 index 00000000000..be7a69575dc --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/ZooKeeperListChildrenResponse.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.HashMap; +import java.util.Map; + +public class ZooKeeperListChildrenResponse extends ExperimentalResponse { + @JsonProperty("stat") + public ZooKeeperStat stat; + + // TODO Currently the list response (when child information is fetched) consists primarily of an + // object with only one key - the name of the root node - with separate objects under there for + // each child. The additional nesting under the root node doesn't serve much purpose afaict + // and should be removed. + public Map> unknownFields = new HashMap<>(); + + @JsonAnyGetter + public Map> unknownProperties() { + return unknownFields; + } + + @JsonAnySetter + public void setUnknownProperty(String field, Map value) { + unknownFields.put(field, value); + } +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/ZooKeeperStat.java b/solr/api/src/java/org/apache/solr/client/api/model/ZooKeeperStat.java new file mode 100644 index 00000000000..302b3885095 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/ZooKeeperStat.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** Represents the data returned by a ZooKeeper 'stat' call */ +public class ZooKeeperStat { + @JsonProperty("version") + public int version; + + @JsonProperty("aversion") + public int aversion; + + @JsonProperty("children") + public int children; + + @JsonProperty("ctime") + public long ctime; + + @JsonProperty("cversion") + public int cversion; + + @JsonProperty("czxid") + public long czxid; + + @JsonProperty("ephemeralOwner") + public long ephemeralOwner; + + @JsonProperty("mtime") + public long mtime; + + @JsonProperty("mzxid") + public long mzxid; + + @JsonProperty("pzxid") + public long pzxid; + + @JsonProperty("dataLength") + public int dataLength; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/util/Constants.java b/solr/api/src/java/org/apache/solr/client/api/util/Constants.java index b4ef56c2050..49e69d37fac 100644 --- a/solr/api/src/java/org/apache/solr/client/api/util/Constants.java +++ b/solr/api/src/java/org/apache/solr/client/api/util/Constants.java @@ -27,6 +27,8 @@ private Constants() { public static final String INDEX_PATH_PREFIX = "/{" + INDEX_TYPE_PATH_PARAMETER + ":cores|collections}/{" + INDEX_NAME_PATH_PARAMETER + "}"; + public static final String CORE_NAME_PATH_PARAMETER = "coreName"; + public static final String OMIT_FROM_CODEGEN_PROPERTY = "omitFromCodegen"; public static final String GENERIC_ENTITY_PROPERTY = "genericEntity"; diff --git a/solr/api/src/java/org/apache/solr/client/api/util/CoreApiParameters.java b/solr/api/src/java/org/apache/solr/client/api/util/CoreApiParameters.java new file mode 100644 index 00000000000..7151ee9eda7 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/util/CoreApiParameters.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.util; + +import static org.apache.solr.client.api.util.Constants.CORE_NAME_PATH_PARAMETER; + +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.enums.ParameterIn; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Concisely collects the parameters shared by APIs that interact with contents of a specific core. + * + *

Not to be used on APIs that apply to both cores AND collections. {@link StoreApiParameters} + * should be used in those cases. + * + *

Used primarily as a way to avoid duplicating these parameter definitions on each relevant + * interface method in {@link org.apache.solr.client.api.endpoint} + */ +@Target({ElementType.METHOD, ElementType.TYPE, ElementType.PARAMETER, ElementType.FIELD}) +@Retention(RetentionPolicy.RUNTIME) +@Parameter(name = CORE_NAME_PATH_PARAMETER, in = ParameterIn.PATH) +public @interface CoreApiParameters {} diff --git a/solr/api/src/java/org/apache/solr/client/api/util/StoreApiParameters.java b/solr/api/src/java/org/apache/solr/client/api/util/StoreApiParameters.java index b82f4cd107a..1d9a66bfa9c 100644 --- a/solr/api/src/java/org/apache/solr/client/api/util/StoreApiParameters.java +++ b/solr/api/src/java/org/apache/solr/client/api/util/StoreApiParameters.java @@ -29,8 +29,10 @@ import org.apache.solr.client.api.model.IndexType; /** - * Concisely collects the parameters shared by APIs that interact with contents of a specific - * core/collection. + * Concisely collects the parameters shared by APIs that interact with contents of a specific core + * OR collection. + * + *

Not to be used on APIs that are only available on cores or only on collections. * *

Used primarily as a way to avoid duplicating these parameter definitions on each relevant * interface method in {@link org.apache.solr.client.api.endpoint} diff --git a/solr/benchmark/README.md b/solr/benchmark/README.md index 7075ef111a7..9b1b8cdf623 100644 --- a/solr/benchmark/README.md +++ b/solr/benchmark/README.md @@ -1,356 +1,418 @@ -JMH-Benchmarks module -===================== +# Solr JMH Benchmark Module -This module contains benchmarks written using [JMH](https://openjdk.java.net/projects/code-tools/jmh/) from OpenJDK. -Writing correct micro-benchmarks in Java (or another JVM language) is difficult and there are many non-obvious -pitfalls (many due to compiler optimizations). JMH is a framework for running and analyzing benchmarks (micro or macro) -written in Java (or another JVM language). +![](https://user-images.githubusercontent.com/448788/140059718-de183e23-414e-4499-883a-34ec3cfbd2b6.png) -* [JMH-Benchmarks module](#jmh-benchmarks-module) - * [Running benchmarks](#running-benchmarks) - * [Using JMH with async profiler](#using-jmh-with-async-profiler) - * [Using JMH GC profiler](#using-jmh-gc-profiler) - * [Using JMH Java Flight Recorder profiler](#using-jmh-java-flight-recorder-profiler) - * [JMH Options](#jmh-options) - * [Writing benchmarks](#writing-benchmarks) - * [SolrCloud MiniCluster Benchmark Setup](#solrcloud-minicluster-benchmark-setup) - * [MiniCluster Metrics](#minicluster-metrics) - * [Benchmark Repeatability](#benchmark-repeatability) +**_`profile, compare and introspect`_** -## Running benchmarks +**A flexible, developer-friendly, microbenchmark framework** -If you want to set specific JMH flags or only run certain benchmarks, passing arguments via gradle tasks is cumbersome. -The process has been simplified by the provided `jmh.sh` script. +![](https://img.shields.io/badge/developer-tool-blue) -The default behavior is to run all benchmarks: +## Table Of Content -`./jmh.sh` +- [](#) + - [Table Of Content](#table-of-content) + - [Overview](#overview) + - [Getting Started](#getting-started) + - [Running `jmh.sh` with no Arguments](#running-jmhsh-with-no-arguments) + - [Pass a regex pattern or name after the command to select the benchmark(s) to run](#pass-a-regex-pattern-or-name-after-the-command-to-select-the-benchmarks-to-run) + - [The argument `-l` will list all the available benchmarks](#the-argument--l-will-list-all-the-available-benchmarks) + - [Check which benchmarks will run by entering a pattern after the -l argument](#check-which-benchmarks-will-run-by-entering-a-pattern-after-the--l-argument) + - [Further Pattern Examples](#further-pattern-examples) + - [`jmh.sh` accepts all the standard arguments that the standard JMH main-class handles](#jmhsh-accepts-all-the-standard-arguments-that-the-standard-jmh-main-class-handles) + - [Overriding Benchmark Parameters](#overriding-benchmark-parameters) + - [Format and Write Results to Files](#format-and-write-results-to-files) + - [JMH Command-Line Arguments](#jmh-command-line-arguments) + - [The JMH Command-Line Syntax](#the-jmh-command-line-syntax) + - [The Full List of JMH Arguments](#the-full-list-of-jmh-arguments) + - [Writing JMH benchmarks](#writing-jmh-benchmarks) + - [Continued Documentation](#continued-documentation) -Pass a pattern or name after the command to select the benchmarks: +--- -`./jmh.sh CloudIndexing` +## Overview -Check which benchmarks match the provided pattern: +JMH is a Java **microbenchmark** framework from some of the developers that work on +OpenJDK. Not surprisingly, OpenJDK is where you will find JMH's home today, alongside some +other useful little Java libraries such as JOL (Java Object Layout). -`./jmh.sh -l CloudIndexing` +The significant value in JMH is that you get to stand on the shoulders of some brilliant +engineers that have done some tricky groundwork that many an ambitious Java benchmark writer +has merrily wandered past. -Run a specific test and overrides the number of forks, iterations and sets warm-up iterations to `2`: +Rather than simply providing a boilerplate framework for driving iterations and measuring +elapsed times, which JMH does happily do, the focus is on the many forces that +deceive and disorient the earnest benchmark enthusiast. -`./jmh.sh -f 2 -i 2 -wi 2 CloudIndexing` +From spinning your benchmark into all new generated source code +in an attempt to avoid falling victim to undesirable optimizations, to offering +**BlackHoles** and a solid collection of convention and cleverly thought out yet +simple boilerplate, the goal of JMH is to lift the developer off the +microbenchmark floor and at least to their knees. -Run a specific test with async and GC profilers on Linux and flame graph output: +JMH reaches out a hand to both the best and most regular among us in a solid, cautious +effort to promote the willing into the real, often-obscured world of the microbenchmark. -`./jmh.sh -prof gc -prof async:libPath=/path/to/libasyncProfiler.so\;output=flamegraph\;dir=profile-results CloudIndexing` +## Code Organization Breakdown -### Using JMH with async profiler +![](https://img.shields.io/badge/data-...move-blue) -It's good practice to check profiler output for micro-benchmarks in order to verify that they represent the expected -application behavior and measure what you expect to measure. Some example pitfalls include the use of expensive mocks or -accidental inclusion of test setup code in the benchmarked code. JMH includes -[async-profiler](https://github.com/jvm-profiling-tools/async-profiler) integration that makes this easy: +- **JMH:** microbenchmark classes and some common base code to support them. -`./jmh.sh -prof async:libPath=/path/to/libasyncProfiler.so\;dir=profile-results` +- **Random Data:** a framework for easily generating specific and repeatable random data. + +## Getting Started + +Running **JMH** is handled via the `jmh.sh` shell script. This script uses Gradle to +extract the correct classpath and configures a handful of helpful Java +command prompt arguments and system properties. For the most part, `jmh.sh` script +will pass any arguments it receives directly to JMH. You run the script +from the root benchmark module directory (i.e. `solr/benchmark`). + +### Running `jmh.sh` with no Arguments -With flame graph output: - -`./jmh.sh -prof async:libPath=/path/to/libasyncProfiler.so\;output=flamegraph\;dir=profile-results` - -Simultaneous cpu, allocation and lock profiling with async profiler 2.0 and jfr output: - -`./jmh.sh -prof async:libPath=/path/to/libasyncProfiler.so\;output=jfr\;alloc\;lock\;dir=profile-results CloudIndexing` - -A number of arguments can be passed to configure async profiler, run the following for a description: - -`./jmh.sh -prof async:help` - -You can also skip specifying libPath if you place the async profiler lib in a predefined location, such as one of the -locations in the env variable `LD_LIBRARY_PATH` if it has been set (many Linux distributions set this env variable, Arch -by default does not), or `/usr/lib` should work. - -#### OS Permissions for Async Profiler - -Async Profiler uses perf to profile native code in addition to Java code. It will need the following for the necessary -access. - -```bash -echo 0 > /proc/sys/kernel/kptr_restrict -echo 1 > /proc/sys/kernel/perf_event_paranoid -``` - -or - -```bash -sudo sysctl -w kernel.kptr_restrict=0 -sudo sysctl -w kernel.perf_event_paranoid=1 -``` - -### Using JMH GC profiler - -You can run a benchmark with `-prof gc` to measure its allocation rate: - -`./jmh.sh -prof gc:dir=profile-results` - -Of particular importance is the `norm` alloc rates, which measure the allocations per operation rather than allocations -per second. - -### Using JMH Java Flight Recorder profiler - -JMH comes with a variety of built-in profilers. Here is an example of using JFR: - -`./jmh.sh -prof jfr:dir=profile-results\;configName=jfr-profile.jfc` - -In this example we point to the included configuration file with configName, but you could also do something like -settings=default or settings=profile. - -### Benchmark Outputs - -By default, output that benchmarks generate is created in the build/work directory. You can change this location by setting the workBaseDir system property like this: - - -jvmArgsAppend -DworkBaseDir=/data3/bench_work - -If a profiler generates output, it will generally be written to the current working directory - that is the benchmark module directory itself. You can usually change this via the dir option, for example: - - ./jmh.sh -prof jfr:dir=build/work/profile-results JsonFaceting - -### Using a Separate MiniCluster Base Directory - -If you have a special case MiniCluster you have generated, such as one you have prepared with very large indexes for a search benchmark run, you can change the base directory used by the profiler -for the MiniCluster with the miniClusterBaseDir system property. This is for search based benchmarks in general and the MiniCluster wil not be removed automatically by the benchmark. - -### JMH Options - -Some common JMH options are: - -```text +> +> ```zsh +> # run all benchmarks found in subdirectories +> ./jmh.sh +> ``` + +### Pass a regex pattern or name after the command to select the benchmark(s) to run + +> +> ```zsh +> ./jmh.sh BenchmarkClass +> ``` + +### The argument `-l` will list all the available benchmarks + +> +> ```zsh +> ./jmh.sh -l +> ``` + +### Check which benchmarks will run by entering a pattern after the -l argument + +Use the full benchmark class name, the simple class name, the benchmark +method name, or a substring. + +> +> ```zsh +> ./jmh.sh -l Ben +> ``` + +### Further Pattern Examples + +> +> ```shell +>./jmh.sh -l org.apache.solr.benchmark.search.BenchmarkClass +>./jmh.sh -l BenchmarkClass +>./jmh.sh -l BenchmarkClass.benchmethod +>./jmh.sh -l Bench +>./jmh.sh -l benchme + +### The JMH Script Accepts _ALL_ of the Standard JMH Arguments + +Here we tell JMH to run the trial iterations twice, forking a new JVM for each +trial. We also explicitly set the number of warmup iterations and the +measured iterations to 2. + +> +> ```zsh +> ./jmh.sh -f 2 -wi 2 -i 2 BenchmarkClass +> ``` + +### Overriding Benchmark Parameters + +> ![](https://img.shields.io/badge/overridable-params-blue) +> +> ```java +> @Param("1000") +> private int numDocs; +> ``` + +The state objects that can be specified in benchmark classes will often have a +number of input parameters that benchmark method calls will access. The notation +above will default numDocs to 1000 and also allow you to override that value +using the `-p` argument. A benchmark might also use a @Param annotation such as: + +> ![](https://img.shields.io/badge/sequenced-params-blue) +> +> ```java +> @Param("1000","5000","1000") +> private int numDocs; +> ``` + +By default, that would cause the benchmark +to be run enough times to use each of the specified values. If multiple input +parameters are specified this way, the number of runs needed will quickly +expand. You can pass multiple `-p` +arguments and each will completely replace the behavior of any default +annotation values. + +> +> ```zsh +> # use 2000 docs instead of 1000 +> ./jmh.sh BenchmarkClass -p numDocs=2000 +> +> +> # use 5 docs, then 50, then 500 +> ./jmh.sh BenchmarkClass -p numDocs=5,50,500 +> +> +> # run the benchmark enough times to satisfy every combination of two +> # multi-valued input parameters +> ./jmh.sh BenchmarkClass -p numDocs=10,20,30 -p docSize 250,500 +> ``` + +### Format and Write Results to Files + +Rather than just dumping benchmark results to the console, you can specify the +`-rf` argument to control the output format; for example, you can choose CSV or +JSON. The `-rff` argument will dictate the filename and output location. + +> +> ```zsh +> # format output to JSON and write the file to the `work` directory relative to +> # the JMH working directory. +> ./jmh.sh BenchmarkClass -rf json -rff work/jmh-results.json +> ``` +> +> 💡 **If you pass only the `-rf` argument, JMH will write out a file to the +> current working directory with the appropriate extension, e.g.,** `jmh-results.csv`. + +## JMH Command-Line Arguments + +### The JMH Command-Line Syntax + +> ![](https://img.shields.io/badge/Help-output-blue) +> +> ```zsh +> Usage: ./jmh.sh [regexp*] [options] +> [opt] means optional argument. +> means required argument. +> "+" means comma-separated list of values. +> "time" arguments accept time suffixes, like "100ms". +> +> Command-line options usually take precedence over annotations. +> ``` + +### The Full List of JMH Arguments + +```zsh Usage: ./jmh.sh [regexp*] [options] [opt] means optional argument. means required argument. - "+" means comma-separated list of values. + "+" means a comma-separated list of values. "time" arguments accept time suffixes, like "100ms". -Command line options usually take precedence over annotations. +Command-line options usually take precedence over annotations. [arguments] Benchmarks to run (regexp+). (default: .*) - -bm Benchmark mode. Available modes are: [Throughput/thrpt, - AverageTime/avgt, SampleTime/sample, SingleShotTime/ss, + -bm Benchmark mode. Available modes are: + [Throughput/thrpt, AverageTime/avgt, + SampleTime/sample, SingleShotTime/ss, All/all]. (default: Throughput) -bs Batch size: number of benchmark method calls per operation. Some benchmark modes may ignore this - setting, please check this separately. (default: - 1) + setting; please check this separately. + (default: 1) -e Benchmarks to exclude from the run. - -f How many times to fork a single benchmark. Use 0 to - disable forking altogether. Warning: disabling - forking may have detrimental impact on benchmark - and infrastructure reliability, you might want - to use different warmup mode instead. (default: - 5) - - -foe Should JMH fail immediately if any benchmark had - experienced an unrecoverable error? This helps - to make quick sanity tests for benchmark suites, - as well as make the automated runs with checking error + -f How many times to fork a single benchmark. Use 0 + to disable forking altogether. Warning: + disabling forking may have a detrimental impact on + benchmark and infrastructure reliability. You might + want to use a different warmup mode instead. (default: 1) + + -foe Should JMH fail immediately if any benchmark has + experienced an unrecoverable error? Failing fast + helps to make quick sanity tests for benchmark + suites and allows automated runs to do error + checking. codes. (default: false) -gc Should JMH force GC between iterations? Forcing - the GC may help to lower the noise in GC-heavy benchmarks, - at the expense of jeopardizing GC ergonomics decisions. + GC may help lower the noise in GC-heavy benchmarks + at the expense of jeopardizing GC ergonomics + decisions. Use with care. (default: false) - -h Display help, and exit. + -h Displays this help output and exits. - -i Number of measurement iterations to do. Measurement - iterations are counted towards the benchmark score. - (default: 1 for SingleShotTime, and 5 for all other - modes) + -i Number of measurement iterations to do. + Measurement + iterations are counted towards the benchmark + score. + (default: 1 for SingleShotTime, and 5 for all + other modes) - -jvm Use given JVM for runs. This option only affects forked - runs. + -jvm Use given JVM for runs. This option only affects + forked runs. - -jvmArgs Use given JVM arguments. Most options are inherited - from the host VM options, but in some cases you want - to pass the options only to a forked VM. Either single - space-separated option line, or multiple options - are accepted. This option only affects forked runs. + -jvmArgs Use given JVM arguments. Most options are + inherited from the host VM options, but in some + cases, you want to pass the options only to a forked + VM. Either single space-separated option line or + multiple options are accepted. This option only + affects forked runs. - -jvmArgsAppend Same as jvmArgs, but append these options after the - already given JVM args. + -jvmArgsAppend Same as jvmArgs, but append these options after + the already given JVM args. -jvmArgsPrepend Same as jvmArgs, but prepend these options before the already given JVM arg. - -l List the benchmarks that match a filter, and exit. + -l List the benchmarks that match a filter and exit. - -lp List the benchmarks that match a filter, along with + -lp List the benchmarks that match a filter, along + with parameters, and exit. - -lprof List profilers, and exit. + -lprof List profilers and exit. - -lrf List machine-readable result formats, and exit. + -lrf List machine-readable result formats and exit. -o Redirect human-readable output to a given file. - -opi Override operations per invocation, see @OperationsPerInvocation - Javadoc for details. (default: 1) + -opi Override operations per invocation, see + @OperationsPerInvocation Javadoc for details. + (default: 1) - -p Benchmark parameters. This option is expected to - be used once per parameter. Parameter name and parameter - values should be separated with equals sign. Parameter - values should be separated with commas. + -p Benchmark parameters. This option is expected to + be used once per parameter. The parameter name and + parameter values should be separated with an + equal sign. Parameter values should be separated + with commas. - -prof Use profilers to collect additional benchmark data. - Some profilers are not available on all JVMs and/or - all OSes. Please see the list of available profilers - with -lprof. + -prof Use profilers to collect additional benchmark + data. + Some profilers are not available on all JVMs or + all OSes. '-lprof' will list the available + profilers that are available and that can run + with the current OS configuration and installed dependencies. - -r

+ +![](https://user-images.githubusercontent.com/448788/137610116-eff6d0b7-e862-40fb-af04-452aaf585387.png) + +```Shell +./jmh.sh -lprof` +``` + +
+ + +
+ +In our case, we will start with very **minimal** Arch and Ubuntu clean installations, and so we already know there is _**no chance**_ that async-profiler or Perfasm +are going to run. + +In fact, first we have to install a few project build requirements before thinking too much about JMH profiler support. + +We will run on **Arch/Manjaro**, but there should not be any difference than on **Debian/Ubuntu** for this stage. + +
+ +![](https://user-images.githubusercontent.com/448788/137610116-eff6d0b7-e862-40fb-af04-452aaf585387.png) + +```Shell +sudo pacman -S wget jdk-openjdk11 +``` + +
+ +
+ +Here we give **async-profiler** a try on **Arch** anyway and observe the failure indicating that we need to obtain the async-profiler library and +put it in the correct location at a minimum. + +
+ + +```Shell +./jmh.sh BenchMark -prof async +``` + +
+     Profilers failed to initialize, exiting.
+
+    Unable to load async-profiler. Ensure asyncProfiler library is on LD_LIBRARY_PATH (Linux)
+    DYLD_LIBRARY_PATH (Mac OS), or -Djava.library.path.
+
+    Alternatively, point to explicit library location with: '-prof async:libPath={path}'
+
+    no asyncProfiler in java.library.path: [/usr/java/packages/lib, /usr/lib64, /lib64, /lib, /usr/lib]
+    
+ +
+ +### Async-Profiler + +#### Install async-profiler + +
+ +![](https://user-images.githubusercontent.com/448788/137610116-eff6d0b7-e862-40fb-af04-452aaf585387.png) + +```Shell +wget -c https://github.com/jvm-profiling-tools/async-profiler/releases/download/v2.5/async-profiler-2.5-linux-x64.tar.gz -O - | tar -xz +sudo mkdir -p /usr/java/packages/lib +sudo cp async-profiler-2.5-linux-x64/build/* /usr/java/packages/lib +``` + +
+ +
+ +That should work out better, but there is still an issue that will prevent a successful profiling run. async-profiler relies on Linux's perf, +and in any recent Linux kernel, perf is restricted from doing its job without some configuration loosening. + +Manjaro should have perf available, but you may need to install it in the other cases. + +
+ +![](https://user-images.githubusercontent.com/448788/137563908-738a7431-88db-47b0-96a4-baaed7e5024b.png) + +
+ +![](https://user-images.githubusercontent.com/448788/137610566-883825b7-e66c-4d8b-a6a5-61542bc08d23.png) + +```Shell +apt-get install linux-tools-common linux-tools-generic linux-tools-`uname -r` +``` + +
+ +
+ +![](https://user-images.githubusercontent.com/448788/137563725-0195a732-da40-4c8b-a5e8-fd904a43bb79.png) + +
+ +![](https://user-images.githubusercontent.com/448788/137610566-883825b7-e66c-4d8b-a6a5-61542bc08d23.png) + +```Shell +pacman -S perf +``` + +
+ + +
+ +And now the permissions issue. The following changes will persist across restarts, and that is likely how you should leave things. + +
+ +```zsh +sudo sysctl -w kernel.kptr_restrict=0 +sudo sysctl -w kernel.perf_event_paranoid=1 +``` + +
+ +
+ +Now we **should** see some success: + +
+ +![](https://user-images.githubusercontent.com/448788/137610116-eff6d0b7-e862-40fb-af04-452aaf585387.png) + +```Shell +./jmh.sh FuzzyQuery -prof async:output=flamegraph +``` + +
+ +
+ +![](https://user-images.githubusercontent.com/448788/138650315-82adeb18-54cd-43ee-810e-24f1e22719c7.png) + +
+ +But you will also find an important _warning_ if you look closely at the logs. + +
+ +![](https://user-images.githubusercontent.com/448788/137613526-a188ff03-545c-465d-928d-bc433d2d204f.png) +[WARN] `Install JVM debug symbols to improve profile accuracy` + +
+ +Ensuring that **Debug symbols** remain available provides the best experience for optimal profiling accuracy and heap-analysis. + +And it also turns out that if we use async-profiler's **alloc** option to sample and create flamegraphs for heap usage, the **debug** symbols +are _required_. + +
+ +#### Install Java Debug Symbols + +--- + +##### Ubuntu + +![](https://user-images.githubusercontent.com/448788/137563908-738a7431-88db-47b0-96a4-baaed7e5024b.png) + +Grab the debug package of OpenJdk using your package manager for the correct Java version. + +
+ +![](https://user-images.githubusercontent.com/448788/137610566-883825b7-e66c-4d8b-a6a5-61542bc08d23.png) + +```Shell +sudo apt update +sudo apt upgrade +sudo apt install openjdk-11-dbg +``` + +
+ +--- + +##### Arch + +![](https://user-images.githubusercontent.com/448788/137563725-0195a732-da40-4c8b-a5e8-fd904a43bb79.png) + +On the **Arch** side we will rebuild the Java 11 package, but turn off the option that strips debug symbols. Often, large OS package and Java repositories originated in SVN and can be a bit a of a bear to wrestle with git about for just a fraction +of the repository, we do so GitHub API workaround efficiency. + +
+ +![](https://user-images.githubusercontent.com/448788/137610116-eff6d0b7-e862-40fb-af04-452aaf585387.png) + +```Shell +sudo pacman -S dkms base-devel linux-headers dkms git vi jq --needed --noconfirm + +curl -sL "https://api.github.com/repos/archlinux/svntogit-packages/contents/java11-openjdk/repos/extra-x86_64" \ +| jq -r '.[] | .download_url' | xargs -n1 wget +``` + +
+ +
+ +Now we need to change that option in PKGBUILD. Choose your favorite editor. (nano, vim, emacs, ne, nvim, tilde etc) + +
+ +![](https://user-images.githubusercontent.com/448788/137610116-eff6d0b7-e862-40fb-af04-452aaf585387.png) + +```Shell +vi PKGBUILD +``` + +
+ +
+ +Insert a single option line: + +
+ +```Diff +arch=('x86_64') +url='https://openjdk.java.net/' +license=('custom') ++ options=('debug' '!strip') +makedepends=('java-environment>=10' 'java-environment<12' 'cpio' 'unzip' 'zip' 'libelf' 'libcups' 'libx11' 'libxrender' 'libxtst' 'libxt' 'libxext' 'libxrandr' 'alsa-lib' 'pandoc' +``` + +
+ +
+ +Then build and install. (`-s: --syncdeps -i: --install -f: --force`) + +
+ +![](https://user-images.githubusercontent.com/448788/137610116-eff6d0b7-e862-40fb-af04-452aaf585387.png) + +```Shell +makepkg -sif +``` + +
+ +
+ +When that is done, if everything went well, we should be able to successfully run async-profiler in alloc mode to generate a flame graph based on memory rather than cpu. + +
+ + +
+ +![](https://user-images.githubusercontent.com/448788/137610116-eff6d0b7-e862-40fb-af04-452aaf585387.png) + +```Shell +./jmh.sh FuzzyQuery -prof async:output=flamegraph +``` + +
+ +
+ +![](https://user-images.githubusercontent.com/448788/138661737-333bf265-343a-4002-b8a8-97d72c38ced0.png) + +## Perfasm + +Perfasm will run perf to collect hardware counter infromation (cycles by default) and it will also pass an argument to Java +to cause it to log assembly output (amoung other things). The performance data from perf is married with the assembly from the Java output log and Perfasm then does its thing to produce human parsable output. Java generally cannot output assembly as shipped however, so now +we must install **hsdis** to allow for `-XX+PrintAssembly` + +* * * + +### Arch + +![](https://user-images.githubusercontent.com/448788/137563725-0195a732-da40-4c8b-a5e8-fd904a43bb79.png) + +
+ +[//]: # ( https://aur.archlinux.org/packages/java11-openjdk-hsdis/) + +If you have `yay` or another **AUR** helper available, or if you have the **AUR** enabled in your package manager, simply install `java11-openjdk-hdis` + +If you do not have simple access to **AUR**, set it up or just grab the package manually: + +
+ +![](https://user-images.githubusercontent.com/448788/137610116-eff6d0b7-e862-40fb-af04-452aaf585387.png) + +```Shell +wget -c https://aur.archlinux.org/cgit/aur.git/snapshot/java11-openjdk-hsdis.tar.gz -O - | tar -xz +cd java11-openjdk-hsdis/ +makepkg -si +``` + +
+ +
+ +--- + +### Ubuntu + +![](https://user-images.githubusercontent.com/448788/137563908-738a7431-88db-47b0-96a4-baaed7e5024b.png) + +
+ +
+ +![](https://user-images.githubusercontent.com/448788/137610566-883825b7-e66c-4d8b-a6a5-61542bc08d23.png) + +```Shell +sudo apt update +sudo apt -y upgrade +sudo apt -y install openjdk-11-jdk git wget jq +``` + +
+ +
+ +
+ +![](https://user-images.githubusercontent.com/448788/137610566-883825b7-e66c-4d8b-a6a5-61542bc08d23.png) + +```Shell +curl -sL "https://api.github.com/repos/openjdk/jdk11/contents/src/utils/hsdis" | jq -r '.[] | .download_url' | xargs -n1 wget + +# Newer versions of binutils don't appear to compile, must use 2.28 for JDK 11 +wget http://ftp.heanet.ie/mirrors/ftp.gnu.org/gnu/binutils/binutils-2.28.tar.gz +tar xzvf binutils-2.28.tar.gz +make BINUTILS=binutils-2.28 ARCH=amd64 +``` + +
+ +
+ +Now we should be able to do a little Perfasm: + +
+ +![](https://user-images.githubusercontent.com/448788/137610116-eff6d0b7-e862-40fb-af04-452aaf585387.png) + +```Shell +./jmh.sh FuzzyQuery -prof perfasm +``` + +
diff --git a/solr/benchmark/docs/jmh-profilers.md b/solr/benchmark/docs/jmh-profilers.md new file mode 100644 index 00000000000..e700add93ae --- /dev/null +++ b/solr/benchmark/docs/jmh-profilers.md @@ -0,0 +1,189 @@ + + +# JMH Profilers + +- [JMH Profilers](#jmh-profilers) + - [Introduction](#introduction) + - [Using JMH Profilers](#using-jmh-profilers) + - [Using JMH with the Async-Profiler](#using-jmh-with-the-async-profiler) + - [OS Permissions for Async-Profiler](#os-permissions-for-async-profiler) + - [Using JMH with the GC Profiler](#using-jmh-with-the-gc-profiler) + - [Using JMH with the Java Flight Recorder Profiler](#using-jmh-with-the-java-flight-recorder-profiler) + +## Introduction + +Some may think that the appeal of a micro-benchmark is in the relatively easy +learning curve and the often isolated nature of what is being measured. But +this perspective is actually what can often make them dangerous. Benchmarking +can be easy to approach from a non-rigorous, casual angle that results in the +feeling that they are a relatively straightforward part of the developer's +purview. From this viewpoint, microbenchmarks can appear downright easy. But good +benchmarking is hard. Microbenchmarks are very hard. Java and HotSpot make "hard" +even harder. + +JMH was developed by engineers that understood the dark side of benchmarks very +well. They also work on OpenJDK, so they are abnormally suited to building a +java microbenchmark framework that tackles many common issues that naive +approaches and go-it-alone efforts are likely to trip on. Even still, they will +tell you, JMH is a sharp blade. Best to be cautious and careful when swinging it +around. + +The good folks working on JMH did not just build a better than average java +micro-benchmark framework and then leave us to the still many wolves, though. They +also built-in first-class support for the essential tools that the +ambitious developer absolutely needs for defense when bravely trying to +understand performance. This brings us to the JMH profiler options. + +## Using JMH Profilers + +### Using JMH with the Async-Profiler + +It's good practice to check profiler output for micro-benchmarks in order to +verify that they represent the expected application behavior and measure what +you expect to measure. Some example pitfalls include the use of expensive mocks +or accidental inclusion of test setup code in the benchmarked code. JMH includes +[async-profiler](https://github.com/jvm-profiling-tools/async-profiler) +integration that makes this easy: + +
+ +![](https://user-images.githubusercontent.com/448788/137610566-883825b7-e66c-4d8b-a6a5-61542bc08d23.png) + +```Shell +./jmh.sh -prof async:libPath=/path/to/libasyncProfiler.so\;dir=profile-results +``` + +
+ +Run a specific test with async and GC profilers on Linux and flame graph output. + +
+ +![](https://user-images.githubusercontent.com/448788/137610566-883825b7-e66c-4d8b-a6a5-61542bc08d23.png) + +```Shell + ./jmh.sh -prof gc -prof async:libPath=/path/to/libasyncProfiler.so\;output=flamegraph\;dir=profile-results BenchmarkClass +``` + +
+ +With flame graph output: + +
+ +![](https://user-images.githubusercontent.com/448788/137610566-883825b7-e66c-4d8b-a6a5-61542bc08d23.png) + +```Shell +./jmh.sh -prof async:libPath=/path/to/libasyncProfiler.so\;output=flamegraph\;dir=profile-results +``` + +
+ +Simultaneous CPU, allocation, and lock profiling with async profiler 2.0 and Java Flight Recorder +output: + +
+ +![](https://user-images.githubusercontent.com/448788/137610566-883825b7-e66c-4d8b-a6a5-61542bc08d23.png) + +```Shell +./jmh.sh -prof async:libPath=/path/to/libasyncProfiler.so\;output=jfr\;alloc\;lock\;dir=profile-results BenchmarkClass +``` + +
+ +A number of arguments can be passed to configure async profiler, run the +following for a description: + +
+ +![](https://user-images.githubusercontent.com/448788/137610566-883825b7-e66c-4d8b-a6a5-61542bc08d23.png) + +```Shell +./jmh.sh -prof async:help +``` + +
+ +You can also skip specifying libPath if you place the async profiler lib in a +predefined location, such as one of the locations in the env +variable `LD_LIBRARY_PATH` if it has been set (many Linux distributions set this +env variable, Arch by default does not), or `/usr/lib` should work. + +#### OS Permissions for Async-Profiler + +Async Profiler uses perf to profile native code in addition to Java code. It +will need the following for the necessary access. + +
+ +![](https://user-images.githubusercontent.com/448788/137610566-883825b7-e66c-4d8b-a6a5-61542bc08d23.png) + +```Shell +echo 0 > /proc/sys/kernel/kptr_restrict +echo 1 > /proc/sys/kernel/perf_event_paranoid +``` + +
+ +or + +
+ +![](https://user-images.githubusercontent.com/448788/137610566-883825b7-e66c-4d8b-a6a5-61542bc08d23.png) + +```Shell +sudo sysctl -w kernel.kptr_restrict=0 +sudo sysctl -w kernel.perf_event_paranoid=1 +``` + +
+ +### Using JMH with the GC Profiler + +You can run a benchmark with `-prof gc` to measure its allocation rate: + +
+ +![](https://user-images.githubusercontent.com/448788/137610566-883825b7-e66c-4d8b-a6a5-61542bc08d23.png) + +```Shell +./jmh.sh -prof gc:dir=profile-results +``` + +
+ +Of particular importance is the `norm` alloc rates, which measure the +allocations per operation rather than allocations per second. + +### Using JMH with the Java Flight Recorder Profiler + +JMH comes with a variety of built-in profilers. Here is an example of using JFR: + +
+ +![](https://user-images.githubusercontent.com/448788/137610566-883825b7-e66c-4d8b-a6a5-61542bc08d23.png) + +```Shell +./jmh.sh -prof jfr:dir=profile-results\;configName=jfr-profile.jfc BenchmarkClass +``` + +
+ +In this example, we point to the included configuration file with config name, but +you could also do something like settings=default or settings=profile. diff --git a/solr/benchmark/src/java/org/apache/solr/bench/MiniClusterState.java b/solr/benchmark/src/java/org/apache/solr/bench/MiniClusterState.java index 34a970b3392..39c1745d387 100755 --- a/solr/benchmark/src/java/org/apache/solr/bench/MiniClusterState.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/MiniClusterState.java @@ -267,7 +267,7 @@ public void startMiniCluster(int nodeCount) { cluster = new MiniSolrCloudCluster.Builder(nodeCount, miniClusterBaseDir) .formatZkServer(false) - .addConfig("conf", getFile("src/resources/configs/cloud-minimal/conf").toPath()) + .addConfig("conf", getFile("src/resources/configs/cloud-minimal/conf")) .configure(); } catch (Exception e) { if (Files.exists(miniClusterBaseDir)) { @@ -551,12 +551,12 @@ public static ModifiableSolrParams params(ModifiableSolrParams params, String... * @param name the name * @return the file */ - public static File getFile(String name) { + public static Path getFile(String name) { final URL url = MiniClusterState.class.getClassLoader().getResource(name.replace(File.separatorChar, '/')); if (url != null) { try { - return new File(url.toURI()); + return Path.of(url.toURI()); } catch (Exception e) { throw new RuntimeException( "Resource was found on classpath, but cannot be resolved to a " @@ -564,12 +564,12 @@ public static File getFile(String name) { + name); } } - File file = new File(name); - if (file.exists()) { + Path file = Path.of(name); + if (Files.exists(file)) { return file; } else { - file = new File("../../../", name); - if (file.exists()) { + file = Path.of("../../../", name); + if (Files.exists(file)) { return file; } } diff --git a/solr/benchmark/src/java/org/apache/solr/bench/generators/RandomDataHistogram.java b/solr/benchmark/src/java/org/apache/solr/bench/generators/RandomDataHistogram.java index 0f9062331ef..8cd55a7964a 100644 --- a/solr/benchmark/src/java/org/apache/solr/bench/generators/RandomDataHistogram.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/generators/RandomDataHistogram.java @@ -436,8 +436,7 @@ public String toString() { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof Surrogate)) return false; - Surrogate surrogate = (Surrogate) o; + if (!(o instanceof Surrogate surrogate)) return false; return hashCode.equals(surrogate.hashCode) && identityHashcode.equals(surrogate.identityHashcode); } diff --git a/solr/bin/solr b/solr/bin/solr index 38d7ed10d82..73b5b516338 100755 --- a/solr/bin/solr +++ b/solr/bin/solr @@ -318,13 +318,13 @@ fi if [ -z "${SOLR_AUTH_TYPE:-}" ] && [ -n "${SOLR_AUTHENTICATION_OPTS:-}" ]; then echo "WARNING: SOLR_AUTHENTICATION_OPTS environment variable configured without associated SOLR_AUTH_TYPE variable" echo " Please configure SOLR_AUTH_TYPE environment variable with the authentication type to be used." - echo " Currently supported authentication types are [kerberos, basic]" + echo " Currently supported authentication types are [basic]" fi if [ -n "${SOLR_AUTH_TYPE:-}" ] && [ -n "${SOLR_AUTHENTICATION_CLIENT_BUILDER:-}" ]; then echo "WARNING: SOLR_AUTHENTICATION_CLIENT_BUILDER and SOLR_AUTH_TYPE environment variables are configured together." echo " Use SOLR_AUTH_TYPE environment variable to configure authentication type to be used. " - echo " Currently supported authentication types are [kerberos, basic]" + echo " Currently supported authentication types are [basic]" echo " The value of SOLR_AUTHENTICATION_CLIENT_BUILDER environment variable will be ignored" fi @@ -333,9 +333,6 @@ if [ -n "${SOLR_AUTH_TYPE:-}" ]; then basic) SOLR_AUTHENTICATION_CLIENT_BUILDER="org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory" ;; - kerberos) - SOLR_AUTHENTICATION_CLIENT_BUILDER="org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder" - ;; *) echo "ERROR: Value specified for SOLR_AUTH_TYPE environment variable is invalid." exit 1 diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd index baa3ba47957..b153b4bb086 100755 --- a/solr/bin/solr.cmd +++ b/solr/bin/solr.cmd @@ -54,7 +54,11 @@ IF NOT DEFINED JAVA_HOME ( set "JAVA_HOME=%%B" ) ) -IF NOT DEFINED JAVA_HOME goto need_java_home +IF NOT DEFINED JAVA_HOME ( + REM Need java home + @echo "Please set the JAVA_HOME environment variable to the path where you installed Java !REQUIRED_JAVA_VERSION!+" + goto done +) set JAVA_HOME=%JAVA_HOME:"=% IF %JAVA_HOME:~-1%==\ SET JAVA_HOME=%JAVA_HOME:~0,-1% IF NOT EXIST "%JAVA_HOME%\bin\java.exe" ( @@ -199,7 +203,7 @@ IF NOT DEFINED SOLR_AUTH_TYPE ( IF DEFINED SOLR_AUTHENTICATION_OPTS ( echo WARNING: SOLR_AUTHENTICATION_OPTS variable configured without associated SOLR_AUTH_TYPE variable echo Please configure SOLR_AUTH_TYPE variable with the authentication type to be used. - echo Currently supported authentication types are [kerberos, basic] + echo Currently supported authentication types are [basic] ) ) @@ -207,7 +211,7 @@ IF DEFINED SOLR_AUTH_TYPE ( IF DEFINED SOLR_AUTHENTICATION_CLIENT_BUILDER ( echo WARNING: SOLR_AUTHENTICATION_CLIENT_BUILDER and SOLR_AUTH_TYPE variables are configured together echo Use SOLR_AUTH_TYPE variable to configure authentication type to be used - echo Currently supported authentication types are [kerberos, basic] + echo Currently supported authentication types are [basic] echo The value of SOLR_AUTHENTICATION_CLIENT_BUILDER configuration variable will be ignored ) ) @@ -216,12 +220,8 @@ IF DEFINED SOLR_AUTH_TYPE ( IF /I "%SOLR_AUTH_TYPE%" == "basic" ( set SOLR_AUTHENTICATION_CLIENT_BUILDER="org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory" ) ELSE ( - IF /I "%SOLR_AUTH_TYPE%" == "kerberos" ( - set SOLR_AUTHENTICATION_CLIENT_BUILDER="org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory" - ) ELSE ( - echo ERROR: Value specified for SOLR_AUTH_TYPE configuration variable is invalid. - goto err - ) + echo ERROR: Value specified for SOLR_AUTH_TYPE configuration variable is invalid. + goto err ) ) @@ -244,59 +244,36 @@ IF "%SOLR_JETTY_HOST%"=="" ( set "SOLR_JETTY_HOST=127.0.0.1" ) -set FIRST_ARG=%1 - -IF [%1]==[] goto usage -IF "%1"=="-h" goto run_solrcli -IF "%1"=="--help" goto run_solrcli -IF "%1"=="status" goto run_solrcli -IF "%1"=="version" goto run_solrcli -IF "%1"=="-v" goto run_solrcli -IF "%1"=="--version" goto run_solrcli -IF "%1"=="assert" goto run_solrcli -IF "%1"=="zk" goto run_solrcli -IF "%1"=="export" goto run_solrcli -IF "%1"=="package" goto run_solrcli -IF "%1"=="api" goto run_solrcli -IF "%1"=="post" goto run_solrcli - -REM Only allow the command to be the first argument, assume start if not supplied +REM Handle special commands IF "%1"=="start" goto set_script_cmd IF "%1"=="stop" goto set_script_cmd IF "%1"=="restart" goto set_script_cmd -IF "%1"=="healthcheck" goto run_solrcli -IF "%1"=="create" goto run_solrcli -IF "%1"=="delete" goto run_solrcli -IF "%1"=="postlogs" goto run_solrcli +IF "%1"=="auth" goto set_script_cmd -IF "%1"=="auth" ( - set SCRIPT_CMD=auth - SHIFT - goto run_auth +REM Handle all other commands by simply running SolrCLI +"%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% %SOLR_TOOL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" ^ + -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ + -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ + org.apache.solr.cli.SolrCLI %* +if errorlevel 1 ( + exit /b 1 ) -IF "%1"=="config" goto run_solrcli +goto done +:set_script_cmd +set SCRIPT_CMD=%1 +SHIFT +IF "%SCRIPT_CMD%"=="auth" goto run_auth goto parse_args :usage IF NOT "%SCRIPT_ERROR%"=="" ECHO %SCRIPT_ERROR% -IF [%FIRST_ARG%]==[] goto run_solrcli -IF "%FIRST_ARG%"=="-h" goto run_solrcli -IF "%FIRST_ARG%"=="--help" goto run_solrcli IF "%SCRIPT_CMD%"=="start" goto start_usage IF "%SCRIPT_CMD%"=="restart" goto start_usage IF "%SCRIPT_CMD%"=="stop" goto stop_usage -IF "%SCRIPT_CMD%"=="healthcheck" goto run_solrcli -IF "%SCRIPT_CMD%"=="create" goto run_solrcli -IF "%SCRIPT_CMD%"=="delete" goto run_solrcli -IF "%SCRIPT_CMD%"=="cluster" goto run_solrcli -IF "%SCRIPT_CMD%"=="zk" goto run_solrcli -IF "%SCRIPT_CMD%"=="auth" goto run_solrcli -IF "%SCRIPT_CMD%"=="package" goto run_solrcli -IF "%SCRIPT_CMD%"=="status" goto run_solrcli -IF "%SCRIPT_CMD%"=="postlogs" goto run_solrcli -goto done +REM Should not be reachable, but just in case +goto err :start_usage @echo. @@ -374,16 +351,19 @@ goto done @echo. goto done - -REM Really basic command-line arg parsing +REM Parse arguments for special commands (start, stop, restart) :parse_args set "arg=%~1" set "firstTwo=%arg:~0,2%" -IF "%SCRIPT_CMD%"=="" set SCRIPT_CMD=start -IF [%1]==[] goto process_script_cmd -IF "%1"=="--help" goto usage -IF "%1"=="-h" goto usage + +REM In case no arguments left, run special command +IF [%1]==[] goto run_special_command + +REM Skip start / restart arguments if stop command +IF "%SCRIPT_CMD%"=="stop" goto parse_stop_args + +:parse_start_args IF "%1"=="-f" goto set_foreground_mode IF "%1"=="--foreground" goto set_foreground_mode IF "%1"=="--verbose" goto set_verbose @@ -398,8 +378,6 @@ IF "%1"=="--example" goto set_example IF "%1"=="--host" goto set_host IF "%1"=="-m" goto set_memory IF "%1"=="--memory" goto set_memory -IF "%1"=="-p" goto set_port -IF "%1"=="--port" goto set_port IF "%1"=="-z" goto set_zookeeper IF "%1"=="--zk-host" goto set_zookeeper IF "%1"=="-s" goto set_solr_url @@ -407,20 +385,33 @@ IF "%1"=="--solr-url" goto set_solr_url IF "%1"=="--jvm-opts" goto set_jvm_opts IF "%1"=="-j" goto set_addl_jetty_config IF "%1"=="--jettyconfig" goto set_addl_jetty_config -IF "%1"=="--no-prompt" goto set_noprompt IF "%1"=="-y" goto set_noprompt +IF "%1"=="--no-prompt" goto set_noprompt + +REM Skip stop arg parsing if not stop command +IF NOT "%SCRIPT_CMD%"=="stop" goto parse_general_args + +:parse_stop_args IF "%1"=="-k" goto set_stop_key IF "%1"=="--key" goto set_stop_key IF "%1"=="--all" goto set_stop_all + +:parse_general_args + +REM Print usage of command in case help option included +IF "%1"=="--help" goto usage +IF "%1"=="-h" goto usage + +REM other args supported by all special commands +IF "%1"=="-p" goto set_port +IF "%1"=="--port" goto set_port IF "%firstTwo%"=="-D" goto set_passthru + +REM Argument not supported / found IF NOT "%1"=="" goto invalid_cmd_line +REM Not reachable, but just in case goto invalid_cmd_line -:set_script_cmd -set SCRIPT_CMD=%1 -SHIFT -goto parse_args - :set_foreground_mode set FG=1 SHIFT @@ -652,15 +643,36 @@ SHIFT goto parse_args :set_passthru -set "PASSTHRU=%~1=%~2" +set "PASSTHRU_KEY=%~1" +set "PASSTHRU_VALUES=" + +SHIFT +:repeat_passthru +set "arg=%~1" +if "%arg%"=="" goto end_passthru +set firstChar=%arg:~0,1% +IF "%firstChar%"=="-" ( + goto end_passthru +) + +if defined PASSTHRU_VALUES ( + set "PASSTHRU_VALUES=%PASSTHRU_VALUES%,%arg%" +) else ( + set "PASSTHRU_VALUES=%arg%" +) +SHIFT +goto repeat_passthru + +:end_passthru +set "PASSTHRU=%PASSTHRU_KEY%=%PASSTHRU_VALUES%" + IF NOT "%SOLR_OPTS%"=="" ( set "SOLR_OPTS=%SOLR_OPTS% %PASSTHRU%" ) ELSE ( set "SOLR_OPTS=%PASSTHRU%" ) set "PASS_TO_RUN_EXAMPLE=%PASSTHRU% !PASS_TO_RUN_EXAMPLE!" -SHIFT -SHIFT + goto parse_args :set_noprompt @@ -670,8 +682,27 @@ set "PASS_TO_RUN_EXAMPLE=--no-prompt !PASS_TO_RUN_EXAMPLE!" SHIFT goto parse_args -REM Perform the requested command after processing args -:process_script_cmd +REM Handle invalid arguments passed to special commands (start, stop, restart) +:invalid_cmd_line +@echo. +IF "!SCRIPT_ERROR!"=="" ( + @echo Invalid command-line option: %1 +) ELSE ( + @echo ERROR: !SCRIPT_ERROR! +) +@echo. +IF "%SCRIPT_CMD%"=="start" ( + goto start_usage +) ELSE IF "%SCRIPT_CMD%"=="restart" ( + goto start_usage +) ELSE IF "%SCRIPT_CMD%"=="stop" ( + goto stop_usage +) +REM Not reachable, but just in case +goto err + +REM Process special commands (start, stop, restart) +:run_special_command IF "%verbose%"=="1" ( CALL :safe_echo "Using Solr root directory: %SOLR_TIP%" @@ -712,36 +743,20 @@ IF NOT EXIST "%SOLR_SERVER_DIR%" ( goto err ) -IF NOT "%EXAMPLE%"=="" goto run_example - -:start_solr -IF "%SOLR_HOME%"=="" set "SOLR_HOME=%SOLR_SERVER_DIR%\solr" -IF EXIST "%cd%\%SOLR_HOME%" set "SOLR_HOME=%cd%\%SOLR_HOME%" - -IF NOT EXIST "%SOLR_HOME%\" ( - IF EXIST "%SOLR_SERVER_DIR%\%SOLR_HOME%" ( - set "SOLR_HOME=%SOLR_SERVER_DIR%\%SOLR_HOME%" - ) ELSE ( - set "SCRIPT_ERROR=Solr home directory %SOLR_HOME% not found!" - goto err - ) -) - IF "%STOP_KEY%"=="" set STOP_KEY=solrrocks -@REM This is quite hacky, but examples rely on a different log4j2.xml -@REM so that we can write logs for examples to %SOLR_HOME%\..\logs -IF [%SOLR_LOGS_DIR%] == [] ( - set "SOLR_LOGS_DIR=%SOLR_SERVER_DIR%\logs" -) ELSE ( - set SOLR_LOGS_DIR=%SOLR_LOGS_DIR:"=% -) +IF NOT "%EXAMPLE%"=="" ( + REM Run the requested example -set "EXAMPLE_DIR=%SOLR_TIP%\example" -set TMP_SOLR_HOME=!SOLR_HOME:%EXAMPLE_DIR%=! -IF NOT "%TMP_SOLR_HOME%"=="%SOLR_HOME%" ( - set "SOLR_LOGS_DIR=%SOLR_HOME%\..\logs" - set "LOG4J_CONFIG=%SOLR_SERVER_DIR%\resources\log4j2.xml" + "%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% %SOLR_TOOL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" ^ + -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ + -Dsolr.install.symDir="%SOLR_TIP%" ^ + -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ + org.apache.solr.cli.SolrCLI run_example --script "%SDIR%\solr.cmd" -e %EXAMPLE% --server-dir "%SOLR_SERVER_DIR%" ^ + --url-scheme !SOLR_URL_SCHEME! !PASS_TO_RUN_EXAMPLE! + + REM End of run_example + goto done ) set IS_RESTART=0 @@ -754,77 +769,111 @@ IF "%SCRIPT_CMD%"=="restart" ( set IS_RESTART=1 ) +REM Skipt to start if not stop or restart (that executes stop first) +IF "%SCRIPT_CMD%"=="start" goto start_solr + @REM stop logic here +:stop_solr IF "%SOLR_STOP_WAIT%"=="" ( set SOLR_STOP_WAIT=180 ) -IF "%SCRIPT_CMD%"=="stop" ( - IF "%SOLR_PORT%"=="" ( - IF "%STOP_ALL%"=="1" ( - set found_it=0 - for /f "usebackq" %%i in (`dir /b "%SOLR_TIP%\bin" ^| findstr /i "^solr-.*\.port$"`) do ( - set SOME_SOLR_PORT= - For /F "delims=" %%J In ('type "%SOLR_TIP%\bin\%%i"') do set SOME_SOLR_PORT=%%~J - if NOT "!SOME_SOLR_PORT!"=="" ( - for /f "tokens=2,5" %%j in ('netstat -aon ^| find "TCP " ^| find ":0 " ^| find ":!SOME_SOLR_PORT! "') do ( - @REM j is the ip:port and k is the pid - IF NOT "%%k"=="0" ( - IF "%%j"=="%SOLR_JETTY_HOST%:!SOME_SOLR_PORT!" ( - set found_it=1 - @echo Stopping Solr process %%k running on port !SOME_SOLR_PORT! - IF "%STOP_PORT%"=="" ( - set /A LOCAL_STOP_PORT=!SOME_SOLR_PORT! - 1000 - ) else ( - set LOCAL_STOP_PORT=%STOP_PORT% - ) - "%JAVA%" %SOLR_SSL_OPTS% -Djetty.home="%SOLR_SERVER_DIR%" -jar "%SOLR_SERVER_DIR%\start.jar" STOP.PORT=!LOCAL_STOP_PORT! STOP.KEY=%STOP_KEY% --stop - del "%SOLR_TIP%"\bin\solr-!SOME_SOLR_PORT!.port - REM wait for the process to terminate - CALL :wait_for_process_exit %%k !SOLR_STOP_WAIT! - REM Kill it if it is still running after the graceful shutdown - IF EXIST "%JAVA_HOME%\bin\jstack.exe" ( - qprocess "%%k" >nul 2>nul && "%JAVA_HOME%\bin\jstack.exe" %%k && taskkill /f /PID %%k - ) else ( - qprocess "%%k" >nul 2>nul && taskkill /f /PID %%k - ) +IF "%SOLR_PORT%"=="" ( + IF "%STOP_ALL%"=="1" ( + REM Stop all running Solr instances + set found_it=0 + for /f "usebackq" %%i in (`dir /b "%SOLR_TIP%\bin" ^| findstr /i "^solr-.*\.port$"`) do ( + set SOME_SOLR_PORT= + For /F "delims=" %%J In ('type "%SOLR_TIP%\bin\%%i"') do set SOME_SOLR_PORT=%%~J + if NOT "!SOME_SOLR_PORT!"=="" ( + for /f "tokens=2,5" %%j in ('netstat -aon ^| find "TCP " ^| find ":0 " ^| find ":!SOME_SOLR_PORT! "') do ( + @REM j is the ip:port and k is the pid + IF NOT "%%k"=="0" ( + IF "%%j"=="%SOLR_JETTY_HOST%:!SOME_SOLR_PORT!" ( + set found_it=1 + @echo Stopping Solr process %%k running on port !SOME_SOLR_PORT! + IF "%STOP_PORT%"=="" ( + set /A LOCAL_STOP_PORT=!SOME_SOLR_PORT! - 1000 + ) else ( + set LOCAL_STOP_PORT=%STOP_PORT% + ) + "%JAVA%" %SOLR_SSL_OPTS% -Djetty.home="%SOLR_SERVER_DIR%" -jar "%SOLR_SERVER_DIR%\start.jar" STOP.PORT=!LOCAL_STOP_PORT! STOP.KEY=%STOP_KEY% --stop + del "%SOLR_TIP%"\bin\solr-!SOME_SOLR_PORT!.port + REM wait for the process to terminate + CALL :wait_for_process_exit %%k !SOLR_STOP_WAIT! + REM Kill it if it is still running after the graceful shutdown + IF EXIST "%JAVA_HOME%\bin\jstack.exe" ( + qprocess "%%k" >nul 2>nul && "%JAVA_HOME%\bin\jstack.exe" %%k && taskkill /f /PID %%k + ) else ( + qprocess "%%k" >nul 2>nul && taskkill /f /PID %%k ) ) ) ) ) - if "!found_it!"=="0" echo No Solr nodes found to stop. - ) ELSE ( - set "SCRIPT_ERROR=Must specify the port when trying to stop Solr, or use --all to stop all running nodes on this host." - goto err ) + if "!found_it!"=="0" echo No Solr nodes found to stop. ) ELSE ( - set found_it=0 - For /f "tokens=2,5" %%M in ('netstat -nao ^| find "TCP " ^| find ":0 " ^| find ":%SOLR_PORT% "') do ( - IF NOT "%%N"=="0" ( - IF "%%M"=="%SOLR_JETTY_HOST%:%SOLR_PORT%" ( - set found_it=1 - @echo Stopping Solr process %%N running on port %SOLR_PORT% - IF "%STOP_PORT%"=="" set /A STOP_PORT=%SOLR_PORT% - 1000 - "%JAVA%" %SOLR_SSL_OPTS% %SOLR_TOOL_OPTS% -Djetty.home="%SOLR_SERVER_DIR%" -jar "%SOLR_SERVER_DIR%\start.jar" %SOLR_JETTY_CONFIG% STOP.PORT=!STOP_PORT! STOP.KEY=%STOP_KEY% --stop - del "%SOLR_TIP%"\bin\solr-%SOLR_PORT%.port - REM wait for the process to terminate - CALL :wait_for_process_exit %%N !SOLR_STOP_WAIT! - REM Kill it if it is still running after the graceful shutdown - IF EXIST "%JAVA_HOME%\bin\jstack.exe" ( - qprocess "%%N" >nul 2>nul && "%JAVA_HOME%\bin\jstack.exe" %%N && taskkill /f /PID %%N - ) else ( - qprocess "%%N" >nul 2>nul && taskkill /f /PID %%N - ) + set "SCRIPT_ERROR=Must specify the port when trying to stop Solr, or use --all to stop all running nodes on this host." + goto err + ) +) ELSE ( + REM Stop Solr running on specific port + set found_it=0 + For /f "tokens=2,5" %%M in ('netstat -nao ^| find "TCP " ^| find ":0 " ^| find ":%SOLR_PORT% "') do ( + IF NOT "%%N"=="0" ( + IF "%%M"=="%SOLR_JETTY_HOST%:%SOLR_PORT%" ( + set found_it=1 + @echo Stopping Solr process %%N running on port %SOLR_PORT% + IF "%STOP_PORT%"=="" set /A STOP_PORT=%SOLR_PORT% - 1000 + "%JAVA%" %SOLR_SSL_OPTS% %SOLR_TOOL_OPTS% -Djetty.home="%SOLR_SERVER_DIR%" -jar "%SOLR_SERVER_DIR%\start.jar" %SOLR_JETTY_CONFIG% STOP.PORT=!STOP_PORT! STOP.KEY=%STOP_KEY% --stop + del "%SOLR_TIP%"\bin\solr-%SOLR_PORT%.port + REM wait for the process to terminate + CALL :wait_for_process_exit %%N !SOLR_STOP_WAIT! + REM Kill it if it is still running after the graceful shutdown + IF EXIST "%JAVA_HOME%\bin\jstack.exe" ( + qprocess "%%N" >nul 2>nul && "%JAVA_HOME%\bin\jstack.exe" %%N && taskkill /f /PID %%N + ) else ( + qprocess "%%N" >nul 2>nul && taskkill /f /PID %%N ) ) ) - if "!found_it!"=="0" echo No Solr found running on port %SOLR_PORT% ) + if "!found_it!"=="0" echo No Solr found running on port %SOLR_PORT% +) + +IF "!IS_RESTART!"=="0" goto done + +REM Clean state here, continue with starting (start or restart) +set SCRIPT_CMD=start + +:start_solr +REM Prepare for starting Solr +IF "%SOLR_HOME%"=="" set "SOLR_HOME=%SOLR_SERVER_DIR%\solr" +IF EXIST "%cd%\%SOLR_HOME%" set "SOLR_HOME=%cd%\%SOLR_HOME%" + +IF NOT EXIST "%SOLR_HOME%\" ( + IF EXIST "%SOLR_SERVER_DIR%\%SOLR_HOME%" ( + set "SOLR_HOME=%SOLR_SERVER_DIR%\%SOLR_HOME%" + ) ELSE ( + set "SCRIPT_ERROR=Solr home directory %SOLR_HOME% not found!" + goto err + ) +) + +@REM Handle overriding where logs are written to +IF [%SOLR_LOGS_DIR%] == [] ( + set "SOLR_LOGS_DIR=%SOLR_SERVER_DIR%\logs" +) ELSE ( + set SOLR_LOGS_DIR=%SOLR_LOGS_DIR:"=% +) - IF "!IS_RESTART!"=="0" goto done +set "EXAMPLE_DIR=%SOLR_TIP%\example" +set TMP_SOLR_HOME=!SOLR_HOME:%EXAMPLE_DIR%=! +IF NOT "%TMP_SOLR_HOME%"=="%SOLR_HOME%" ( + set "SOLR_LOGS_DIR=%SOLR_HOME%\..\logs" + set "LOG4J_CONFIG=%SOLR_SERVER_DIR%\resources\log4j2.xml" ) -IF "!IS_RESTART!"=="1" set SCRIPT_CMD=start IF "%SOLR_PORT%"=="" set SOLR_PORT=8983 IF "%STOP_PORT%"=="" set /A STOP_PORT=%SOLR_PORT% - 1000 @@ -841,21 +890,19 @@ IF DEFINED SOLR_ZK_EMBEDDED_HOST ( set "SCRIPT_SOLR_OPTS=%SCRIPT_SOLR_OPTS% -Dsolr.zk.embedded.host=%SOLR_ZK_EMBEDDED_HOST%" ) -IF "%SCRIPT_CMD%"=="start" ( - REM see if Solr is already running using netstat - For /f "tokens=2,5" %%j in ('netstat -aon ^| find "TCP " ^| find ":0 " ^| find ":%SOLR_PORT% "') do ( - IF NOT "%%k"=="0" ( - IF "%%j"=="%SOLR_JETTY_HOST%:%SOLR_PORT%" ( - set "SCRIPT_ERROR=Process %%k is already listening on port %SOLR_PORT%. If this is Solr, please stop it first before starting (or use restart). If this is not Solr, then please choose a different port using -p PORT" - goto err - ) +REM Make sure Solr is not running using netstat +For /f "tokens=2,5" %%j in ('netstat -aon ^| find "TCP " ^| find ":0 " ^| find ":%SOLR_PORT% "') do ( + IF NOT "%%k"=="0" ( + IF "%%j"=="%SOLR_JETTY_HOST%:%SOLR_PORT%" ( + set "SCRIPT_ERROR=Process %%k is already listening on port %SOLR_PORT%. If this is Solr, please stop it first before starting (or use restart). If this is not Solr, then please choose a different port using -p PORT" + goto err ) ) +) - IF "%EMPTY_ADDL_JVM_ARGS%"=="true" ( - set "SCRIPT_ERROR=JVM options are required when using the -a or --jvm-opts option!" - goto err - ) +IF "%EMPTY_ADDL_JVM_ARGS%"=="true" ( + set "SCRIPT_ERROR=JVM options are required when using the -a or --jvm-opts option!" + goto err ) @REM determine if -server flag is supported by current JVM @@ -1131,50 +1178,6 @@ IF "%FG%"=="1" ( goto done -:run_example -REM Run the requested example - -"%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% %SOLR_TOOL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" ^ - -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ - -Dsolr.install.symDir="%SOLR_TIP%" ^ - -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ - org.apache.solr.cli.SolrCLI run_example --script "%SDIR%\solr.cmd" -e %EXAMPLE% --server-dir "%SOLR_SERVER_DIR%" ^ - --url-scheme !SOLR_URL_SCHEME! !PASS_TO_RUN_EXAMPLE! - -REM End of run_example -goto done - -:run_solrcli -"%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% %SOLR_TOOL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" ^ - -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ - -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ - org.apache.solr.cli.SolrCLI %* -if errorlevel 1 ( - exit /b 1 -) -goto done - -:parse_config_args -IF [%1]==[] goto run_config -IF "%1"=="-z" goto set_config_zk -IF "%1"=="--zk-host" goto set_config_zk -IF "%1"=="--scheme" goto set_config_url_scheme -set "CONFIG_ARGS=!CONFIG_ARGS! %1" -SHIFT -goto parse_config_args - -:set_config_zk -set ZK_HOST=%~2 -SHIFT -SHIFT -goto parse_config_args - -:set_config_url_scheme -set SOLR_URL_SCHEME=%~2 -SHIFT -SHIFT -goto parse_config_args - :run_auth REM Options parsing. REM Note: With the following technique of parsing, it is not possible @@ -1187,11 +1190,9 @@ for %%a in (%*) do ( if "!arg:~0,1!" equ "-" set "option=!arg!" ) else ( set "option!option!=%%a" - if "!option!" equ "-s" set "SOLR_HOME=%%a" - if "!option!" equ "--solr-home" set "SOLR_HOME=%%a" - if "!option!" equ "-d" set "SOLR_SERVER_DIR=%%a" + if "!option!" equ "--solr-home" set "SOLR_HOME=%%a" if "!option!" equ "--server-dir" set "SOLR_SERVER_DIR=%%a" - if not "!option!" equ "-s" if not "!option!" equ "--solr-home" if not "!option!" equ "-d" if not "!option!" equ "--server-dir" ( + if not "!option!" equ "--solr-home" if not "!option!" equ "--server-dir" ( set "AUTH_PARAMS=!AUTH_PARAMS! !option! %%a" ) set "option=" @@ -1231,49 +1232,12 @@ if "!AUTH_PORT!"=="" ( --solr-url !SOLR_URL_SCHEME!://%SOLR_TOOL_HOST%:!AUTH_PORT! goto done - -:invalid_cmd_line -@echo. -IF "!SCRIPT_ERROR!"=="" ( - @echo Invalid command-line option: %1 -) ELSE ( - @echo ERROR: !SCRIPT_ERROR! -) -@echo. -IF "%FIRST_ARG%"=="start" ( - goto start_usage -) ELSE IF "%FIRST_ARG:~0,1%" == "-" ( - goto start_usage -) ELSE IF "%FIRST_ARG%"=="restart" ( - goto start_usage -) ELSE IF "%FIRST_ARG%"=="stop" ( - goto stop_usage -) ELSE IF "%FIRST_ARG%"=="healthcheck" ( - goto run_solrcli -) ELSE IF "%FIRST_ARG%"=="create" ( - goto run_solrcli -) ELSE IF "%FIRST_ARG%"=="zk" ( - goto run_solrcli -) ELSE IF "%FIRST_ARG%"=="auth" ( - goto run_solrcli -) ELSE IF "%FIRST_ARG%"=="status" ( - goto run_solrcli -) - -:need_java_home -@echo Please set the JAVA_HOME environment variable to the path where you installed Java 21+ -goto done - :err @echo. @echo ERROR: !SCRIPT_ERROR! @echo. exit /b 1 -:done -ENDLOCAL -exit /b 0 - REM Tests what Java we have and sets some global variables :resolve_java_info @@ -1347,3 +1311,7 @@ GOTO :eof ) ) GOTO :eof + +:done +ENDLOCAL +exit /b 0 diff --git a/solr/core/build.gradle b/solr/core/build.gradle index 92bbfbe1120..f406f842201 100644 --- a/solr/core/build.gradle +++ b/solr/core/build.gradle @@ -23,187 +23,187 @@ dependencies { // Spotbugs Annotations are only needed for old findbugs // annotation usage like in Zookeeper during compilation time. // It is not included in the release so exclude from checks. - compileOnly 'com.github.spotbugs:spotbugs-annotations' - testCompileOnly 'com.github.spotbugs:spotbugs-annotations' - permitUnusedDeclared 'com.github.spotbugs:spotbugs-annotations' + compileOnly libs.spotbugs.annotations + testCompileOnly libs.spotbugs.annotations + permitUnusedDeclared libs.spotbugs.annotations // Exclude these from jar validation and license checks. configurations.jarValidation { exclude group: "com.github.spotbugs", module: "spotbugs-annotations" } - implementation 'io.swagger.core.v3:swagger-annotations-jakarta' + implementation libs.swagger3.annotations.jakarta // Export these dependencies so that they're imported transitively by // other modules. // These Lucene modules are the most significant to Solr - api "org.apache.lucene:lucene-core" - api "org.apache.lucene:lucene-analysis-common" - api "org.apache.lucene:lucene-queries" + api libs.apache.lucene.core + api libs.apache.lucene.analysis.common + api libs.apache.lucene.queries // We export logging api with dependencies, which is useful for all modules - api 'org.slf4j:slf4j-api' + api libs.slf4j.api api project(':solr:api') api project(':solr:solrj') api project(':solr:solrj-zookeeper') api project(':solr:solrj-streaming') - api 'io.dropwizard.metrics:metrics-core' - implementation ('io.dropwizard.metrics:metrics-graphite', { + api libs.dropwizard.metrics.core + implementation (libs.dropwizard.metrics.graphite, { exclude group: "com.rabbitmq", module: "amqp-client" }) - implementation 'io.dropwizard.metrics:metrics-jmx' - implementation 'io.dropwizard.metrics:metrics-jvm' + implementation libs.dropwizard.metrics.jmx + implementation libs.dropwizard.metrics.jvm - implementation('org.glassfish.jersey.containers:jersey-container-jetty-http', { + implementation(libs.jersey.containers.jettyhttp, { exclude group: "org.eclipse.jetty", module: "jetty-continuation" exclude group: "org.glassfish.hk2.external", module: "jakarta.inject" }) - permitUnusedDeclared 'org.glassfish.jersey.containers:jersey-container-jetty-http' - implementation 'org.glassfish.jersey.inject:jersey-hk2' - permitUnusedDeclared 'org.glassfish.jersey.inject:jersey-hk2' - implementation ('org.glassfish.jersey.media:jersey-media-json-jackson', { + permitUnusedDeclared libs.jersey.containers.jettyhttp + implementation libs.jersey.inject.hk2 + permitUnusedDeclared libs.jersey.inject.hk2 + implementation (libs.jersey.media.jsonjackson, { exclude group: "jakarta.xml.bind", module: "jakarta.xml.bind-api" }) - permitUnusedDeclared 'org.glassfish.jersey.media:jersey-media-json-jackson' - implementation 'org.glassfish.jersey.core:jersey-common' - implementation 'org.glassfish.jersey.core:jersey-server' - implementation 'org.glassfish.hk2:hk2-api' - implementation 'jakarta.inject:jakarta.inject-api' - implementation 'jakarta.ws.rs:jakarta.ws.rs-api' - implementation 'jakarta.annotation:jakarta.annotation-api' + permitUnusedDeclared libs.jersey.media.jsonjackson + implementation libs.jersey.core.common + implementation libs.jersey.core.server + implementation libs.hk2.api + implementation libs.jakarta.inject.api + implementation libs.jakarta.ws.rsapi + implementation libs.jakarta.annotation.api // Non-API below; although there are exceptions - runtimeOnly "org.apache.lucene:lucene-analysis-kuromoji" - runtimeOnly "org.apache.lucene:lucene-analysis-nori" - runtimeOnly "org.apache.lucene:lucene-analysis-phonetic" - runtimeOnly "org.apache.lucene:lucene-backward-codecs" - implementation "org.apache.lucene:lucene-codecs" - implementation "org.apache.lucene:lucene-backward-codecs" - permitUnusedDeclared "org.apache.lucene:lucene-backward-codecs" - implementation "org.apache.lucene:lucene-classification" - implementation "org.apache.lucene:lucene-expressions" - implementation "org.apache.lucene:lucene-grouping" - implementation "org.apache.lucene:lucene-highlighter" - implementation "org.apache.lucene:lucene-join" - implementation "org.apache.lucene:lucene-misc" - implementation "org.apache.lucene:lucene-queryparser" - implementation "org.apache.lucene:lucene-spatial-extras" - implementation "org.apache.lucene:lucene-suggest" + runtimeOnly libs.apache.lucene.analysis.kuromoji + runtimeOnly libs.apache.lucene.analysis.nori + runtimeOnly libs.apache.lucene.analysis.phonetic + runtimeOnly libs.apache.lucene.backward.codecs + implementation libs.apache.lucene.codecs + implementation libs.apache.lucene.backward.codecs + permitUnusedDeclared libs.apache.lucene.backward.codecs + implementation libs.apache.lucene.classification + implementation libs.apache.lucene.expressions + implementation libs.apache.lucene.grouping + implementation libs.apache.lucene.highlighter + implementation libs.apache.lucene.join + implementation libs.apache.lucene.misc + implementation libs.apache.lucene.queryparser + implementation libs.apache.lucene.spatialextras + implementation libs.apache.lucene.suggest // Collections & lang utilities - implementation 'com.google.guava:guava' - implementation 'org.apache.commons:commons-lang3' - implementation 'org.apache.commons:commons-math3' - implementation 'commons-io:commons-io' - implementation 'com.carrotsearch:hppc' + implementation libs.google.guava + implementation libs.apache.commons.lang3 + implementation libs.apache.commons.math3 + implementation libs.commonsio.commonsio + implementation libs.carrotsearch.hppc - implementation('com.github.ben-manes.caffeine:caffeine') { transitive = false } + implementation(libs.benmanes.caffeine) { transitive = false } - implementation 'commons-codec:commons-codec' + implementation libs.commonscodec.commonscodec - implementation 'commons-cli:commons-cli' + implementation libs.commonscli.commonscli - implementation 'org.locationtech.spatial4j:spatial4j' + implementation libs.locationtech.spatial4j - implementation 'com.fasterxml.jackson.core:jackson-annotations' - implementation 'com.fasterxml.jackson.core:jackson-core' - implementation 'com.fasterxml.jackson.core:jackson-databind' - implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-smile' - implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-cbor' + implementation libs.fasterxml.jackson.core.annotations + implementation libs.fasterxml.jackson.core.core + implementation libs.fasterxml.jackson.core.databind + implementation libs.fasterxml.jackson.dataformat.smile + implementation libs.fasterxml.jackson.dataformat.cbor - implementation 'org.apache.httpcomponents:httpclient' - implementation 'org.apache.httpcomponents:httpcore' + implementation libs.apache.httpcomponents.httpclient + implementation libs.apache.httpcomponents.httpcore - implementation 'org.eclipse.jetty:jetty-client' - implementation 'org.eclipse.jetty:jetty-http' - implementation 'org.eclipse.jetty:jetty-io' - implementation 'org.eclipse.jetty.toolchain:jetty-servlet-api' + implementation libs.eclipse.jetty.client + implementation libs.eclipse.jetty.http + implementation libs.eclipse.jetty.io + implementation libs.eclipse.jetty.toolchain.servletapi // ZooKeeper - implementation('org.apache.curator:curator-framework', { + implementation(libs.apache.curator.framework, { exclude group: 'org.apache.zookeeper', module: 'zookeeper' }) - implementation('org.apache.curator:curator-client', { + implementation(libs.apache.curator.client, { exclude group: 'org.apache.zookeeper', module: 'zookeeper' }) - testImplementation('org.apache.curator:curator-test', { + testImplementation(libs.apache.curator.test, { exclude group: 'org.apache.zookeeper', module: 'zookeeper' }) - implementation('org.apache.zookeeper:zookeeper', { + implementation(libs.apache.zookeeper.zookeeper, { exclude group: "org.apache.yetus", module: "audience-annotations" }) - implementation('org.apache.zookeeper:zookeeper-jute') { + implementation(libs.apache.zookeeper.jute) { exclude group: 'org.apache.yetus', module: 'audience-annotations' } - testImplementation 'org.apache.zookeeper:zookeeper::tests' + testImplementation variantOf(libs.apache.zookeeper.zookeeper) { classifier 'tests' } // required for instantiating a Zookeeper server (for embedding ZK or running tests) - runtimeOnly ('org.xerial.snappy:snappy-java') + runtimeOnly libs.xerial.snappy.java - implementation('com.jayway.jsonpath:json-path', { + implementation(libs.jayway.jsonpath, { exclude group: "net.minidev", module: "json-smart" }) // StatsComponents percentiles - implementation 'com.tdunning:t-digest' + implementation libs.tdunning.tdigest // Distributed Tracing - api 'io.opentelemetry:opentelemetry-api' // Tracer is exposed on some methods - implementation 'io.opentelemetry:opentelemetry-context' + api libs.opentelemetry.api // Tracer is exposed on some methods + implementation libs.opentelemetry.context - implementation 'org.apache.commons:commons-exec' + implementation libs.apache.commons.exec - implementation 'org.apache.logging.log4j:log4j-api' - implementation 'org.apache.logging.log4j:log4j-core' - runtimeOnly 'org.apache.logging.log4j:log4j-slf4j2-impl' + implementation libs.apache.log4j.api + implementation libs.apache.log4j.core + runtimeOnly libs.apache.log4j.slf4j2impl // For the PrometheusResponseWriter - implementation 'io.prometheus:prometheus-metrics-model:1.1.0' - implementation('io.prometheus:prometheus-metrics-exposition-formats:1.1.0', { + implementation libs.prometheus.metrics.model + implementation(libs.prometheus.metrics.expositionformats, { exclude group: "io.prometheus", module: "prometheus-metrics-shaded-protobuf" exclude group: "io.prometheus", module: "prometheus-metrics-config" }) // For faster XML processing than the JDK - implementation 'org.codehaus.woodstox:stax2-api' - implementation 'com.fasterxml.woodstox:woodstox-core' + implementation libs.codehaus.woodstox.stax2api + implementation libs.fasterxml.woodstox.core // See https://issues.apache.org/jira/browse/LOG4J2-3609 due to needing these annotations - compileOnly 'biz.aQute.bnd:biz.aQute.bnd.annotation' - compileOnly 'org.osgi:osgi.annotation' + compileOnly libs.aqute.bnd.annotation + compileOnly libs.osgi.annotation - compileOnly 'com.github.stephenc.jcip:jcip-annotations' + compileOnly libs.stephenc.jcip.annotations - implementation 'com.j256.simplemagic:simplemagic' + implementation libs.j256.simplemagic // -- Test Dependencies - testRuntimeOnly 'org.slf4j:jcl-over-slf4j' + testRuntimeOnly libs.slf4j.jcloverslf4j - testRuntimeOnly "org.apache.lucene:lucene-analysis-icu" + testRuntimeOnly libs.apache.lucene.analysis.icu testRuntimeOnly project(':solr:modules:analysis-extras') testImplementation project(':solr:core') testImplementation project(':solr:test-framework') - testImplementation 'org.apache.lucene:lucene-test-framework' + testImplementation libs.apache.lucene.testframework - testImplementation 'org.eclipse.jetty:jetty-server' - testImplementation 'org.eclipse.jetty:jetty-servlet' + testImplementation libs.eclipse.jetty.server + testImplementation libs.eclipse.jetty.servlet - testImplementation 'com.carrotsearch.randomizedtesting:randomizedtesting-runner' - testImplementation 'junit:junit' - testImplementation 'org.hamcrest:hamcrest' + testImplementation libs.carrotsearch.randomizedtesting.runner + testImplementation libs.junit.junit + testImplementation libs.hamcrest.hamcrest - testImplementation('org.mockito:mockito-core', { + testImplementation(libs.mockito.core, { exclude group: "net.bytebuddy", module: "byte-buddy-agent" }) - testRuntimeOnly('org.mockito:mockito-subclass', { + testRuntimeOnly(libs.mockito.subclass, { exclude group: "net.bytebuddy", module: "byte-buddy-agent" }) } diff --git a/solr/core/src/java/org/apache/solr/api/AnnotatedApi.java b/solr/core/src/java/org/apache/solr/api/AnnotatedApi.java index bfe35c47108..d4c6e454520 100644 --- a/solr/core/src/java/org/apache/solr/api/AnnotatedApi.java +++ b/solr/core/src/java/org/apache/solr/api/AnnotatedApi.java @@ -261,8 +261,7 @@ static class Cmd { } private void readPayloadType(Type t) { - if (t instanceof ParameterizedType) { - ParameterizedType typ = (ParameterizedType) t; + if (t instanceof ParameterizedType typ) { if (typ.getRawType() == PayloadObj.class) { isWrappedInPayloadObj = true; if (typ.getActualTypeArguments().length == 0) { @@ -271,8 +270,7 @@ private void readPayloadType(Type t) { return; } Type t1 = typ.getActualTypeArguments()[0]; - if (t1 instanceof ParameterizedType) { - ParameterizedType parameterizedType = (ParameterizedType) t1; + if (t1 instanceof ParameterizedType parameterizedType) { parameterClass = (Class) parameterizedType.getRawType(); } else { parameterClass = (Class) typ.getActualTypeArguments()[0]; @@ -345,9 +343,8 @@ public int hashCode() { public boolean equals(Object rhs) { if (null == rhs) return false; if (this == rhs) return true; - if (!(rhs instanceof Cmd)) return false; + if (!(rhs instanceof Cmd rhsCast)) return false; - final Cmd rhsCast = (Cmd) rhs; return Objects.equals(command, rhsCast.command) && Objects.equals(method, rhsCast.method) && Objects.equals(obj, rhsCast.obj) @@ -373,8 +370,7 @@ public static Map createSchema(Method m) { t = types[2]; // (SolrQueryRequest req, SolrQueryResponse rsp, PayloadObj) if (types.length == 1) t = types[0]; // (PayloadObj) if (t != null) { - if (t instanceof ParameterizedType) { - ParameterizedType typ = (ParameterizedType) t; + if (t instanceof ParameterizedType typ) { if (typ.getRawType() == PayloadObj.class) { t = typ.getActualTypeArguments()[0]; } diff --git a/solr/core/src/java/org/apache/solr/api/ApiBag.java b/solr/core/src/java/org/apache/solr/api/ApiBag.java index 48359f783b1..1d9a6d2a80d 100644 --- a/solr/core/src/java/org/apache/solr/api/ApiBag.java +++ b/solr/core/src/java/org/apache/solr/api/ApiBag.java @@ -122,16 +122,14 @@ protected void attachValueToNode(PathTrie.Node node, Api o) { // If 'o' and 'node.obj' aren't both AnnotatedApi's then we can't aggregate the commands, so // fallback to the default behavior - if ((!(o instanceof AnnotatedApi)) || (!(node.getObject() instanceof AnnotatedApi))) { + if ((!(o instanceof AnnotatedApi beingRegistered)) + || (!(node.getObject() instanceof AnnotatedApi alreadyRegistered))) { super.attachValueToNode(node, o); return; } - final AnnotatedApi beingRegistered = (AnnotatedApi) o; - final AnnotatedApi alreadyRegistered = (AnnotatedApi) node.getObject(); - if (alreadyRegistered instanceof CommandAggregatingAnnotatedApi) { - final CommandAggregatingAnnotatedApi alreadyRegisteredAsCollapsing = - (CommandAggregatingAnnotatedApi) alreadyRegistered; + if (alreadyRegistered + instanceof CommandAggregatingAnnotatedApi alreadyRegisteredAsCollapsing) { alreadyRegisteredAsCollapsing.combineWith(beingRegistered); } else { final CommandAggregatingAnnotatedApi wrapperApi = @@ -404,11 +402,10 @@ public void registerLazy(PluginBag.PluginHolder holder, Plug public static SpecProvider constructSpec(PluginInfo info) { Object specObj = info == null ? null : info.attributes.get("spec"); - if (specObj != null && specObj instanceof Map) { + if (specObj != null && specObj instanceof Map map) { // Value from Map can be a Map because in PluginInfo(String, Map) we assign a // Map // assert false : "got a map when this should only be Strings"; - Map map = (Map) specObj; return () -> ValidatingJsonMap.getDeepCopy(map, 4, false); } else { return HANDLER_NAME_SPEC_PROVIDER; diff --git a/solr/core/src/java/org/apache/solr/api/ContainerPluginsRegistry.java b/solr/core/src/java/org/apache/solr/api/ContainerPluginsRegistry.java index 4573a6e550e..20cd2440bbe 100644 --- a/solr/core/src/java/org/apache/solr/api/ContainerPluginsRegistry.java +++ b/solr/core/src/java/org/apache/solr/api/ContainerPluginsRegistry.java @@ -158,8 +158,7 @@ static class PluginMetaHolder { @Override public boolean equals(Object obj) { - if (obj instanceof PluginMetaHolder) { - PluginMetaHolder that = (PluginMetaHolder) obj; + if (obj instanceof PluginMetaHolder that) { return Objects.equals(this.original, that.original); } return false; @@ -466,8 +465,7 @@ public static Class getConfigClass(ConfigurablePlugin response = solrClient.request(req); // pretty-print the response to stdout CharArr arr = new CharArr(); - new JSONWriter(arr, 2).write(response.asMap()); + new JSONWriter(arr, 2).write(response.asMap(10)); return arr.toString(); } } diff --git a/solr/core/src/java/org/apache/solr/cli/AssertTool.java b/solr/core/src/java/org/apache/solr/cli/AssertTool.java index b710d0934b2..b111ef7b481 100644 --- a/solr/core/src/java/org/apache/solr/cli/AssertTool.java +++ b/solr/core/src/java/org/apache/solr/cli/AssertTool.java @@ -250,13 +250,13 @@ protected int runAssert(CommandLine cli) throws Exception { if (cli.hasOption(IS_CLOUD_OPTION)) { ret += assertSolrRunningInCloudMode( - SolrCLI.normalizeSolrUrl(cli.getOptionValue(IS_CLOUD_OPTION)), + CLIUtils.normalizeSolrUrl(cli.getOptionValue(IS_CLOUD_OPTION)), cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION)); } if (cli.hasOption(IS_NOT_CLOUD_OPTION)) { ret += assertSolrNotRunningInCloudMode( - SolrCLI.normalizeSolrUrl(cli.getOptionValue(IS_NOT_CLOUD_OPTION)), + CLIUtils.normalizeSolrUrl(cli.getOptionValue(IS_NOT_CLOUD_OPTION)), cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION)); } return ret; @@ -267,7 +267,7 @@ public static int assertSolrRunning(String url, String credentials) throws Excep try { status.waitToSeeSolrUp(url, credentials, timeoutMs, TimeUnit.MILLISECONDS); } catch (Exception se) { - if (SolrCLI.exceptionIsAuthRelated(se)) { + if (CLIUtils.exceptionIsAuthRelated(se)) { throw se; } return exitOrException( @@ -284,10 +284,10 @@ public static int assertSolrNotRunning(String url, String credentials) throws Ex StatusTool status = new StatusTool(); long timeout = System.nanoTime() + TimeUnit.NANOSECONDS.convert(timeoutMs, TimeUnit.MILLISECONDS); - try (SolrClient solrClient = SolrCLI.getSolrClient(url, credentials)) { + try (SolrClient solrClient = CLIUtils.getSolrClient(url, credentials)) { NamedList response = solrClient.request(new HealthCheckRequest()); Integer statusCode = (Integer) response.findRecursive("responseHeader", "status"); - SolrCLI.checkCodeForAuthError(statusCode); + CLIUtils.checkCodeForAuthError(statusCode); } catch (IOException | SolrServerException e) { log.debug("Opening connection to {} failed, Solr does not seem to be running", url, e); return 0; @@ -302,7 +302,7 @@ public static int assertSolrNotRunning(String url, String credentials) throws Ex timeout = 0; // stop looping } } catch (Exception se) { - if (SolrCLI.exceptionIsAuthRelated(se)) { + if (CLIUtils.exceptionIsAuthRelated(se)) { throw se; } return exitOrException(se.getMessage()); @@ -417,7 +417,7 @@ private static boolean isSolrRunningOn(String url, String credentials) throws Ex status.waitToSeeSolrUp(url, credentials, timeoutMs, TimeUnit.MILLISECONDS); return true; } catch (Exception se) { - if (SolrCLI.exceptionIsAuthRelated(se)) { + if (CLIUtils.exceptionIsAuthRelated(se)) { throw se; } return false; @@ -425,8 +425,8 @@ private static boolean isSolrRunningOn(String url, String credentials) throws Ex } private static boolean runningSolrIsCloud(String url, String credentials) throws Exception { - try (final SolrClient client = SolrCLI.getSolrClient(url, credentials)) { - return SolrCLI.isCloudMode(client); + try (final SolrClient client = CLIUtils.getSolrClient(url, credentials)) { + return CLIUtils.isCloudMode(client); } } diff --git a/solr/core/src/java/org/apache/solr/cli/AuthTool.java b/solr/core/src/java/org/apache/solr/cli/AuthTool.java index 817b0093748..6a71164a03d 100644 --- a/solr/core/src/java/org/apache/solr/cli/AuthTool.java +++ b/solr/core/src/java/org/apache/solr/cli/AuthTool.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.Console; -import java.io.File; import java.io.IOException; import java.io.PrintStream; import java.net.URL; @@ -30,7 +29,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; -import java.util.Base64; import java.util.List; import java.util.Locale; import java.util.stream.Collectors; @@ -39,7 +37,6 @@ import org.apache.commons.cli.Options; import org.apache.lucene.util.Constants; import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.util.StrUtils; import org.apache.solr.core.SolrCore; import org.apache.solr.security.Sha256AuthenticationProvider; import org.apache.zookeeper.KeeperException; @@ -52,7 +49,7 @@ public class AuthTool extends ToolBase { .longOpt("type") .hasArg() .desc( - "The authentication mechanism to enable (basicAuth or kerberos). Defaults to 'basicAuth'.") + "The authentication mechanism to enable (currently only basicAuth). Defaults to 'basicAuth'.") .build(); private static final Option PROMPT_OPTION = @@ -64,14 +61,6 @@ public class AuthTool extends ToolBase { "Prompts the user to provide the credentials. Use either --credentials or --prompt, not both.") .build(); - private static final Option CONFIG_OPTION = - Option.builder() - .longOpt("config") - .hasArgs() - .desc( - "Configuration parameters (Solr startup parameters). Required for Kerberos authentication.") - .build(); - private static final Option BLOCK_UNKNOWN_OPTION = Option.builder() .longOpt("block-unknown") @@ -125,14 +114,8 @@ public String getName() { @Override public String getUsage() { - // Jan, the return "" causes - // Exception in thread "main" java.lang.IllegalArgumentException: cmdLineSyntax not provided - // at org.apache.commons.cli.HelpFormatter.printHelp(HelpFormatter.java:584) - // however return " " seems okay. - // return " "; return "\n bin/solr auth enable [--type basicAuth] --credentials user:pass [--block-unknown ] [--update-include-file-only ] [-v]\n" + " bin/solr auth enable [--type basicAuth] --prompt [--block-unknown ] [--update-include-file-only ] [-v]\n" - + " bin/solr auth enable --type kerberos --config \\\"\\\" [--update-include-file-only ] [-v]\n" + " bin/solr auth disable [--update-include-file-only ] [-v]\n"; } @@ -152,7 +135,6 @@ public Options getOptions() { return super.getOptions() .addOption(TYPE_OPTION) .addOption(PROMPT_OPTION) - .addOption(CONFIG_OPTION) .addOption(BLOCK_UNKNOWN_OPTION) .addOption(SOLR_INCLUDE_FILE_OPTION) .addOption(UPDATE_INCLUDE_FILE_OPTION) @@ -176,118 +158,6 @@ private void ensureArgumentIsValidBooleanIfPresent(CommandLine cli, Option optio } } - private void handleKerberos(CommandLine cli) throws Exception { - String cmd = cli.getArgs()[0]; - boolean updateIncludeFileOnly = - Boolean.parseBoolean(cli.getOptionValue(UPDATE_INCLUDE_FILE_OPTION, "false")); - String securityJson = - "{" - + "\n \"authentication\":{" - + "\n \"class\":\"solr.KerberosPlugin\"" - + "\n }" - + "\n}"; - - switch (cmd) { - case "enable": - String zkHost = null; - boolean zkInaccessible = false; - - if (!updateIncludeFileOnly) { - try { - zkHost = SolrCLI.getZkHost(cli); - } catch (Exception ex) { - CLIO.out( - "Unable to access ZooKeeper. Please add the following security.json to ZooKeeper (in case of SolrCloud):\n" - + securityJson - + "Dsolr.httpclient.config=[basicAuthConfFile]\n"); - zkInaccessible = true; - } - if (zkHost == null) { - if (!zkInaccessible) { - CLIO.out( - "Unable to access ZooKeeper. Please add the following security.json to ZooKeeper (in case of SolrCloud):\n" - + securityJson - + "\n"); - zkInaccessible = true; - } - } - - // check if security is already enabled or not - if (!zkInaccessible) { - try (SolrZkClient zkClient = SolrCLI.getSolrZkClient(cli, zkHost)) { - checkSecurityJsonExists(zkClient); - } catch (Exception ex) { - CLIO.out( - "Unable to access ZooKeeper. Please add the following security.json to ZooKeeper (in case of SolrCloud):\n" - + securityJson - + "\n"); - zkInaccessible = true; - } - } - } - - if (!updateIncludeFileOnly) { - if (!zkInaccessible) { - echoIfVerbose("Uploading following security.json: " + securityJson); - try (SolrZkClient zkClient = SolrCLI.getSolrZkClient(cli, zkHost)) { - zkClient.setData( - "/security.json", securityJson.getBytes(StandardCharsets.UTF_8), true); - } catch (Exception ex) { - CLIO.out( - "Unable to access ZooKeeper. Please add the following security.json to ZooKeeper (in case of SolrCloud):\n" - + securityJson); - } - } - } - - String config = StrUtils.join(Arrays.asList(cli.getOptionValues(CONFIG_OPTION)), ' '); - // config is base64 encoded (to get around parsing problems), decode it - config = config.replace(" ", ""); - config = - new String( - Base64.getDecoder().decode(config.getBytes(StandardCharsets.UTF_8)), - StandardCharsets.UTF_8); - config = config.replace("\n", "").replace("\r", ""); - - String solrIncludeFilename = cli.getOptionValue(SOLR_INCLUDE_FILE_OPTION); - File includeFile = new File(solrIncludeFilename); - if (!includeFile.exists() || !includeFile.canWrite()) { - CLIO.out( - "Solr include file " + solrIncludeFilename + " doesn't exist or is not writeable."); - printAuthEnablingInstructions(config); - System.exit(0); - } - - // update the solr.in.sh file to contain the necessary authentication lines - updateIncludeFileEnableAuth(includeFile.toPath(), null, config); - echo( - "Successfully enabled Kerberos authentication; please restart any running Solr nodes."); - return; - case "disable": - clearSecurityJson(cli, updateIncludeFileOnly); - - solrIncludeFilename = cli.getOptionValue(SOLR_INCLUDE_FILE_OPTION); - includeFile = new File(solrIncludeFilename); - if (!includeFile.exists() || !includeFile.canWrite()) { - CLIO.out( - "Solr include file " + solrIncludeFilename + " doesn't exist or is not writeable."); - CLIO.out( - "Security has been disabled. Please remove any SOLR_AUTH_TYPE or SOLR_AUTHENTICATION_OPTS configuration from solr.in.sh/solr.in.cmd.\n"); - System.exit(0); - } - - // update the solr.in.sh file to comment out the necessary authentication lines - updateIncludeFileDisableAuth(includeFile.toPath()); - return; - default: - CLIO.out("Valid auth commands are: enable, disable."); - SolrCLI.exit(1); - } - - CLIO.out("Options not understood."); - SolrCLI.exit(1); - } - private void handleBasicAuth(CommandLine cli) throws Exception { String cmd = cli.getArgs()[0]; boolean prompt = Boolean.parseBoolean(cli.getOptionValue(PROMPT_OPTION, "false")); @@ -309,7 +179,7 @@ private void handleBasicAuth(CommandLine cli) throws Exception { if (!updateIncludeFileOnly) { try { - zkHost = SolrCLI.getZkHost(cli); + zkHost = CLIUtils.getZkHost(cli); } catch (Exception ex) { if (cli.hasOption(CommonCLIOptions.ZK_HOST_OPTION)) { CLIO.out( @@ -332,7 +202,7 @@ private void handleBasicAuth(CommandLine cli) throws Exception { } // check if security is already enabled or not - try (SolrZkClient zkClient = SolrCLI.getSolrZkClient(cli, zkHost)) { + try (SolrZkClient zkClient = CLIUtils.getSolrZkClient(cli, zkHost)) { checkSecurityJsonExists(zkClient); } } @@ -381,50 +251,46 @@ private void handleBasicAuth(CommandLine cli) throws Exception { if (!updateIncludeFileOnly) { echoIfVerbose("Uploading following security.json: " + securityJson); - try (SolrZkClient zkClient = SolrCLI.getSolrZkClient(cli, zkHost)) { + try (SolrZkClient zkClient = CLIUtils.getSolrZkClient(cli, zkHost)) { zkClient.setData("/security.json", securityJson.getBytes(StandardCharsets.UTF_8), true); } } String solrIncludeFilename = cli.getOptionValue(SOLR_INCLUDE_FILE_OPTION); - File includeFile = new File(solrIncludeFilename); - if (!includeFile.exists() || !includeFile.canWrite()) { + Path includeFile = Path.of(solrIncludeFilename); + if (Files.notExists(includeFile) || !Files.isWritable(includeFile)) { CLIO.out( "Solr include file " + solrIncludeFilename + " doesn't exist or is not writeable."); printAuthEnablingInstructions(username, password); System.exit(0); } String authConfDir = cli.getOptionValue(AUTH_CONF_DIR_OPTION); - File basicAuthConfFile = new File(authConfDir + File.separator + "basicAuth.conf"); + Path basicAuthConfFile = Path.of(authConfDir, "basicAuth.conf"); - if (!basicAuthConfFile.getParentFile().canWrite()) { - CLIO.out("Cannot write to file: " + basicAuthConfFile.getAbsolutePath()); + if (!Files.isWritable(basicAuthConfFile.getParent())) { + CLIO.out("Cannot write to file: " + basicAuthConfFile.toAbsolutePath()); printAuthEnablingInstructions(username, password); System.exit(0); } Files.writeString( - basicAuthConfFile.toPath(), + basicAuthConfFile, "httpBasicAuthUser=" + username + "\nhttpBasicAuthPassword=" + password, StandardCharsets.UTF_8); // update the solr.in.sh file to contain the necessary authentication lines - updateIncludeFileEnableAuth( - includeFile.toPath(), basicAuthConfFile.getAbsolutePath(), null); + updateIncludeFileEnableAuth(includeFile, basicAuthConfFile); final String successMessage = String.format( - Locale.ROOT, - "Successfully enabled basic auth with username [%s] and password [%s].", - username, - password); + Locale.ROOT, "Successfully enabled basic auth with username [%s].", username); echo(successMessage); return; case "disable": clearSecurityJson(cli, updateIncludeFileOnly); solrIncludeFilename = cli.getOptionValue(SOLR_INCLUDE_FILE_OPTION); - includeFile = new File(solrIncludeFilename); - if (!includeFile.exists() || !includeFile.canWrite()) { + includeFile = Path.of(solrIncludeFilename); + if (Files.notExists(includeFile) || !Files.isWritable(includeFile)) { CLIO.out( "Solr include file " + solrIncludeFilename + " doesn't exist or is not writeable."); CLIO.out( @@ -433,7 +299,7 @@ private void handleBasicAuth(CommandLine cli) throws Exception { } // update the solr.in.sh file to comment out the necessary authentication lines - updateIncludeFileDisableAuth(includeFile.toPath()); + updateIncludeFileDisableAuth(includeFile); return; default: CLIO.out("Valid auth commands are: enable, disable."); @@ -460,7 +326,7 @@ private void checkSecurityJsonExists(SolrZkClient zkClient) private void clearSecurityJson(CommandLine cli, boolean updateIncludeFileOnly) throws Exception { String zkHost; if (!updateIncludeFileOnly) { - zkHost = SolrCLI.getZkHost(cli); + zkHost = CLIUtils.getZkHost(cli); if (zkHost == null) { stdout.print("ZK Host not found. Solr should be running in cloud mode."); SolrCLI.exit(1); @@ -468,7 +334,7 @@ private void clearSecurityJson(CommandLine cli, boolean updateIncludeFileOnly) t echoIfVerbose("Uploading following security.json: {}"); - try (SolrZkClient zkClient = SolrCLI.getSolrZkClient(cli, zkHost)) { + try (SolrZkClient zkClient = CLIUtils.getSolrZkClient(cli, zkHost)) { zkClient.setData("/security.json", "{}".getBytes(StandardCharsets.UTF_8), true); } } @@ -498,26 +364,6 @@ private void printAuthEnablingInstructions(String username, String password) { } } - private void printAuthEnablingInstructions(String kerberosConfig) { - if (Constants.WINDOWS) { - CLIO.out( - "\nAdd the following lines to the solr.in.cmd file so that the solr.cmd script can use subsequently.\n"); - CLIO.out( - "set SOLR_AUTH_TYPE=kerberos\n" - + "set SOLR_AUTHENTICATION_OPTS=\"" - + kerberosConfig - + "\"\n"); - } else { - CLIO.out( - "\nAdd the following lines to the solr.in.sh file so that the ./solr script can use subsequently.\n"); - CLIO.out( - "SOLR_AUTH_TYPE=\"kerberos\"\n" - + "SOLR_AUTHENTICATION_OPTS=\"" - + kerberosConfig - + "\"\n"); - } - } - /** * This will update the include file (e.g. solr.in.sh / solr.in.cmd) with the authentication * parameters. @@ -525,13 +371,9 @@ private void printAuthEnablingInstructions(String kerberosConfig) { * @param includeFile The include file * @param basicAuthConfFile If basicAuth, the path of the file containing credentials. If not, * null. - * @param kerberosConfig If kerberos, the config string containing startup parameters. If not, - * null. */ - private void updateIncludeFileEnableAuth( - Path includeFile, String basicAuthConfFile, String kerberosConfig) throws IOException { - assert !(basicAuthConfFile != null - && kerberosConfig != null); // only one of the two needs to be populated + private void updateIncludeFileEnableAuth(Path includeFile, Path basicAuthConfFile) + throws IOException { List includeFileLines = Files.readAllLines(includeFile, StandardCharsets.UTF_8); for (int i = 0; i < includeFileLines.size(); i++) { String line = includeFileLines.get(i); @@ -558,17 +400,6 @@ private void updateIncludeFileEnableAuth( includeFileLines.add( "SOLR_AUTHENTICATION_OPTS=\"-Dsolr.httpclient.config=" + basicAuthConfFile + "\""); } - } else { // for kerberos - if (Constants.WINDOWS) { - includeFileLines.add("REM The following lines added by solr.cmd for enabling BasicAuth"); - includeFileLines.add("set SOLR_AUTH_TYPE=kerberos"); - includeFileLines.add( - "set SOLR_AUTHENTICATION_OPTS=\"-Dsolr.httpclient.config=basicAuthConfFile\""); - } else { - includeFileLines.add("# The following lines added by ./solr for enabling BasicAuth"); - includeFileLines.add("SOLR_AUTH_TYPE=\"kerberos\""); - includeFileLines.add("SOLR_AUTHENTICATION_OPTS=\"" + kerberosConfig + "\""); - } } String lines = includeFileLines.stream().collect(Collectors.joining(System.lineSeparator())); @@ -609,15 +440,13 @@ public void runImpl(CommandLine cli) throws Exception { ensureArgumentIsValidBooleanIfPresent(cli, UPDATE_INCLUDE_FILE_OPTION); String type = cli.getOptionValue(TYPE_OPTION, "basicAuth"); + // switch structure is here to support future auth options like oAuth switch (type) { case "basicAuth": handleBasicAuth(cli); break; - case "kerberos": - handleKerberos(cli); - break; default: - throw new IllegalStateException("Only type=basicAuth or kerberos supported at the moment."); + throw new IllegalStateException("Only type=basicAuth supported at the moment."); } } } diff --git a/solr/core/src/java/org/apache/solr/cli/CLIUtils.java b/solr/core/src/java/org/apache/solr/cli/CLIUtils.java new file mode 100644 index 00000000000..b3336eb2361 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/cli/CLIUtils.java @@ -0,0 +1,351 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cli; + +import static org.apache.solr.common.SolrException.ErrorCode.FORBIDDEN; +import static org.apache.solr.common.SolrException.ErrorCode.UNAUTHORIZED; +import static org.apache.solr.common.params.CommonParams.NAME; +import static org.apache.solr.common.params.CommonParams.SYSTEM_INFO_PATH; + +import java.io.IOException; +import java.net.SocketException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.apache.commons.cli.CommandLine; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.Http2SolrClient; +import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.util.EnvUtils; +import org.apache.solr.common.util.NamedList; + +/** Utility class that holds various helper methods for the CLI. */ +public final class CLIUtils { + + private CLIUtils() {} + + public static String RED = "\u001B[31m"; + + public static String GREEN = "\u001B[32m"; + + public static String YELLOW = "\u001B[33m"; + + private static final long MAX_WAIT_FOR_CORE_LOAD_NANOS = + TimeUnit.NANOSECONDS.convert(1, TimeUnit.MINUTES); + + public static String getDefaultSolrUrl() { + // note that ENV_VAR syntax (and the env vars too) are mapped to env.var sys props + String scheme = EnvUtils.getProperty("solr.url.scheme", "http"); + String host = EnvUtils.getProperty("solr.tool.host", "localhost"); + String port = EnvUtils.getProperty("jetty.port", "8983"); // from SOLR_PORT env + return String.format(Locale.ROOT, "%s://%s:%s", scheme.toLowerCase(Locale.ROOT), host, port); + } + + /** + * Determine if a request to Solr failed due to a communication error, which is generally + * retry-able. + */ + public static boolean checkCommunicationError(Exception exc) { + Throwable rootCause = SolrException.getRootCause(exc); + return (rootCause instanceof SolrServerException || rootCause instanceof SocketException); + } + + public static void checkCodeForAuthError(int code) { + if (code == UNAUTHORIZED.code || code == FORBIDDEN.code) { + throw new SolrException( + SolrException.ErrorCode.getErrorCode(code), + "Solr requires authentication for request. Please supply valid credentials. HTTP code=" + + code); + } + } + + public static boolean exceptionIsAuthRelated(Exception exc) { + return (exc instanceof SolrException + && Arrays.asList(UNAUTHORIZED.code, FORBIDDEN.code).contains(((SolrException) exc).code())); + } + + public static SolrClient getSolrClient(String solrUrl, String credentials, boolean barePath) { + // today we require all urls to end in /solr, however in the future we will need to support the + // /api url end point instead. Eventually we want to have this method always + // return a bare url, and then individual calls decide if they are /solr or /api + // The /solr/ check is because sometimes a full url is passed in, like + // http://localhost:8983/solr/films_shard1_replica_n1/. + if (!barePath && !solrUrl.endsWith("/solr") && !solrUrl.contains("/solr/")) { + solrUrl = solrUrl + "/solr"; + } + Http2SolrClient.Builder builder = + new Http2SolrClient.Builder(solrUrl) + .withMaxConnectionsPerHost(32) + .withKeyStoreReloadInterval(-1, TimeUnit.SECONDS) + .withOptionalBasicAuthCredentials(credentials); + + return builder.build(); + } + + /** + * Helper method for all the places where we assume a /solr on the url. + * + * @param solrUrl The solr url that you want the client for + * @param credentials The username:password for basic auth. + * @return The SolrClient + */ + public static SolrClient getSolrClient(String solrUrl, String credentials) { + return getSolrClient(solrUrl, credentials, false); + } + + public static SolrClient getSolrClient(CommandLine cli, boolean barePath) throws Exception { + String solrUrl = normalizeSolrUrl(cli); + String credentials = cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION); + return getSolrClient(solrUrl, credentials, barePath); + } + + public static SolrClient getSolrClient(CommandLine cli) throws Exception { + String solrUrl = normalizeSolrUrl(cli); + String credentials = cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION); + return getSolrClient(solrUrl, credentials, false); + } + + /** + * Strips off the end of solrUrl any /solr when a legacy solrUrl like http://localhost:8983/solr + * is used, and warns those users. In the future we'll have urls ending with /api as well. + * + * @param solrUrl The user supplied url to Solr. + * @return the solrUrl in the format that Solr expects to see internally. + */ + public static String normalizeSolrUrl(String solrUrl) { + return normalizeSolrUrl(solrUrl, true); + } + + /** + * Strips off the end of solrUrl any /solr when a legacy solrUrl like http://localhost:8983/solr + * is used, and optionally logs a warning. In the future we'll have urls ending with /api as well. + * + * @param solrUrl The user supplied url to Solr. + * @param logUrlFormatWarning If a warning message should be logged about the url format + * @return the solrUrl in the format that Solr expects to see internally. + */ + public static String normalizeSolrUrl(String solrUrl, boolean logUrlFormatWarning) { + if (solrUrl != null) { + URI uri = URI.create(solrUrl); + String urlPath = uri.getRawPath(); + if (urlPath != null && urlPath.contains("/solr")) { + String newSolrUrl = + uri.resolve(urlPath.substring(0, urlPath.lastIndexOf("/solr") + 1)).toString(); + if (logUrlFormatWarning) { + CLIO.err( + "WARNING: URLs provided to this tool needn't include Solr's context-root (e.g. \"/solr\"). Such URLs are deprecated and support for them will be removed in a future release. Correcting from [" + + solrUrl + + "] to [" + + newSolrUrl + + "]."); + } + solrUrl = newSolrUrl; + } + if (solrUrl.endsWith("/")) { + solrUrl = solrUrl.substring(0, solrUrl.length() - 1); + } + } + return solrUrl; + } + + /** + * Get the base URL of a live Solr instance from either the --solr-url command-line option or from + * ZooKeeper. + */ + public static String normalizeSolrUrl(CommandLine cli) throws Exception { + String solrUrl = cli.getOptionValue(CommonCLIOptions.SOLR_URL_OPTION); + + if (solrUrl == null) { + String zkHost = cli.getOptionValue(CommonCLIOptions.ZK_HOST_OPTION); + if (zkHost == null) { + solrUrl = getDefaultSolrUrl(); + CLIO.err( + "Neither --zk-host or --solr-url parameters provided so assuming solr url is " + + solrUrl + + "."); + } else { + try (CloudSolrClient cloudSolrClient = getCloudHttp2SolrClient(zkHost)) { + cloudSolrClient.connect(); + Set liveNodes = cloudSolrClient.getClusterState().getLiveNodes(); + if (liveNodes.isEmpty()) + throw new IllegalStateException( + "No live nodes found! Cannot determine 'solrUrl' from ZooKeeper: " + zkHost); + + String firstLiveNode = liveNodes.iterator().next(); + solrUrl = ZkStateReader.from(cloudSolrClient).getBaseUrlForNodeName(firstLiveNode); + solrUrl = normalizeSolrUrl(solrUrl, false); + } + } + } + solrUrl = normalizeSolrUrl(solrUrl); + return solrUrl; + } + + /** + * Get the ZooKeeper connection string from either the zk-host command-line option or by looking + * it up from a running Solr instance based on the solr-url option. + */ + public static String getZkHost(CommandLine cli) throws Exception { + + String zkHost = cli.getOptionValue(CommonCLIOptions.ZK_HOST_OPTION); + if (zkHost != null && !zkHost.isBlank()) { + return zkHost; + } + + try (SolrClient solrClient = getSolrClient(cli)) { + // hit Solr to get system info + NamedList systemInfo = + solrClient.request( + new GenericSolrRequest(SolrRequest.METHOD.GET, CommonParams.SYSTEM_INFO_PATH)); + + // convert raw JSON into user-friendly output + StatusTool statusTool = new StatusTool(); + Map status = statusTool.reportStatus(systemInfo, solrClient); + @SuppressWarnings("unchecked") + Map cloud = (Map) status.get("cloud"); + if (cloud != null) { + String zookeeper = (String) cloud.get("ZooKeeper"); + if (zookeeper.endsWith("(embedded)")) { + zookeeper = zookeeper.substring(0, zookeeper.length() - "(embedded)".length()); + } + zkHost = zookeeper; + } + } + + return zkHost; + } + + public static SolrZkClient getSolrZkClient(CommandLine cli, String zkHost) throws Exception { + if (zkHost == null) { + throw new IllegalStateException( + "Solr at " + + cli.getOptionValue(CommonCLIOptions.SOLR_URL_OPTION) + + " is running in standalone server mode, this command can only be used when running in SolrCloud mode.\n"); + } + return new SolrZkClient.Builder() + .withUrl(zkHost) + .withTimeout(SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT, TimeUnit.MILLISECONDS) + .build(); + } + + public static CloudHttp2SolrClient getCloudHttp2SolrClient(String zkHost) { + return getCloudHttp2SolrClient(zkHost, null); + } + + public static CloudHttp2SolrClient getCloudHttp2SolrClient( + String zkHost, Http2SolrClient.Builder builder) { + return new CloudHttp2SolrClient.Builder(Collections.singletonList(zkHost), Optional.empty()) + .withInternalClientBuilder(builder) + .build(); + } + + /** + * Extracts the port from the provided {@code solrUrl}. If a URL is provided with https scheme and + * not explicitly defines the port, the default port for HTTPS (443) is used. + * + *

If URL does not contain a port nor https as scheme, it falls back to port 80. + * + * @param solrUrl the URL to extract the port from + * @return The port that was found. + * @throws NullPointerException If solrUrl is null + * @throws URISyntaxException If the given string violates RFC 2396, as augmented by the above + * deviations + */ + public static int portFromUrl(String solrUrl) throws URISyntaxException { + URI uri = new URI(solrUrl); + int port = uri.getPort(); + if (port == -1) { + return uri.getScheme().equals("https") ? 443 : 80; + } else { + return port; + } + } + + public static boolean safeCheckCollectionExists( + String solrUrl, String collection, String credentials) { + boolean exists = false; + try (var solrClient = getSolrClient(solrUrl, credentials)) { + NamedList existsCheckResult = solrClient.request(new CollectionAdminRequest.List()); + @SuppressWarnings("unchecked") + List collections = (List) existsCheckResult.get("collections"); + exists = collections != null && collections.contains(collection); + } catch (Exception exc) { + // just ignore it since we're only interested in a positive result here + } + return exists; + } + + @SuppressWarnings("unchecked") + public static boolean safeCheckCoreExists(String solrUrl, String coreName, String credentials) { + boolean exists = false; + try (var solrClient = getSolrClient(solrUrl, credentials)) { + boolean wait = false; + final long startWaitAt = System.nanoTime(); + do { + if (wait) { + final int clamPeriodForStatusPollMs = 1000; + Thread.sleep(clamPeriodForStatusPollMs); + } + NamedList existsCheckResult = + CoreAdminRequest.getStatus(coreName, solrClient).getResponse(); + NamedList status = (NamedList) existsCheckResult.get("status"); + NamedList coreStatus = (NamedList) status.get(coreName); + Map failureStatus = + (Map) existsCheckResult.get("initFailures"); + String errorMsg = (String) failureStatus.get(coreName); + final boolean hasName = coreStatus != null && coreStatus.get(NAME) != null; + exists = hasName || errorMsg != null; + wait = hasName && errorMsg == null && "true".equals(coreStatus.get("isLoading")); + } while (wait && System.nanoTime() - startWaitAt < MAX_WAIT_FOR_CORE_LOAD_NANOS); + } catch (Exception exc) { + // just ignore it since we're only interested in a positive result here + } + return exists; + } + + public static boolean isCloudMode(SolrClient solrClient) throws SolrServerException, IOException { + NamedList systemInfo = + solrClient.request(new GenericSolrRequest(SolrRequest.METHOD.GET, SYSTEM_INFO_PATH)); + return "solrcloud".equals(systemInfo.get("mode")); + } + + public static Path getConfigSetsDir(Path solrInstallDir) { + Path configSetsPath = Paths.get("server/solr/configsets/"); + return solrInstallDir.resolve(configSetsPath); + } +} diff --git a/solr/core/src/java/org/apache/solr/cli/ClusterTool.java b/solr/core/src/java/org/apache/solr/cli/ClusterTool.java index 1c71b7b8fee..714654829cc 100644 --- a/solr/core/src/java/org/apache/solr/cli/ClusterTool.java +++ b/solr/core/src/java/org/apache/solr/cli/ClusterTool.java @@ -79,7 +79,7 @@ public void runImpl(CommandLine cli) throws Exception { String propertyName = cli.getOptionValue(PROPERTY_OPTION); String propertyValue = cli.getOptionValue(VALUE_OPTION); - String zkHost = SolrCLI.getZkHost(cli); + String zkHost = CLIUtils.getZkHost(cli); if (!ZkController.checkChrootPath(zkHost, true)) { throw new IllegalStateException( diff --git a/solr/core/src/java/org/apache/solr/cli/CommonCLIOptions.java b/solr/core/src/java/org/apache/solr/cli/CommonCLIOptions.java index 93f02c3e359..e6425abe1f3 100644 --- a/solr/core/src/java/org/apache/solr/cli/CommonCLIOptions.java +++ b/solr/core/src/java/org/apache/solr/cli/CommonCLIOptions.java @@ -47,7 +47,7 @@ private CommonCLIOptions() {} .argName("HOST") .desc( "Base Solr URL, which can be used to determine the zk-host if that's not known; defaults to: " - + SolrCLI.getDefaultSolrUrl() + + CLIUtils.getDefaultSolrUrl() + '.') .build(); diff --git a/solr/core/src/java/org/apache/solr/cli/ConfigSetDownloadTool.java b/solr/core/src/java/org/apache/solr/cli/ConfigSetDownloadTool.java index b55b939117b..264023640f7 100644 --- a/solr/core/src/java/org/apache/solr/cli/ConfigSetDownloadTool.java +++ b/solr/core/src/java/org/apache/solr/cli/ConfigSetDownloadTool.java @@ -79,13 +79,13 @@ public String getUsage() { @Override public void runImpl(CommandLine cli) throws Exception { - String zkHost = SolrCLI.getZkHost(cli); + String zkHost = CLIUtils.getZkHost(cli); String confName = cli.getOptionValue(CONF_NAME_OPTION); String confDir = cli.getOptionValue(CONF_DIR_OPTION); echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ..."); - try (SolrZkClient zkClient = SolrCLI.getSolrZkClient(cli, zkHost)) { + try (SolrZkClient zkClient = CLIUtils.getSolrZkClient(cli, zkHost)) { Path configSetPath = Paths.get(confDir); // we try to be nice about having the "conf" in the directory, and we create it if it's not // there. diff --git a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java index b959245af2e..a9fc0631e11 100644 --- a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java +++ b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java @@ -81,7 +81,7 @@ public String getUsage() { @Override public void runImpl(CommandLine cli) throws Exception { - String zkHost = SolrCLI.getZkHost(cli); + String zkHost = CLIUtils.getZkHost(cli); final String solrInstallDir = System.getProperty("solr.install.dir"); Path solrInstallDirPath = Paths.get(solrInstallDir); @@ -90,8 +90,8 @@ public void runImpl(CommandLine cli) throws Exception { String confDir = cli.getOptionValue(CONF_DIR_OPTION); echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ..."); - try (SolrZkClient zkClient = SolrCLI.getSolrZkClient(cli, zkHost)) { - final Path configsetsDirPath = SolrCLI.getConfigSetsDir(solrInstallDirPath); + try (SolrZkClient zkClient = CLIUtils.getSolrZkClient(cli, zkHost)) { + final Path configsetsDirPath = CLIUtils.getConfigSetsDir(solrInstallDirPath); Path confPath = ConfigSetService.getConfigsetPath(confDir, configsetsDirPath.toString()); echo( diff --git a/solr/core/src/java/org/apache/solr/cli/ConfigTool.java b/solr/core/src/java/org/apache/solr/cli/ConfigTool.java index fdd18f24d4b..e5e7c96791c 100644 --- a/solr/core/src/java/org/apache/solr/cli/ConfigTool.java +++ b/solr/core/src/java/org/apache/solr/cli/ConfigTool.java @@ -98,7 +98,7 @@ public Options getOptions() { @Override public void runImpl(CommandLine cli) throws Exception { - String solrUrl = SolrCLI.normalizeSolrUrl(cli); + String solrUrl = CLIUtils.normalizeSolrUrl(cli); String action = cli.getOptionValue(ACTION_OPTION, "set-property"); String collection = cli.getOptionValue(COLLECTION_NAME_OPTION); String property = cli.getOptionValue(PROPERTY_OPTION); @@ -127,7 +127,7 @@ public void runImpl(CommandLine cli) throws Exception { echoIfVerbose(jsonBody); try (SolrClient solrClient = - SolrCLI.getSolrClient(solrUrl, cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION))) { + CLIUtils.getSolrClient(solrUrl, cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION))) { NamedList result = SolrCLI.postJsonToSolr(solrClient, updatePath, jsonBody); Integer statusCode = (Integer) result.findRecursive("responseHeader", "status"); if (statusCode == 0) { diff --git a/solr/core/src/java/org/apache/solr/cli/CreateTool.java b/solr/core/src/java/org/apache/solr/cli/CreateTool.java index 5fc57501d93..8c46bc13c4f 100644 --- a/solr/core/src/java/org/apache/solr/cli/CreateTool.java +++ b/solr/core/src/java/org/apache/solr/cli/CreateTool.java @@ -22,7 +22,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Locale; -import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; import org.apache.commons.cli.CommandLine; @@ -138,8 +137,8 @@ public Options getOptions() { @Override public void runImpl(CommandLine cli) throws Exception { - try (var solrClient = SolrCLI.getSolrClient(cli)) { - if (SolrCLI.isCloudMode(solrClient)) { + try (var solrClient = CLIUtils.getSolrClient(cli)) { + if (CLIUtils.isCloudMode(solrClient)) { createCollection(cli); } else { createCore(cli, solrClient); @@ -150,7 +149,7 @@ public void runImpl(CommandLine cli) throws Exception { protected void createCore(CommandLine cli, SolrClient solrClient) throws Exception { String coreName = cli.getOptionValue(COLLECTION_NAME_OPTION); String solrUrl = - cli.getOptionValue(CommonCLIOptions.SOLR_URL_OPTION, SolrCLI.getDefaultSolrUrl()); + cli.getOptionValue(CommonCLIOptions.SOLR_URL_OPTION, CLIUtils.getDefaultSolrUrl()); final String solrInstallDir = System.getProperty("solr.install.dir"); final String confDirName = @@ -167,15 +166,14 @@ protected void createCore(CommandLine cli, SolrClient solrClient) throws Excepti String coreRootDirectory; // usually same as solr home, but not always - Map systemInfo = - solrClient - .request(new GenericSolrRequest(SolrRequest.METHOD.GET, CommonParams.SYSTEM_INFO_PATH)) - .asMap(); + NamedList systemInfo = + solrClient.request( + new GenericSolrRequest(SolrRequest.METHOD.GET, CommonParams.SYSTEM_INFO_PATH)); // convert raw JSON into user-friendly output coreRootDirectory = (String) systemInfo.get("core_root"); - if (SolrCLI.safeCheckCoreExists( + if (CLIUtils.safeCheckCoreExists( solrUrl, coreName, cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION))) { throw new IllegalArgumentException( "\nCore '" @@ -224,9 +222,9 @@ protected void createCollection(CommandLine cli) throws Exception { .withKeyStoreReloadInterval(-1, TimeUnit.SECONDS) .withOptionalBasicAuthCredentials( cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION)); - String zkHost = SolrCLI.getZkHost(cli); + String zkHost = CLIUtils.getZkHost(cli); echoIfVerbose("Connecting to ZooKeeper at " + zkHost); - try (CloudSolrClient cloudSolrClient = SolrCLI.getCloudHttp2SolrClient(zkHost, builder)) { + try (CloudSolrClient cloudSolrClient = CLIUtils.getCloudHttp2SolrClient(zkHost, builder)) { cloudSolrClient.connect(); createCollection(cloudSolrClient, cli); } @@ -277,7 +275,7 @@ protected void createCollection(CloudSolrClient cloudSolrClient, CommandLine cli } // TODO: This should be done using the configSet API - final Path configsetsDirPath = SolrCLI.getConfigSetsDir(solrInstallDirPath); + final Path configsetsDirPath = CLIUtils.getConfigSetsDir(solrInstallDirPath); ConfigSetService configSetService = new ZkConfigSetService(ZkStateReader.from(cloudSolrClient).getZkClient()); Path confPath = ConfigSetService.getConfigsetPath(confDir, configsetsDirPath.toString()); @@ -294,7 +292,7 @@ protected void createCollection(CloudSolrClient cloudSolrClient, CommandLine cli } // since creating a collection is a heavy-weight operation, check for existence first - if (SolrCLI.safeCheckCollectionExists( + if (CLIUtils.safeCheckCollectionExists( solrUrl, collectionName, cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION))) { throw new IllegalStateException( "\nCollection '" @@ -321,7 +319,7 @@ protected void createCollection(CloudSolrClient cloudSolrClient, CommandLine cli if (isVerbose()) { // pretty-print the response to stdout CharArr arr = new CharArr(); - new JSONWriter(arr, 2).write(response.asMap()); + new JSONWriter(arr, 2).write(response.asMap(10)); echo(arr.toString()); } String endMessage = @@ -339,7 +337,7 @@ protected void createCollection(CloudSolrClient cloudSolrClient, CommandLine cli } private Path getFullConfDir(Path solrInstallDir, Path confDirName) { - return SolrCLI.getConfigSetsDir(solrInstallDir).resolve(confDirName); + return CLIUtils.getConfigSetsDir(solrInstallDir).resolve(confDirName); } private void ensureConfDirExists(Path solrInstallDir, Path confDirName) { @@ -362,7 +360,7 @@ private void printDefaultConfigsetWarningIfNecessary(CommandLine cli) { && (confName.equals("") || confName.equals("_default"))) { final String collectionName = cli.getOptionValue(COLLECTION_NAME_OPTION); final String solrUrl = - cli.getOptionValue(CommonCLIOptions.SOLR_URL_OPTION, SolrCLI.getDefaultSolrUrl()); + cli.getOptionValue(CommonCLIOptions.SOLR_URL_OPTION, CLIUtils.getDefaultSolrUrl()); final String curlCommand = String.format( Locale.ROOT, diff --git a/solr/core/src/java/org/apache/solr/cli/DeleteTool.java b/solr/core/src/java/org/apache/solr/cli/DeleteTool.java index 61e02161416..c06e5ddf63c 100644 --- a/solr/core/src/java/org/apache/solr/cli/DeleteTool.java +++ b/solr/core/src/java/org/apache/solr/cli/DeleteTool.java @@ -57,9 +57,6 @@ public class DeleteTool extends ToolBase { private static final Option DELETE_CONFIG_OPTION = Option.builder() .longOpt("delete-config") - .hasArg() - .argName("true|false") - .type(Boolean.class) .desc( "Flag to indicate if the underlying configuration directory for a collection should also be deleted; default is true.") .build(); @@ -86,10 +83,8 @@ public String getName() { @Override public String getHeader() { - return "Deletes a core or collection depending on whether Solr is running in standalone (core) or SolrCloud" - + " mode (collection). If you're deleting a collection in SolrCloud mode, the default behavior is to also" - + " delete the configuration directory from Zookeeper so long as it is not being used by another collection.\n" - + " You can override this behavior by passing --delete-config false when running this command.\n" + return "Deletes a collection or core depending on whether Solr is running in SolrCloud or standalone mode. " + + "Deleting a collection does not delete it's configuration unless you pass in the --delete-config flag.\n" + "\n" + "List of options:"; } @@ -106,8 +101,8 @@ public Options getOptions() { @Override public void runImpl(CommandLine cli) throws Exception { - try (var solrClient = SolrCLI.getSolrClient(cli)) { - if (SolrCLI.isCloudMode(solrClient)) { + try (var solrClient = CLIUtils.getSolrClient(cli)) { + if (CLIUtils.isCloudMode(solrClient)) { deleteCollection(cli); } else { deleteCore(cli, solrClient); @@ -124,8 +119,8 @@ protected void deleteCollection(CommandLine cli) throws Exception { .withOptionalBasicAuthCredentials( cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION)); - String zkHost = SolrCLI.getZkHost(cli); - try (CloudSolrClient cloudSolrClient = SolrCLI.getCloudHttp2SolrClient(zkHost, builder)) { + String zkHost = CLIUtils.getZkHost(cli); + try (CloudSolrClient cloudSolrClient = CLIUtils.getCloudHttp2SolrClient(zkHost, builder)) { echoIfVerbose("Connecting to ZooKeeper at " + zkHost); cloudSolrClient.connect(); deleteCollection(cloudSolrClient, cli); @@ -148,7 +143,7 @@ protected void deleteCollection(CloudSolrClient cloudSolrClient, CommandLine cli String configName = zkStateReader.getClusterState().getCollection(collectionName).getConfigName(); - boolean deleteConfig = Boolean.parseBoolean(cli.getOptionValue(DELETE_CONFIG_OPTION, "true")); + boolean deleteConfig = cli.hasOption(DELETE_CONFIG_OPTION); if (deleteConfig && configName != null) { if (cli.hasOption(FORCE_OPTION)) { @@ -216,7 +211,7 @@ protected void deleteCollection(CloudSolrClient cloudSolrClient, CommandLine cli if (isVerbose() && response != null) { // pretty-print the response to stdout CharArr arr = new CharArr(); - new JSONWriter(arr, 2).write(response.asMap()); + new JSONWriter(arr, 2).write(response.asMap(10)); echo(arr.toString()); echo("\n"); } diff --git a/solr/core/src/java/org/apache/solr/cli/ExportTool.java b/solr/core/src/java/org/apache/solr/cli/ExportTool.java index 5018855e597..b5a58377ba3 100644 --- a/solr/core/src/java/org/apache/solr/cli/ExportTool.java +++ b/solr/core/src/java/org/apache/solr/cli/ExportTool.java @@ -25,7 +25,6 @@ import static org.apache.solr.common.util.JavaBinCodec.SOLRINPUTDOC; import java.io.BufferedOutputStream; -import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; @@ -284,7 +283,7 @@ public void runImpl(CommandLine cli) throws Exception { throw new IllegalArgumentException( "Must specify -c / --name parameter with --solr-url to post documents."); } - url = SolrCLI.normalizeSolrUrl(cli) + "/solr/" + cli.getOptionValue(COLLECTION_NAME_OPTION); + url = CLIUtils.normalizeSolrUrl(cli) + "/solr/" + cli.getOptionValue(COLLECTION_NAME_OPTION); } else { // think about support --zk-host someday. @@ -363,8 +362,7 @@ public synchronized void accept(SolrDocument doc) throws IOException { } } field = constructDateStr(field); - if (field instanceof List) { - List list = (List) field; + if (field instanceof List list) { if (hasdate(list)) { ArrayList listCopy = new ArrayList<>(list.size()); for (Object o : list) listCopy.add(constructDateStr(o)); @@ -443,8 +441,7 @@ public synchronized void accept(SolrDocument doc) throws IOException { } } field = constructDateStr(field); - if (field instanceof List) { - List list = (List) field; + if (field instanceof List list) { if (hasdate(list)) { ArrayList listCopy = new ArrayList<>(list.size()); for (Object o : list) listCopy.add(constructDateStr(o)); @@ -601,7 +598,7 @@ void exportDocs() throws Exception { consumerThreadpool.shutdownNow(); if (failed) { try { - Files.delete(new File(out).toPath()); + Files.delete(Path.of(out)); } catch (IOException e) { // ignore } @@ -663,8 +660,8 @@ class CoreHandler { } boolean exportDocsFromCore() throws IOException, SolrServerException { - - try (SolrClient client = SolrCLI.getSolrClient(baseurl, credentials)) { + // reference the replica's node URL, not the baseUrl in scope, which could be anywhere + try (SolrClient client = CLIUtils.getSolrClient(replica.getBaseUrl(), credentials)) { expectedDocs = getDocCount(replica.getCoreName(), client, query); QueryRequest request; ModifiableSolrParams params = new ModifiableSolrParams(); diff --git a/solr/core/src/java/org/apache/solr/cli/HealthcheckTool.java b/solr/core/src/java/org/apache/solr/cli/HealthcheckTool.java index 6ec08f77d6c..950ac3430b9 100644 --- a/solr/core/src/java/org/apache/solr/cli/HealthcheckTool.java +++ b/solr/core/src/java/org/apache/solr/cli/HealthcheckTool.java @@ -89,12 +89,12 @@ public HealthcheckTool(PrintStream stdout) { @Override public void runImpl(CommandLine cli) throws Exception { - String zkHost = SolrCLI.getZkHost(cli); + String zkHost = CLIUtils.getZkHost(cli); if (zkHost == null) { CLIO.err("Healthcheck tool only works in Solr Cloud mode."); System.exit(1); } - try (CloudHttp2SolrClient cloudSolrClient = SolrCLI.getCloudHttp2SolrClient(zkHost)) { + try (CloudHttp2SolrClient cloudSolrClient = CLIUtils.getCloudHttp2SolrClient(zkHost)) { echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ..."); cloudSolrClient.connect(); runCloudTool(cloudSolrClient, cli); @@ -125,7 +125,7 @@ protected void runCloudTool(CloudSolrClient cloudSolrClient, CommandLine cli) th SolrQuery q = new SolrQuery("*:*"); q.setRows(0); QueryResponse qr = cloudSolrClient.query(collection, q); - SolrCLI.checkCodeForAuthError(qr.getStatus()); + CLIUtils.checkCodeForAuthError(qr.getStatus()); String collErr = null; long docCount = -1; try { @@ -169,12 +169,12 @@ protected void runCloudTool(CloudSolrClient cloudSolrClient, CommandLine cli) th q.setRows(0); q.set(DISTRIB, "false"); try (var solrClientForCollection = - SolrCLI.getSolrClient( + CLIUtils.getSolrClient( coreUrl, cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION))) { qr = solrClientForCollection.query(q); numDocs = qr.getResults().getNumFound(); try (var solrClient = - SolrCLI.getSolrClient( + CLIUtils.getSolrClient( replicaCoreProps.getBaseUrl(), cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION))) { NamedList systemInfo = @@ -192,7 +192,7 @@ protected void runCloudTool(CloudSolrClient cloudSolrClient, CommandLine cli) th } catch (Exception exc) { log.error("ERROR: {} when trying to reach: {}", exc, coreUrl); - if (SolrCLI.checkCommunicationError(exc)) { + if (CLIUtils.checkCommunicationError(exc)) { replicaStatus = Replica.State.DOWN.toString(); } else { replicaStatus = "error: " + exc; @@ -286,8 +286,7 @@ public int hashCode() { public boolean equals(Object obj) { if (this == obj) return true; if (obj == null) return false; - if (!(obj instanceof ReplicaHealth)) return true; - ReplicaHealth that = (ReplicaHealth) obj; + if (!(obj instanceof ReplicaHealth that)) return true; return this.shard.equals(that.shard) && this.isLeader == that.isLeader; } diff --git a/solr/core/src/java/org/apache/solr/cli/LinkConfigTool.java b/solr/core/src/java/org/apache/solr/cli/LinkConfigTool.java index 2261b96474d..619e1d82e5c 100644 --- a/solr/core/src/java/org/apache/solr/cli/LinkConfigTool.java +++ b/solr/core/src/java/org/apache/solr/cli/LinkConfigTool.java @@ -78,7 +78,7 @@ public void runImpl(CommandLine cli) throws Exception { String collection = cli.getOptionValue(COLLECTION_NAME_OPTION); String confName = cli.getOptionValue(CONF_NAME_OPTION); - String zkHost = SolrCLI.getZkHost(cli); + String zkHost = CLIUtils.getZkHost(cli); try (SolrZkClient zkClient = new SolrZkClient.Builder() diff --git a/solr/core/src/java/org/apache/solr/cli/PackageTool.java b/solr/core/src/java/org/apache/solr/cli/PackageTool.java index cb8bc6493ad..b71c86896cb 100644 --- a/solr/core/src/java/org/apache/solr/cli/PackageTool.java +++ b/solr/core/src/java/org/apache/solr/cli/PackageTool.java @@ -115,8 +115,8 @@ public String getName() { + "don't print stack traces, hence special treatment is needed here.") public void runImpl(CommandLine cli) throws Exception { try { - String solrUrl = SolrCLI.normalizeSolrUrl(cli); - String zkHost = SolrCLI.getZkHost(cli); + String solrUrl = CLIUtils.normalizeSolrUrl(cli); + String zkHost = CLIUtils.getZkHost(cli); if (zkHost == null) { throw new SolrException(ErrorCode.INVALID_STATE, "Package manager runs only in SolrCloud"); } @@ -125,7 +125,7 @@ public void runImpl(CommandLine cli) throws Exception { String cmd = cli.getArgs()[0]; - try (SolrClient solrClient = SolrCLI.getSolrClient(cli, true)) { + try (SolrClient solrClient = CLIUtils.getSolrClient(cli, true)) { packageManager = new PackageManager(solrClient, solrUrl, zkHost); try { repositoryManager = new RepositoryManager(solrClient, packageManager); diff --git a/solr/core/src/java/org/apache/solr/cli/PostLogsTool.java b/solr/core/src/java/org/apache/solr/cli/PostLogsTool.java index e1efeb0b920..c14d9c216db 100644 --- a/solr/core/src/java/org/apache/solr/cli/PostLogsTool.java +++ b/solr/core/src/java/org/apache/solr/cli/PostLogsTool.java @@ -93,7 +93,7 @@ public Options getOptions() { public void runImpl(CommandLine cli) throws Exception { String url = null; if (cli.hasOption(CommonCLIOptions.SOLR_URL_OPTION)) { - url = SolrCLI.normalizeSolrUrl(cli) + "/solr/" + cli.getOptionValue(COLLECTION_NAME_OPTION); + url = CLIUtils.normalizeSolrUrl(cli) + "/solr/" + cli.getOptionValue(COLLECTION_NAME_OPTION); } else { // Could be required arg, but maybe we want to support --zk-host option too? diff --git a/solr/core/src/java/org/apache/solr/cli/PostTool.java b/solr/core/src/java/org/apache/solr/cli/PostTool.java index 0e50e619043..5ec8eb3eb7e 100644 --- a/solr/core/src/java/org/apache/solr/cli/PostTool.java +++ b/solr/core/src/java/org/apache/solr/cli/PostTool.java @@ -22,9 +22,6 @@ import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileFilter; -import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -41,10 +38,12 @@ import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; import java.security.GeneralSecurityException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Base64; +import java.util.Collection; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -54,8 +53,11 @@ import java.util.Map; import java.util.Set; import java.util.TimeZone; +import java.util.function.Predicate; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; +import java.util.stream.Collectors; +import java.util.stream.Stream; import java.util.zip.GZIPInputStream; import java.util.zip.Inflater; import java.util.zip.InflaterInputStream; @@ -202,7 +204,7 @@ public class PostTool extends ToolBase { private int currentDepth; static HashMap mimeMap; - FileFilter fileFilter; + Predicate fileFilter; // Backlog for crawling List> backlog = new ArrayList<>(); Set visited = new HashSet<>(); @@ -279,7 +281,7 @@ public void runImpl(CommandLine cli) throws Exception { solrUpdateUrl = null; if (cli.hasOption(CommonCLIOptions.SOLR_URL_OPTION)) { String url = - SolrCLI.normalizeSolrUrl(cli) + CLIUtils.normalizeSolrUrl(cli) + "/solr/" + cli.getOptionValue(COLLECTION_NAME_OPTION) + "/update"; @@ -287,7 +289,7 @@ public void runImpl(CommandLine cli) throws Exception { } else { String url = - SolrCLI.getDefaultSolrUrl() + CLIUtils.getDefaultSolrUrl() + "/solr/" + cli.getOptionValue(COLLECTION_NAME_OPTION) + "/update"; @@ -350,7 +352,7 @@ public void execute(String mode) throws SolrServerException, IOException { displayTiming((long) timer.getTime()); } - private void doFilesMode() { + private void doFilesMode() throws IOException { currentDepth = 0; info( @@ -434,8 +436,16 @@ private void displayTiming(long millis) { CLIO.out("Time spent: " + df.format(new Date(millis))); } - private boolean checkIsValidPath(File srcFile) { - return Files.exists(srcFile.toPath()); + private boolean checkIsValidPath(Path srcFile) { + return Files.exists(srcFile); + } + + private static Collection listFiles(Path directory, Predicate fileFilter) + throws IOException { + Predicate filter = fileFilter != null ? fileFilter : p -> true; + try (Stream directoryFiles = Files.list(directory)) { + return directoryFiles.filter(filter).collect(Collectors.toList()); + } } /** @@ -448,8 +458,8 @@ private boolean checkIsValidPath(File srcFile) { boolean recursionPossible(String[] args) { boolean recursionPossible = false; for (String arg : args) { - File f = new File(arg); - if (f.isDirectory()) { + Path f = Path.of(arg); + if (Files.isDirectory(f)) { recursionPossible = true; } } @@ -464,26 +474,29 @@ boolean recursionPossible(String[] args) { * @param out output stream to post data to * @param type default content-type to use when posting (this may be overridden in auto mode) * @return number of files posted + * @throws IOException if an I/O error occurs */ - public int postFiles(String[] args, int startIndexInArgs, OutputStream out, String type) { + public int postFiles(String[] args, int startIndexInArgs, OutputStream out, String type) + throws IOException { reset(); int filesPosted = 0; for (int j = startIndexInArgs; j < args.length; j++) { - File srcFile = new File(args[j]); - filesPosted = getFilesPosted(out, type, srcFile); + filesPosted = getFilesPosted(out, type, args[j]); } return filesPosted; } - private int getFilesPosted(final OutputStream out, final String type, final File srcFile) { + private int getFilesPosted(final OutputStream out, final String type, final String src) + throws IOException { int filesPosted = 0; + Path srcFile = Path.of(src).toAbsolutePath(); boolean isValidPath = checkIsValidPath(srcFile); - if (isValidPath && srcFile.isDirectory() && srcFile.canRead()) { + if (isValidPath && Files.isDirectory(srcFile) && Files.isReadable(srcFile)) { filesPosted += postDirectory(srcFile, out, type); - } else if (isValidPath && srcFile.isFile() && srcFile.canRead()) { - filesPosted += postFiles(new File[] {srcFile}, out, type); + } else if (isValidPath && Files.isRegularFile(srcFile) && Files.isReadable(srcFile)) { + filesPosted += postFiles(List.of(srcFile), out, type); } else { - filesPosted += handleGlob(srcFile, out, type); + filesPosted += handleGlob(src, out, type); } return filesPosted; } @@ -493,42 +506,40 @@ private int getFilesPosted(final OutputStream out, final String type, final File * * @return number of files posted total */ - private int postDirectory(File dir, OutputStream out, String type) { - if (dir.isHidden() && !dir.getName().equals(".")) { - return (0); + private int postDirectory(Path dir, OutputStream out, String type) throws IOException { + if (Files.isHidden(dir) && !dir.getFileName().toString().equals(".")) { + return 0; } info( "Indexing directory " - + dir.getPath() + + dir + " (" - + dir.listFiles(fileFilter).length + + listFiles(dir, fileFilter).size() + " files, depth=" + currentDepth + ")"); int posted = 0; - posted += postFiles(dir.listFiles(fileFilter), out, type); + posted += postFiles(listFiles(dir, fileFilter), out, type); if (recursive > currentDepth) { - for (File d : dir.listFiles()) { - if (d.isDirectory()) { - currentDepth++; - posted += postDirectory(d, out, type); - currentDepth--; - } + for (Path d : listFiles(dir, Files::isDirectory)) { + currentDepth++; + posted += postDirectory(d, out, type); + currentDepth--; } } return posted; } /** - * Posts a list of file names + * Posts a collection of files identified by their paths * * @return number of files posted */ - int postFiles(File[] files, OutputStream out, String type) { + int postFiles(Collection files, OutputStream out, String type) throws IOException { int filesPosted = 0; - for (File srcFile : files) { + for (Path srcFile : files) { try { - if (!srcFile.isFile() || srcFile.isHidden()) { + if (!Files.isRegularFile(srcFile) || Files.isHidden(srcFile)) { continue; } postFile(srcFile, out, type); @@ -544,22 +555,24 @@ int postFiles(File[] files, OutputStream out, String type) { /** * This only handles file globs not full path globbing. * - * @param globFile file holding glob path + * @param globPathPattern glob pattern * @param out outputStream to write results to * @param type default content-type to use when posting (this may be overridden in auto mode) * @return number of files posted + * @throws IOException if an I/O error occurs */ - int handleGlob(File globFile, OutputStream out, String type) { + int handleGlob(String globPathPattern, OutputStream out, String type) throws IOException { int filesPosted = 0; - File parent = globFile.getParentFile(); + Path globPath = Path.of(globPathPattern); + Path parent = globPath.getParent(); if (parent == null) { - parent = new File("."); + parent = Path.of("."); } - String fileGlob = globFile.getName(); - PostTool.GlobFileFilter ff = new PostTool.GlobFileFilter(fileGlob, false); - File[] fileList = parent.listFiles(ff); - if (fileList == null || fileList.length == 0) { - warn("No files or directories matching " + globFile); + String fileGlob = globPath.getFileName().toString(); + GlobFilter ff = new GlobFilter(fileGlob, false); + Collection fileList = listFiles(parent, ff); + if (fileList.isEmpty()) { + warn("No files or directories matching " + globPath); } else { filesPosted = postFiles(fileList, out, type); } @@ -765,7 +778,7 @@ public void commit() throws IOException, SolrServerException { info("COMMITting Solr index changes to " + solrUpdateUrl + "..."); String url = solrUpdateUrl.toString(); url = url.substring(0, url.lastIndexOf("/update")); - try (final SolrClient client = SolrCLI.getSolrClient(url, credentials)) { + try (final SolrClient client = CLIUtils.getSolrClient(url, credentials)) { client.commit(); } } @@ -775,7 +788,7 @@ public void optimize() throws IOException, SolrServerException { info("Performing an OPTIMIZE to " + solrUpdateUrl + "..."); String url = solrUpdateUrl.toString(); url = url.substring(0, url.lastIndexOf("/update")); - try (final SolrClient client = SolrCLI.getSolrClient(url, credentials)) { + try (final SolrClient client = CLIUtils.getSolrClient(url, credentials)) { client.optimize(); } } @@ -810,7 +823,7 @@ public static String appendParam(String url, String param) { } /** Opens the file and posts its contents to the solrUrl, writes to response to output. */ - public void postFile(File file, OutputStream output, String type) + public void postFile(Path file, OutputStream output, String type) throws MalformedURLException, URISyntaxException { InputStream is = null; @@ -838,11 +851,14 @@ public void postFile(File file, OutputStream output, String type) if (!urlStr.contains("resource.name")) { urlStr = appendParam( - urlStr, "resource.name=" + URLEncoder.encode(file.getAbsolutePath(), UTF_8)); + urlStr, + "resource.name=" + URLEncoder.encode(file.toAbsolutePath().toString(), UTF_8)); } if (!urlStr.contains("literal.id")) { urlStr = - appendParam(urlStr, "literal.id=" + URLEncoder.encode(file.getAbsolutePath(), UTF_8)); + appendParam( + urlStr, + "literal.id=" + URLEncoder.encode(file.toAbsolutePath().toString(), UTF_8)); } uri = new URI(urlStr); } @@ -854,7 +870,7 @@ public void postFile(File file, OutputStream output, String type) if (dryRun) { info( "DRY RUN of POSTing file " - + file.getName() + + file.getFileName() + (auto ? " (" + type + ")" : "") + " to [base]" + suffix); @@ -862,12 +878,12 @@ public void postFile(File file, OutputStream output, String type) try { info( "POSTing file " - + file.getName() + + file.getFileName() + (auto ? " (" + type + ")" : "") + " to [base]" + suffix); - is = new FileInputStream(file); - postData(is, file.length(), output, type, uri); + is = Files.newInputStream(file); + postData(is, Files.size(file), output, type, uri); } catch (IOException e) { warn("Can't open/read file: " + file); } finally { @@ -909,11 +925,11 @@ protected static URI appendUrlPath(URI uri, String append) { * Guesses the type of file, based on file name suffix Returns "application/octet-stream" if no * corresponding mimeMap type. * - * @param file the file + * @param path path to the file * @return the content-type guessed */ - protected static String guessType(File file) { - String name = file.getName(); + protected static String guessType(Path path) { + String name = path.getFileName().toString(); String suffix = name.substring(name.lastIndexOf('.') + 1); String type = mimeMap.get(suffix.toLowerCase(Locale.ROOT)); return (type != null) ? type : "application/octet-stream"; @@ -930,7 +946,7 @@ public boolean postData( return true; } - if (params.length() > 0) { + if (!params.isEmpty()) { try { uri = new URI(appendParam(uri.toString(), params)); } catch (URISyntaxException e) { @@ -1078,14 +1094,14 @@ private static void pipe(InputStream source, OutputStream dest) throws IOExcepti dest.flush(); } - public FileFilter getFileFilterFromFileTypes(String fileTypes) { + public Predicate getFileFilterFromFileTypes(String fileTypes) { String glob; if (fileTypes.equals("*")) { glob = ".*"; } else { glob = "^.*\\.(" + fileTypes.replace(",", "|") + ")$"; } - return new PostTool.GlobFileFilter(glob, true); + return new GlobFilter(glob, true); } // @@ -1131,10 +1147,10 @@ public static Document makeDom(byte[] in) } /** Inner class to filter files based on glob wildcards */ - static class GlobFileFilter implements FileFilter { + static class GlobFilter implements Predicate { private final Pattern p; - public GlobFileFilter(String pattern, boolean isRegex) { + public GlobFilter(String pattern, boolean isRegex) { String _pattern = pattern; if (!isRegex) { _pattern = @@ -1159,8 +1175,8 @@ public GlobFileFilter(String pattern, boolean isRegex) { } @Override - public boolean accept(File file) { - return p.matcher(file.getName()).find(); + public boolean test(Path path) { + return p.matcher(path.getFileName().toString()).find(); } } diff --git a/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java b/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java index 99307cf74f0..a3e1cc39f71 100644 --- a/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java +++ b/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java @@ -17,7 +17,6 @@ package org.apache.solr.cli; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; @@ -25,6 +24,8 @@ import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -43,10 +44,9 @@ import org.apache.commons.exec.Executor; import org.apache.commons.exec.OS; import org.apache.commons.exec.environment.EnvironmentUtils; -import org.apache.commons.io.FileUtils; +import org.apache.commons.io.file.PathUtils; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.impl.Http2SolrClient; import org.apache.solr.common.SolrException; import org.noggit.CharArr; import org.noggit.JSONWriter; @@ -163,8 +163,8 @@ public class RunExampleTool extends ToolBase { protected InputStream userInput; protected Executor executor; protected String script; - protected File serverDir; - protected File exampleDir; + protected Path serverDir; + protected Path exampleDir; protected String urlScheme; /** Default constructor used by the framework when running as a command-line application. */ @@ -205,26 +205,26 @@ public Options getOptions() { public void runImpl(CommandLine cli) throws Exception { this.urlScheme = cli.getOptionValue(URL_SCHEME_OPTION, "http"); - serverDir = new File(cli.getOptionValue(SERVER_DIR_OPTION)); - if (!serverDir.isDirectory()) + serverDir = Path.of(cli.getOptionValue(SERVER_DIR_OPTION)); + if (!Files.isDirectory(serverDir)) throw new IllegalArgumentException( "Value of --server-dir option is invalid! " - + serverDir.getAbsolutePath() + + serverDir.toAbsolutePath() + " is not a directory!"); script = cli.getOptionValue(SCRIPT_OPTION); if (script != null) { - if (!(new File(script)).isFile()) + if (!Files.isRegularFile(Path.of(script))) throw new IllegalArgumentException( "Value of --script option is invalid! " + script + " not found"); } else { - File scriptFile = new File(serverDir.getParentFile(), "bin/solr"); - if (scriptFile.isFile()) { - script = scriptFile.getAbsolutePath(); + Path scriptFile = serverDir.getParent().resolve("bin").resolve("solr"); + if (Files.isRegularFile(scriptFile)) { + script = scriptFile.toAbsolutePath().toString(); } else { - scriptFile = new File(serverDir.getParentFile(), "bin/solr.cmd"); - if (scriptFile.isFile()) { - script = scriptFile.getAbsolutePath(); + scriptFile = serverDir.getParent().resolve("bin").resolve("solr.cmd"); + if (Files.isRegularFile(scriptFile)) { + script = scriptFile.toAbsolutePath().toString(); } else { throw new IllegalArgumentException( "Cannot locate the bin/solr script! Please pass --script to this application."); @@ -234,19 +234,19 @@ public void runImpl(CommandLine cli) throws Exception { exampleDir = (cli.hasOption(EXAMPLE_DIR_OPTION)) - ? new File(cli.getOptionValue(EXAMPLE_DIR_OPTION)) - : new File(serverDir.getParent(), "example"); - if (!exampleDir.isDirectory()) + ? Path.of(cli.getOptionValue(EXAMPLE_DIR_OPTION)) + : serverDir.getParent().resolve("example"); + if (!Files.isDirectory(exampleDir)) throw new IllegalArgumentException( "Value of --example-dir option is invalid! " - + exampleDir.getAbsolutePath() + + exampleDir.toAbsolutePath() + " is not a directory!"); echoIfVerbose( "Running with\nserverDir=" - + serverDir.getAbsolutePath() + + serverDir.toAbsolutePath() + ",\nexampleDir=" - + exampleDir.getAbsolutePath() + + exampleDir.toAbsolutePath() + "\nscript=" + script); @@ -266,7 +266,6 @@ public void runImpl(CommandLine cli) throws Exception { } protected void runExample(CommandLine cli, String exampleName) throws Exception { - File exDir = setupExampleDir(serverDir, exampleDir, exampleName); String collectionName = "schemaless".equals(exampleName) ? "gettingstarted" : exampleName; String configSet = "techproducts".equals(exampleName) ? "sample_techproducts_configs" : "_default"; @@ -277,16 +276,16 @@ protected void runExample(CommandLine cli, String exampleName) throws Exception Integer.parseInt( cli.getOptionValue(PORT_OPTION, System.getenv().getOrDefault("SOLR_PORT", "8983"))); Map nodeStatus = - startSolr(new File(exDir, "solr"), isCloudMode, cli, port, zkHost, 30); + startSolr(serverDir.resolve("solr"), isCloudMode, cli, port, zkHost, 30); - String solrUrl = (String) nodeStatus.get("baseUrl"); + String solrUrl = CLIUtils.normalizeSolrUrl((String) nodeStatus.get("baseUrl")); // If the example already exists then let the user know they should delete it, or // they may get unusual behaviors. boolean alreadyExists = false; boolean cloudMode = nodeStatus.get("cloud") != null; if (cloudMode) { - if (SolrCLI.safeCheckCollectionExists( + if (CLIUtils.safeCheckCollectionExists( solrUrl, collectionName, cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION))) { alreadyExists = true; echo( @@ -296,7 +295,7 @@ protected void runExample(CommandLine cli, String exampleName) throws Exception } } else { String coreName = collectionName; - if (SolrCLI.safeCheckCoreExists( + if (CLIUtils.safeCheckCoreExists( solrUrl, coreName, cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION))) { alreadyExists = true; echo( @@ -310,7 +309,7 @@ protected void runExample(CommandLine cli, String exampleName) throws Exception echo( "You may want to run 'bin/solr delete -c " + collectionName - + "' first before running the example to ensure a fresh state."); + + " --delete-config' first before running the example to ensure a fresh state."); } if (!alreadyExists) { @@ -333,16 +332,16 @@ protected void runExample(CommandLine cli, String exampleName) throws Exception if ("techproducts".equals(exampleName) && !alreadyExists) { - File exampledocsDir = new File(exampleDir, "exampledocs"); - if (!exampledocsDir.isDirectory()) { - File readOnlyExampleDir = new File(serverDir.getParentFile(), "example"); - if (readOnlyExampleDir.isDirectory()) { - exampledocsDir = new File(readOnlyExampleDir, "exampledocs"); + Path exampledocsDir = this.exampleDir.resolve("exampledocs"); + if (!Files.isDirectory(exampledocsDir)) { + Path readOnlyExampleDir = serverDir.resolveSibling("example"); + if (Files.isDirectory(readOnlyExampleDir)) { + exampledocsDir = readOnlyExampleDir.resolve("exampledocs"); } } - if (exampledocsDir.isDirectory()) { - echo("Indexing tech product example docs from " + exampledocsDir.getAbsolutePath()); + if (Files.isDirectory(exampledocsDir)) { + echo("Indexing tech product example docs from " + exampledocsDir.toAbsolutePath()); String[] args = new String[] { @@ -353,7 +352,9 @@ protected void runExample(CommandLine cli, String exampleName) throws Exception collectionName, "--type", "application/xml", - exampledocsDir.getAbsolutePath() + "/*.xml" + "--filetypes", + "xml", + exampledocsDir.toAbsolutePath().toString() }; PostTool postTool = new PostTool(); CommandLine postToolCli = SolrCLI.parseCmdLine(postTool, args); @@ -364,7 +365,9 @@ protected void runExample(CommandLine cli, String exampleName) throws Exception "exampledocs directory not found, skipping indexing step for the techproducts example"); } } else if ("films".equals(exampleName) && !alreadyExists) { - try (SolrClient solrClient = new Http2SolrClient.Builder(solrUrl).build()) { + try (SolrClient solrClient = + CLIUtils.getSolrClient( + solrUrl, cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION))) { echo("Adding dense vector field type to films schema"); SolrCLI.postJsonToSolr( solrClient, @@ -424,8 +427,8 @@ protected void runExample(CommandLine cli, String exampleName) throws Exception + " }\n" + " }\n"); - File filmsJsonFile = new File(exampleDir, "films/films.json"); - echo("Indexing films example docs from " + filmsJsonFile.getAbsolutePath()); + Path filmsJsonFile = this.exampleDir.resolve("films").resolve("films.json"); + echo("Indexing films example docs from " + filmsJsonFile.toAbsolutePath()); String[] args = new String[] { "post", @@ -435,7 +438,7 @@ protected void runExample(CommandLine cli, String exampleName) throws Exception collectionName, "--type", "application/json", - filmsJsonFile.getAbsolutePath() + filmsJsonFile.toAbsolutePath().toString() }; PostTool postTool = new PostTool(); CommandLine postToolCli = SolrCLI.parseCmdLine(postTool, args); @@ -466,12 +469,12 @@ protected void runCloudExample(CommandLine cli) throws Exception { // Override the old default port numbers if user has started the example overriding SOLR_PORT cloudPorts = new int[] {defaultPort, defaultPort + 1, defaultPort + 2, defaultPort + 3}; } - File cloudDir = new File(exampleDir, "cloud"); - if (!cloudDir.isDirectory()) cloudDir.mkdir(); + Path cloudDir = exampleDir.resolve("cloud"); + if (!Files.isDirectory(cloudDir)) Files.createDirectory(cloudDir); echo("\nWelcome to the SolrCloud example!\n"); - Scanner readInput = prompt ? new Scanner(userInput, StandardCharsets.UTF_8.name()) : null; + Scanner readInput = prompt ? new Scanner(userInput, StandardCharsets.UTF_8) : null; if (prompt) { echo( "This interactive session will help you launch a SolrCloud cluster on your local workstation."); @@ -513,14 +516,14 @@ protected void runCloudExample(CommandLine cli) throws Exception { } // setup a unique solr.solr.home directory for each node - File node1Dir = setupExampleDir(serverDir, cloudDir, "node1"); + Path node1Dir = setupExampleDir(serverDir, cloudDir, "node1"); for (int n = 2; n <= numNodes; n++) { - File nodeNDir = new File(cloudDir, "node" + n); - if (!nodeNDir.isDirectory()) { - echo("Cloning " + node1Dir.getAbsolutePath() + " into\n " + nodeNDir.getAbsolutePath()); - FileUtils.copyDirectory(node1Dir, nodeNDir); + Path nodeNDir = cloudDir.resolve("node" + n); + if (!Files.isDirectory(nodeNDir)) { + echo("Cloning " + node1Dir.toAbsolutePath() + " into\n " + nodeNDir.toAbsolutePath()); + PathUtils.copyDirectory(node1Dir, nodeNDir, StandardCopyOption.REPLACE_EXISTING); } else { - echo(nodeNDir.getAbsolutePath() + " already exists."); + echo(nodeNDir.toAbsolutePath() + " already exists."); } } @@ -529,7 +532,7 @@ protected void runCloudExample(CommandLine cli) throws Exception { // start the first node (most likely with embedded ZK) Map nodeStatus = - startSolr(new File(node1Dir, "solr"), true, cli, cloudPorts[0], zkHost, 30); + startSolr(node1Dir.resolve("solr"), true, cli, cloudPorts[0], zkHost, 30); if (zkHost == null) { @SuppressWarnings("unchecked") @@ -546,11 +549,15 @@ protected void runCloudExample(CommandLine cli) throws Exception { // start the other nodes for (int n = 1; n < numNodes; n++) startSolr( - new File(cloudDir, "node" + (n + 1) + "/solr"), true, cli, cloudPorts[n], zkHost, 30); + cloudDir.resolve("node" + (n + 1)).resolve("solr"), + true, + cli, + cloudPorts[n], + zkHost, + 30); } - String solrUrl = (String) nodeStatus.get("baseUrl"); - if (solrUrl.endsWith("/")) solrUrl = solrUrl.substring(0, solrUrl.length() - 1); + String solrUrl = CLIUtils.normalizeSolrUrl((String) nodeStatus.get("baseUrl"), false); // wait until live nodes == numNodes waitToSeeLiveNodes(zkHost, numNodes); @@ -606,7 +613,7 @@ protected void waitToSeeLiveNodes(String zkHost, int numNodes) { } protected Map startSolr( - File solrHomeDir, + Path solrHomeDir, boolean cloudMode, CommandLine cli, int port, @@ -629,23 +636,28 @@ protected Map startSolr( String jvmOpts = cli.getOptionValue(JVM_OPTS_OPTION); String jvmOptsArg = (jvmOpts != null) ? " --jvm-opts \"" + jvmOpts + "\"" : ""; - File cwd = new File(System.getProperty("user.dir")); - File binDir = (new File(script)).getParentFile(); + Path cwd = Path.of(System.getProperty("user.dir")); + Path binDir = Path.of(script).getParent(); boolean isWindows = (OS.isFamilyDOS() || OS.isFamilyWin9x() || OS.isFamilyWindows()); - String callScript = (!isWindows && cwd.equals(binDir.getParentFile())) ? "bin/solr" : script; + String callScript = (!isWindows && cwd.equals(binDir.getParent())) ? "bin/solr" : script; - String cwdPath = cwd.getAbsolutePath(); - String solrHome = solrHomeDir.getAbsolutePath(); + String cwdPath = cwd.toAbsolutePath().toString(); + String solrHome = solrHomeDir.toAbsolutePath().toString(); // don't display a huge path for solr home if it is relative to the cwd if (!isWindows && cwdPath.length() > 1 && solrHome.startsWith(cwdPath)) solrHome = solrHome.substring(cwdPath.length() + 1); + final var syspropArg = + ("techproducts".equals(cli.getOptionValue(EXAMPLE_OPTION))) + ? "-Dsolr.modules=clustering,extraction,langid,ltr,scripting -Dsolr.ltr.enabled=true -Dsolr.clustering.enabled=true" + : ""; + String startCmd = String.format( Locale.ROOT, - "\"%s\" start %s -p %d --solr-home \"%s\" %s %s %s %s %s %s %s", + "\"%s\" start %s -p %d --solr-home \"%s\" %s %s %s %s %s %s %s %s", callScript, cloudModeArg, port, @@ -656,7 +668,8 @@ protected Map startSolr( forceArg, verboseArg, extraArgs, - jvmOptsArg); + jvmOptsArg, + syspropArg); startCmd = startCmd.replaceAll("\\s+", " ").trim(); // for pretty printing echo("\nStarting up Solr on port " + port + " using command:"); @@ -720,7 +733,7 @@ protected Map startSolr( } protected Map checkPortConflict( - String solrUrl, String credentials, File solrHomeDir, int port) { + String solrUrl, String credentials, Path solrHomeDir, int port) { // quickly check if the port is in use if (isPortAvailable(port)) return null; // not in use ... try to start @@ -734,7 +747,7 @@ protected Map checkPortConflict( if (nodeStatus != null) { String solr_home = (String) nodeStatus.get("solr_home"); if (solr_home != null) { - String solrHomePath = solrHomeDir.getAbsolutePath(); + String solrHomePath = solrHomeDir.toAbsolutePath().toString(); if (!solrHomePath.endsWith("/")) solrHomePath += "/"; if (!solr_home.endsWith("/")) solr_home += "/"; @@ -783,7 +796,7 @@ protected String createCloudExampleCollection( String cloudConfig = "_default"; String collectionName = "gettingstarted"; - File configsetsDir = new File(serverDir, "solr/configsets"); + Path configsetsDir = serverDir.resolve("solr").resolve("configsets"); if (prompt) { echo( @@ -800,7 +813,7 @@ protected String createCloudExampleCollection( // Test for existence and then prompt to either create another collection or skip the // creation step - if (SolrCLI.safeCheckCollectionExists(solrUrl, credentials, collectionName)) { + if (CLIUtils.safeCheckCollectionExists(solrUrl, credentials, collectionName)) { echo("\nCollection '" + collectionName + "' already exists!"); int oneOrTwo = promptForInt( @@ -856,7 +869,7 @@ protected String createCloudExampleCollection( } } else { // must verify if default collection exists - if (SolrCLI.safeCheckCollectionExists(solrUrl, collectionName, credentials)) { + if (CLIUtils.safeCheckCollectionExists(solrUrl, collectionName, credentials)) { echo( "\nCollection '" + collectionName @@ -886,13 +899,13 @@ protected String createCloudExampleCollection( return collectionName; } - protected boolean isValidConfig(File configsetsDir, String config) { - File configDir = new File(configsetsDir, config); - if (configDir.isDirectory()) return true; + protected boolean isValidConfig(Path configsetsDir, String config) { + Path configDir = configsetsDir.resolve(config); + if (Files.isDirectory(configDir)) return true; // not a built-in configset ... maybe it's a custom directory? - configDir = new File(config); - return configDir.isDirectory(); + configDir = Path.of(config); + return Files.isDirectory(configDir); } protected Map getNodeStatus(String solrUrl, String credentials, int maxWaitSecs) @@ -914,37 +927,37 @@ protected Map getNodeStatus(String solrUrl, String credentials, return nodeStatus; } - protected File setupExampleDir(File serverDir, File exampleParentDir, String dirName) + protected Path setupExampleDir(Path serverDir, Path exampleParentDir, String dirName) throws IOException { - File solrXml = new File(serverDir, "solr/solr.xml"); - if (!solrXml.isFile()) + Path solrXml = serverDir.resolve("solr").resolve("solr.xml"); + if (!Files.isRegularFile(solrXml)) throw new IllegalArgumentException( - "Value of --server-dir option is invalid! " + solrXml.getAbsolutePath() + " not found!"); + "Value of --server-dir option is invalid! " + solrXml.toAbsolutePath() + " not found!"); - File zooCfg = new File(serverDir, "solr/zoo.cfg"); - if (!zooCfg.isFile()) + Path zooCfg = serverDir.resolve("solr").resolve("zoo.cfg"); + if (!Files.isRegularFile(zooCfg)) throw new IllegalArgumentException( - "Value of --server-dir option is invalid! " + zooCfg.getAbsolutePath() + " not found!"); + "Value of --server-dir option is invalid! " + zooCfg.toAbsolutePath() + " not found!"); - File solrHomeDir = new File(exampleParentDir, dirName + "/solr"); - if (!solrHomeDir.isDirectory()) { + Path solrHomeDir = exampleParentDir.resolve(dirName).resolve("solr"); + if (!Files.isDirectory(solrHomeDir)) { echo("Creating Solr home directory " + solrHomeDir); - solrHomeDir.mkdirs(); + Files.createDirectories(solrHomeDir); } else { - echo("Solr home directory " + solrHomeDir.getAbsolutePath() + " already exists."); + echo("Solr home directory " + solrHomeDir.toAbsolutePath() + " already exists."); } - copyIfNeeded(solrXml, new File(solrHomeDir, "solr.xml")); - copyIfNeeded(zooCfg, new File(solrHomeDir, "zoo.cfg")); + copyIfNeeded(solrXml, solrHomeDir.resolve("solr.xml")); + copyIfNeeded(zooCfg, solrHomeDir.resolve("zoo.cfg")); - return solrHomeDir.getParentFile(); + return solrHomeDir.getParent(); } - protected void copyIfNeeded(File src, File dest) throws IOException { - if (!dest.isFile()) Files.copy(src.toPath(), dest.toPath()); + protected void copyIfNeeded(Path src, Path dest) throws IOException { + if (!Files.isRegularFile(dest)) Files.copy(src, dest); - if (!dest.isFile()) - throw new IllegalStateException("Required file " + dest.getAbsolutePath() + " not found!"); + if (!Files.isRegularFile(dest)) + throw new IllegalStateException("Required file " + dest.toAbsolutePath() + " not found!"); } protected boolean isPortAvailable(int port) { diff --git a/solr/core/src/java/org/apache/solr/cli/SnapshotCreateTool.java b/solr/core/src/java/org/apache/solr/cli/SnapshotCreateTool.java index 236a7416832..49cce02a15c 100644 --- a/solr/core/src/java/org/apache/solr/cli/SnapshotCreateTool.java +++ b/solr/core/src/java/org/apache/solr/cli/SnapshotCreateTool.java @@ -71,7 +71,7 @@ public Options getOptions() { public void runImpl(CommandLine cli) throws Exception { String snapshotName = cli.getOptionValue(SNAPSHOT_NAME_OPTION); String collectionName = cli.getOptionValue(COLLECTION_NAME_OPTION); - try (var solrClient = SolrCLI.getSolrClient(cli)) { + try (var solrClient = CLIUtils.getSolrClient(cli)) { createSnapshot(solrClient, collectionName, snapshotName); } } diff --git a/solr/core/src/java/org/apache/solr/cli/SnapshotDeleteTool.java b/solr/core/src/java/org/apache/solr/cli/SnapshotDeleteTool.java index 53fa8ad002a..c9dc90abab7 100644 --- a/solr/core/src/java/org/apache/solr/cli/SnapshotDeleteTool.java +++ b/solr/core/src/java/org/apache/solr/cli/SnapshotDeleteTool.java @@ -71,7 +71,7 @@ public Options getOptions() { public void runImpl(CommandLine cli) throws Exception { String snapshotName = cli.getOptionValue(SNAPSHOT_NAME_OPTION); String collectionName = cli.getOptionValue(COLLECTION_NAME_OPTION); - try (var solrClient = SolrCLI.getSolrClient(cli)) { + try (var solrClient = CLIUtils.getSolrClient(cli)) { deleteSnapshot(solrClient, collectionName, snapshotName); } } diff --git a/solr/core/src/java/org/apache/solr/cli/SnapshotDescribeTool.java b/solr/core/src/java/org/apache/solr/cli/SnapshotDescribeTool.java index b76f36a7420..dcae8619f3e 100644 --- a/solr/core/src/java/org/apache/solr/cli/SnapshotDescribeTool.java +++ b/solr/core/src/java/org/apache/solr/cli/SnapshotDescribeTool.java @@ -84,7 +84,7 @@ public Options getOptions() { public void runImpl(CommandLine cli) throws Exception { String snapshotName = cli.getOptionValue(SNAPSHOT_NAME_OPTION); String collectionName = cli.getOptionValue(COLLECTION_NAME_OPTION); - try (var solrClient = SolrCLI.getSolrClient(cli)) { + try (var solrClient = CLIUtils.getSolrClient(cli)) { describeSnapshot(solrClient, collectionName, snapshotName); } } diff --git a/solr/core/src/java/org/apache/solr/cli/SnapshotExportTool.java b/solr/core/src/java/org/apache/solr/cli/SnapshotExportTool.java index a51e91badb3..6b8cf3a45d0 100644 --- a/solr/core/src/java/org/apache/solr/cli/SnapshotExportTool.java +++ b/solr/core/src/java/org/apache/solr/cli/SnapshotExportTool.java @@ -106,7 +106,7 @@ public void runImpl(CommandLine cli) throws Exception { Optional backupRepo = Optional.ofNullable(cli.getOptionValue(BACKUP_REPO_NAME_OPTION)); Optional asyncReqId = Optional.ofNullable(cli.getOptionValue(ASYNC_ID_OPTION)); - try (var solrClient = SolrCLI.getSolrClient(cli)) { + try (var solrClient = CLIUtils.getSolrClient(cli)) { exportSnapshot(solrClient, collectionName, snapshotName, destDir, backupRepo, asyncReqId); } } diff --git a/solr/core/src/java/org/apache/solr/cli/SnapshotListTool.java b/solr/core/src/java/org/apache/solr/cli/SnapshotListTool.java index 87219ce6021..4b952199e1b 100644 --- a/solr/core/src/java/org/apache/solr/cli/SnapshotListTool.java +++ b/solr/core/src/java/org/apache/solr/cli/SnapshotListTool.java @@ -62,7 +62,7 @@ public Options getOptions() { @Override public void runImpl(CommandLine cli) throws Exception { String collectionName = cli.getOptionValue(COLLECTION_NAME_OPTION); - try (var solrClient = SolrCLI.getSolrClient(cli)) { + try (var solrClient = CLIUtils.getSolrClient(cli)) { listSnapshots(solrClient, collectionName); } } diff --git a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java index 00a97b4434c..e88b0dada7f 100755 --- a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java +++ b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java @@ -16,32 +16,20 @@ */ package org.apache.solr.cli; -import static org.apache.solr.common.SolrException.ErrorCode.FORBIDDEN; -import static org.apache.solr.common.SolrException.ErrorCode.UNAUTHORIZED; -import static org.apache.solr.common.params.CommonParams.NAME; -import static org.apache.solr.common.params.CommonParams.SYSTEM_INFO_PATH; - import com.google.common.annotations.VisibleForTesting; -import java.io.File; -import java.io.IOException; import java.lang.invoke.MethodHandles; -import java.net.SocketException; import java.net.URI; import java.net.URL; +import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.Enumeration; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.TreeSet; -import java.util.concurrent.TimeUnit; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import org.apache.commons.cli.CommandLine; @@ -51,22 +39,8 @@ import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.impl.Http2SolrClient; -import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest; -import org.apache.solr.client.solrj.request.CoreAdminRequest; -import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.util.ContentStreamBase; -import org.apache.solr.common.util.EnvUtils; import org.apache.solr.common.util.NamedList; import org.apache.solr.util.configuration.SSLConfigurationsFactory; import org.slf4j.Logger; @@ -75,13 +49,6 @@ /** Command-line utility for working with Solr. */ public class SolrCLI implements CLIO { - public static String RED = "\u001B[31m"; - public static String GREEN = "\u001B[32m"; - public static String YELLOW = "\u001B[33m"; - - private static final long MAX_WAIT_FOR_CORE_LOAD_NANOS = - TimeUnit.NANOSECONDS.convert(1, TimeUnit.MINUTES); - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); public static void exit(int exitStatus) { @@ -188,25 +155,17 @@ public static CommandLine parseCmdLine(Tool tool, String[] args) { return cli; } - public static String getDefaultSolrUrl() { - // note that ENV_VAR syntax (and the env vars too) are mapped to env.var sys props - String scheme = EnvUtils.getProperty("solr.url.scheme", "http"); - String host = EnvUtils.getProperty("solr.tool.host", "localhost"); - String port = EnvUtils.getProperty("jetty.port", "8983"); // from SOLR_PORT env - return String.format(Locale.ROOT, "%s://%s:%s", scheme.toLowerCase(Locale.ROOT), host, port); - } - protected static void checkSslStoreSysProp(String solrInstallDir, String key) { String sysProp = "javax.net.ssl." + key; String keyStore = System.getProperty(sysProp); if (keyStore == null) return; - File keyStoreFile = new File(keyStore); - if (keyStoreFile.isFile()) return; // configured setting is OK + Path keyStoreFile = Path.of(keyStore); + if (Files.isRegularFile(keyStoreFile)) return; // configured setting is OK - keyStoreFile = new File(solrInstallDir, "server/" + keyStore); - if (keyStoreFile.isFile()) { - System.setProperty(sysProp, keyStoreFile.getAbsolutePath()); + keyStoreFile = Path.of(solrInstallDir, "server", keyStore); + if (Files.isRegularFile(keyStoreFile)) { + System.setProperty(sysProp, keyStoreFile.toAbsolutePath().toString()); } else { CLIO.err( "WARNING: " @@ -246,6 +205,7 @@ private static Tool newTool(String toolType) throws Exception { else if ("post".equals(toolType)) return new PostTool(); else if ("postlogs".equals(toolType)) return new PostLogsTool(); else if ("version".equals(toolType)) return new VersionTool(); + else if ("stream".equals(toolType)) return new StreamTool(); else if ("snapshot-create".equals(toolType)) return new SnapshotCreateTool(); else if ("snapshot-delete".equals(toolType)) return new SnapshotDeleteTool(); else if ("snapshot-list".equals(toolType)) return new SnapshotListTool(); @@ -272,7 +232,7 @@ private static Tool newTool(String toolType) throws Exception { * CLI option. */ public static String getOptionWithDeprecatedAndDefault( - CommandLine cli, String opt, String deprecated, String def) { + CommandLine cli, Option opt, Option deprecated, String def) { String val = cli.getOptionValue(opt); if (val == null) { val = cli.getOptionValue(deprecated); @@ -407,70 +367,6 @@ private static Set findClasses(String path, String packageName) throws E return classes; } - /** - * Determine if a request to Solr failed due to a communication error, which is generally - * retry-able. - */ - public static boolean checkCommunicationError(Exception exc) { - Throwable rootCause = SolrException.getRootCause(exc); - return (rootCause instanceof SolrServerException || rootCause instanceof SocketException); - } - - public static void checkCodeForAuthError(int code) { - if (code == UNAUTHORIZED.code || code == FORBIDDEN.code) { - throw new SolrException( - SolrException.ErrorCode.getErrorCode(code), - "Solr requires authentication for request. Please supply valid credentials. HTTP code=" - + code); - } - } - - public static boolean exceptionIsAuthRelated(Exception exc) { - return (exc instanceof SolrException - && Arrays.asList(UNAUTHORIZED.code, FORBIDDEN.code).contains(((SolrException) exc).code())); - } - - public static SolrClient getSolrClient(String solrUrl, String credentials, boolean barePath) { - // today we require all urls to end in /solr, however in the future we will need to support the - // /api url end point instead. Eventually we want to have this method always - // return a bare url, and then individual calls decide if they are /solr or /api - // The /solr/ check is because sometimes a full url is passed in, like - // http://localhost:8983/solr/films_shard1_replica_n1/. - if (!barePath && !solrUrl.endsWith("/solr") && !solrUrl.contains("/solr/")) { - solrUrl = solrUrl + "/solr"; - } - Http2SolrClient.Builder builder = - new Http2SolrClient.Builder(solrUrl) - .withMaxConnectionsPerHost(32) - .withKeyStoreReloadInterval(-1, TimeUnit.SECONDS) - .withOptionalBasicAuthCredentials(credentials); - - return builder.build(); - } - - /** - * Helper method for all the places where we assume a /solr on the url. - * - * @param solrUrl The solr url that you want the client for - * @param credentials The username:password for basic auth. - * @return The SolrClient - */ - public static SolrClient getSolrClient(String solrUrl, String credentials) { - return getSolrClient(solrUrl, credentials, false); - } - - public static SolrClient getSolrClient(CommandLine cli, boolean barePath) throws Exception { - String solrUrl = SolrCLI.normalizeSolrUrl(cli); - String credentials = cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION); - return getSolrClient(solrUrl, credentials, barePath); - } - - public static SolrClient getSolrClient(CommandLine cli) throws Exception { - String solrUrl = SolrCLI.normalizeSolrUrl(cli); - String credentials = cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION); - return getSolrClient(solrUrl, credentials, false); - } - private static final String JSON_CONTENT_TYPE = "application/json"; public static NamedList postJsonToSolr( @@ -511,8 +407,7 @@ private static void printHelp() { print("Usage: solr COMMAND OPTIONS"); print(" where COMMAND is one of: start, stop, restart, status, "); print( - " healthcheck, create, delete, auth, assert, config, export, api, package, post, "); - + " healthcheck, create, delete, auth, assert, config, export, api, package, post, stream,"); print( " zk ls, zk cp, zk rm , zk mv, zk mkroot, zk upconfig, zk downconfig,"); print( @@ -540,204 +435,18 @@ private static void printHelp() { print("For more help on how to use Solr, head to https://solr.apache.org/"); } - /** - * Strips off the end of solrUrl any /solr when a legacy solrUrl like http://localhost:8983/solr - * is used, and warns those users. In the future we'll have urls ending with /api as well. - * - * @param solrUrl The user supplied url to Solr. - * @return the solrUrl in the format that Solr expects to see internally. - */ - public static String normalizeSolrUrl(String solrUrl) { - return normalizeSolrUrl(solrUrl, true); - } - - /** - * Strips off the end of solrUrl any /solr when a legacy solrUrl like http://localhost:8983/solr - * is used, and optionally logs a warning. In the future we'll have urls ending with /api as well. - * - * @param solrUrl The user supplied url to Solr. - * @param logUrlFormatWarning If a warning message should be logged about the url format - * @return the solrUrl in the format that Solr expects to see internally. - */ - public static String normalizeSolrUrl(String solrUrl, boolean logUrlFormatWarning) { - if (solrUrl != null) { - URI uri = URI.create(solrUrl); - String urlPath = uri.getRawPath(); - if (urlPath != null && urlPath.contains("/solr")) { - String newSolrUrl = - uri.resolve(urlPath.substring(0, urlPath.lastIndexOf("/solr") + 1)).toString(); - if (logUrlFormatWarning) { - CLIO.err( - "WARNING: URLs provided to this tool needn't include Solr's context-root (e.g. \"/solr\"). Such URLs are deprecated and support for them will be removed in a future release. Correcting from [" - + solrUrl - + "] to [" - + newSolrUrl - + "]."); - } - solrUrl = newSolrUrl; - } - if (solrUrl.endsWith("/")) { - solrUrl = solrUrl.substring(0, solrUrl.length() - 1); - } - } - return solrUrl; - } - - /** - * Get the base URL of a live Solr instance from either the --solr-url command-line option or from - * ZooKeeper. - */ - public static String normalizeSolrUrl(CommandLine cli) throws Exception { - String solrUrl = cli.getOptionValue(CommonCLIOptions.SOLR_URL_OPTION); - if (solrUrl == null) { - String zkHost = cli.getOptionValue(CommonCLIOptions.ZK_HOST_OPTION); - if (zkHost == null) { - solrUrl = SolrCLI.getDefaultSolrUrl(); - CLIO.err( - "Neither --zk-host or --solr-url parameters provided so assuming solr url is " - + solrUrl - + "."); - } else { - try (CloudSolrClient cloudSolrClient = getCloudHttp2SolrClient(zkHost)) { - cloudSolrClient.connect(); - Set liveNodes = cloudSolrClient.getClusterState().getLiveNodes(); - if (liveNodes.isEmpty()) - throw new IllegalStateException( - "No live nodes found! Cannot determine 'solrUrl' from ZooKeeper: " + zkHost); - - String firstLiveNode = liveNodes.iterator().next(); - solrUrl = ZkStateReader.from(cloudSolrClient).getBaseUrlForNodeName(firstLiveNode); - solrUrl = normalizeSolrUrl(solrUrl, false); - } - } - } - solrUrl = normalizeSolrUrl(solrUrl); - return solrUrl; - } - - /** - * Get the ZooKeeper connection string from either the zk-host command-line option or by looking - * it up from a running Solr instance based on the solr-url option. - */ - public static String getZkHost(CommandLine cli) throws Exception { - - String zkHost = cli.getOptionValue(CommonCLIOptions.ZK_HOST_OPTION); - if (zkHost != null && !zkHost.isBlank()) { - return zkHost; - } - - try (SolrClient solrClient = getSolrClient(cli)) { - // hit Solr to get system info - NamedList systemInfo = - solrClient.request( - new GenericSolrRequest(SolrRequest.METHOD.GET, CommonParams.SYSTEM_INFO_PATH)); - - // convert raw JSON into user-friendly output - StatusTool statusTool = new StatusTool(); - Map status = statusTool.reportStatus(systemInfo, solrClient); - @SuppressWarnings("unchecked") - Map cloud = (Map) status.get("cloud"); - if (cloud != null) { - String zookeeper = (String) cloud.get("ZooKeeper"); - if (zookeeper.endsWith("(embedded)")) { - zookeeper = zookeeper.substring(0, zookeeper.length() - "(embedded)".length()); - } - zkHost = zookeeper; - } - } - - return zkHost; - } - - public static SolrZkClient getSolrZkClient(CommandLine cli, String zkHost) throws Exception { - if (zkHost == null) { - throw new IllegalStateException( - "Solr at " - + cli.getOptionValue(CommonCLIOptions.SOLR_URL_OPTION) - + " is running in standalone server mode, this command can only be used when running in SolrCloud mode.\n"); - } - return new SolrZkClient.Builder() - .withUrl(zkHost) - .withTimeout(SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT, TimeUnit.MILLISECONDS) - .build(); - } - - public static CloudHttp2SolrClient getCloudHttp2SolrClient(String zkHost) { - return getCloudHttp2SolrClient(zkHost, null); - } - - public static CloudHttp2SolrClient getCloudHttp2SolrClient( - String zkHost, Http2SolrClient.Builder builder) { - return new CloudHttp2SolrClient.Builder(Collections.singletonList(zkHost), Optional.empty()) - .withInternalClientBuilder(builder) - .build(); - } - - public static boolean safeCheckCollectionExists( - String solrUrl, String collection, String credentials) { - boolean exists = false; - try (var solrClient = getSolrClient(solrUrl, credentials)) { - NamedList existsCheckResult = solrClient.request(new CollectionAdminRequest.List()); - @SuppressWarnings("unchecked") - List collections = (List) existsCheckResult.get("collections"); - exists = collections != null && collections.contains(collection); - } catch (Exception exc) { - // just ignore it since we're only interested in a positive result here - } - return exists; - } - - @SuppressWarnings("unchecked") - public static boolean safeCheckCoreExists(String solrUrl, String coreName, String credentials) { - boolean exists = false; - try (var solrClient = getSolrClient(solrUrl, credentials)) { - boolean wait = false; - final long startWaitAt = System.nanoTime(); - do { - if (wait) { - final int clamPeriodForStatusPollMs = 1000; - Thread.sleep(clamPeriodForStatusPollMs); - } - NamedList existsCheckResult = - CoreAdminRequest.getStatus(coreName, solrClient).getResponse(); - NamedList status = (NamedList) existsCheckResult.get("status"); - NamedList coreStatus = (NamedList) status.get(coreName); - Map failureStatus = - (Map) existsCheckResult.get("initFailures"); - String errorMsg = (String) failureStatus.get(coreName); - final boolean hasName = coreStatus != null && coreStatus.asMap().containsKey(NAME); - exists = hasName || errorMsg != null; - wait = hasName && errorMsg == null && "true".equals(coreStatus.get("isLoading")); - } while (wait && System.nanoTime() - startWaitAt < MAX_WAIT_FOR_CORE_LOAD_NANOS); - } catch (Exception exc) { - // just ignore it since we're only interested in a positive result here - } - return exists; - } - - public static boolean isCloudMode(SolrClient solrClient) throws SolrServerException, IOException { - NamedList systemInfo = - solrClient.request(new GenericSolrRequest(SolrRequest.METHOD.GET, SYSTEM_INFO_PATH)); - return "solrcloud".equals(systemInfo.get("mode")); - } - - public static Path getConfigSetsDir(Path solrInstallDir) { - Path configSetsPath = Paths.get("server/solr/configsets/"); - return solrInstallDir.resolve(configSetsPath); - } - public static void print(Object message) { print(null, message); } /** Console print using green color */ public static void printGreen(Object message) { - print(GREEN, message); + print(CLIUtils.GREEN, message); } /** Console print using red color */ public static void printRed(Object message) { - print(RED, message); + print(CLIUtils.RED, message); } public static void print(String color, Object message) { diff --git a/solr/core/src/java/org/apache/solr/cli/StatusTool.java b/solr/core/src/java/org/apache/solr/cli/StatusTool.java index dcf0200e511..fa6c2cbf2bd 100644 --- a/solr/core/src/java/org/apache/solr/cli/StatusTool.java +++ b/solr/core/src/java/org/apache/solr/cli/StatusTool.java @@ -18,8 +18,6 @@ package org.apache.solr.cli; import java.io.PrintStream; -import java.net.URI; -import java.net.URISyntaxException; import java.util.Collection; import java.util.LinkedHashMap; import java.util.List; @@ -188,25 +186,15 @@ private void printProcessStatus(SolrProcess process, CommandLine cli) throws Exc CLIO.out(""); } - private Integer portFromUrl(String solrUrl) { + public void waitForSolrUpAndPrintStatus(String solrUrl, CommandLine cli, int maxWaitSecs) + throws Exception { + int solrPort = -1; try { - URI uri = new URI(solrUrl); - int port = uri.getPort(); - if (port == -1) { - return uri.getScheme().equals("https") ? 443 : 80; - } else { - return port; - } - } catch (URISyntaxException e) { + solrPort = CLIUtils.portFromUrl(solrUrl); + } catch (Exception e) { CLIO.err("Invalid URL provided, does not contain port"); - System.exit(1); - return null; + SolrCLI.exit(1); } - } - - public void waitForSolrUpAndPrintStatus(String solrUrl, CommandLine cli, int maxWaitSecs) - throws Exception { - int solrPort = portFromUrl(solrUrl); echo("Waiting up to " + maxWaitSecs + " seconds to see Solr running on port " + solrPort); boolean solrUp = waitForSolrUp(solrUrl, cli, maxWaitSecs); if (solrUp) { @@ -268,10 +256,10 @@ public String statusFromRunningSolr(String solrUrl, CommandLine cli) throws Exce .write(getStatus(solrUrl, cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION))); return arr.toString(); } catch (Exception exc) { - if (SolrCLI.exceptionIsAuthRelated(exc)) { + if (CLIUtils.exceptionIsAuthRelated(exc)) { throw exc; } - if (SolrCLI.checkCommunicationError(exc)) { + if (CLIUtils.checkCommunicationError(exc)) { // this is not actually an error from the tool as it's ok if Solr is not online. return null; } else { @@ -289,7 +277,7 @@ public Map waitToSeeSolrUp( try { return getStatus(solrUrl, credentials); } catch (Exception exc) { - if (SolrCLI.exceptionIsAuthRelated(exc)) { + if (CLIUtils.exceptionIsAuthRelated(exc)) { throw exc; } try { @@ -308,7 +296,7 @@ public Map waitToSeeSolrUp( } public Map getStatus(String solrUrl, String credentials) throws Exception { - try (var solrClient = SolrCLI.getSolrClient(solrUrl, credentials)) { + try (var solrClient = CLIUtils.getSolrClient(solrUrl, credentials)) { return getStatus(solrClient); } } @@ -358,13 +346,14 @@ protected Map getCloudStatus(SolrClient solrClient, String zkHos Map cloudStatus = new LinkedHashMap<>(); cloudStatus.put("ZooKeeper", (zkHost != null) ? zkHost : "?"); + // TODO add booleans to request just what we want; not everything NamedList json = solrClient.request(new CollectionAdminRequest.ClusterStatus()); List liveNodes = (List) json.findRecursive("cluster", "live_nodes"); cloudStatus.put("liveNodes", String.valueOf(liveNodes.size())); - Map collections = - ((NamedList) json.findRecursive("cluster", "collections")).asMap(); + // TODO get this as a metric from the metrics API instead, or something else. + var collections = (NamedList) json.findRecursive("cluster", "collections"); cloudStatus.put("collections", String.valueOf(collections.size())); return cloudStatus; diff --git a/solr/core/src/java/org/apache/solr/cli/StreamTool.java b/solr/core/src/java/org/apache/solr/cli/StreamTool.java new file mode 100644 index 00000000000..39684310f0e --- /dev/null +++ b/solr/core/src/java/org/apache/solr/cli/StreamTool.java @@ -0,0 +1,525 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cli; + +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.LineNumberReader; +import java.io.PrintStream; +import java.io.Reader; +import java.io.StringReader; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.solr.client.solrj.io.Lang; +import org.apache.solr.client.solrj.io.SolrClientCache; +import org.apache.solr.client.solrj.io.Tuple; +import org.apache.solr.client.solrj.io.comp.StreamComparator; +import org.apache.solr.client.solrj.io.stream.PushBackStream; +import org.apache.solr.client.solrj.io.stream.SolrStream; +import org.apache.solr.client.solrj.io.stream.StreamContext; +import org.apache.solr.client.solrj.io.stream.TupleStream; +import org.apache.solr.client.solrj.io.stream.expr.Explanation; +import org.apache.solr.client.solrj.io.stream.expr.Expressible; +import org.apache.solr.client.solrj.io.stream.expr.StreamExpression; +import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParser; +import org.apache.solr.client.solrj.io.stream.expr.StreamFactory; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.handler.CatStream; + +/** Supports stream command in the bin/solr script. */ +public class StreamTool extends ToolBase { + + public StreamTool() { + this(CLIO.getOutStream()); + } + + public StreamTool(PrintStream stdout) { + super(stdout); + } + + private final SolrClientCache solrClientCache = new SolrClientCache(); + + @Override + public String getName() { + return "stream"; + } + + @Override + public String getUsage() { + // Specify that the last argument is the streaming expression + return "bin/solr stream [--array-delimiter ] [-c ] [--delimiter ] [--execution ] [--fields\n" + + " ] [-h] [--header] [-s ] [-u ] [-v] [-z ] \n"; + } + + private static final Option EXECUTION_OPTION = + Option.builder() + .longOpt("execution") + .hasArg() + .argName("ENVIRONMENT") + .desc( + "Execution environment is either 'local' (i.e CLI process) or via a 'remote' Solr server. Default environment is 'remote'.") + .build(); + + private static final Option COLLECTION_OPTION = + Option.builder("c") + .longOpt("name") + .argName("NAME") + .hasArg() + .desc( + "Name of the specific collection to execute expression on if the execution is set to 'remote'. Required for 'remote' execution environment.") + .build(); + + private static final Option FIELDS_OPTION = + Option.builder() + .longOpt("fields") + .argName("FIELDS") + .hasArg() + .desc( + "The fields in the tuples to output. Defaults to fields in the first tuple of result set.") + .build(); + + private static final Option HEADER_OPTION = + Option.builder().longOpt("header").desc("Specify to include a header line.").build(); + + private static final Option DELIMITER_OPTION = + Option.builder() + .longOpt("delimiter") + .argName("CHARACTER") + .hasArg() + .desc("The output delimiter. Default to using three spaces.") + .build(); + private static final Option ARRAY_DELIMITER_OPTION = + Option.builder() + .longOpt("array-delimiter") + .argName("CHARACTER") + .hasArg() + .desc("The delimiter multi-valued fields. Default to using a pipe (|) delimiter.") + .build(); + + @Override + public Options getOptions() { + + return super.getOptions() + .addOption(EXECUTION_OPTION) + .addOption(COLLECTION_OPTION) + .addOption(FIELDS_OPTION) + .addOption(HEADER_OPTION) + .addOption(DELIMITER_OPTION) + .addOption(ARRAY_DELIMITER_OPTION) + .addOption(CommonCLIOptions.CREDENTIALS_OPTION) + .addOptionGroup(getConnectionOptions()); + } + + @Override + @SuppressWarnings({"rawtypes"}) + public void runImpl(CommandLine cli) throws Exception { + + String expressionArgument = cli.getArgs()[0]; + String execution = cli.getOptionValue(EXECUTION_OPTION, "remote"); + String arrayDelimiter = cli.getOptionValue(ARRAY_DELIMITER_OPTION, "|"); + String delimiter = cli.getOptionValue(DELIMITER_OPTION, " "); + boolean includeHeaders = cli.hasOption(HEADER_OPTION); + String[] outputHeaders = getOutputFields(cli); + + LineNumberReader bufferedReader = null; + String expr; + try { + Reader inputStream = + expressionArgument.toLowerCase(Locale.ROOT).endsWith(".expr") + ? new InputStreamReader( + new FileInputStream(expressionArgument), Charset.defaultCharset()) + : new StringReader(expressionArgument); + + bufferedReader = new LineNumberReader(inputStream); + expr = StreamTool.readExpression(bufferedReader, cli.getArgs()); + echoIfVerbose("Running Expression: " + expr); + } finally { + if (bufferedReader != null) { + bufferedReader.close(); + } + } + + PushBackStream pushBackStream; + if (execution.equalsIgnoreCase("local")) { + pushBackStream = doLocalMode(cli, expr); + } else { + pushBackStream = doRemoteMode(cli, expr); + } + + try { + pushBackStream.open(); + + if (outputHeaders == null) { + + Tuple tuple = pushBackStream.read(); + + if (!tuple.EOF) { + outputHeaders = getHeadersFromFirstTuple(tuple); + } + + pushBackStream.pushBack(tuple); + } + + if (includeHeaders) { + StringBuilder headersOut = new StringBuilder(); + if (outputHeaders != null) { + for (int i = 0; i < outputHeaders.length; i++) { + if (i > 0) { + headersOut.append(delimiter); + } + headersOut.append(outputHeaders[i]); + } + } + CLIO.out(headersOut.toString()); + } + + while (true) { + Tuple tuple = pushBackStream.read(); + if (tuple.EOF) { + break; + } else { + StringBuilder outLine = new StringBuilder(); + if (outputHeaders != null) { + for (int i = 0; i < outputHeaders.length; i++) { + if (i > 0) { + outLine.append(delimiter); + } + + Object o = tuple.get(outputHeaders[i]); + if (o != null) { + if (o instanceof List outfields) { + outLine.append(listToString(outfields, arrayDelimiter)); + } else { + outLine.append(o); + } + } + } + } + CLIO.out(outLine.toString()); + } + } + } finally { + pushBackStream.close(); + solrClientCache.close(); + } + + echoIfVerbose("StreamTool -- Done."); + } + + /** + * Runs a streaming expression in the local process of the CLI. + * + *

Running locally means that parallelization support or those expressions requiring access to + * internal Solr capabilities will not function. + * + * @param cli The CLI invoking the call + * @param expr The streaming expression to be parsed and in the context of the CLI process + * @return A connection to the streaming expression that receives Tuples as they are emitted + * locally. + */ + private PushBackStream doLocalMode(CommandLine cli, String expr) throws Exception { + String zkHost = CLIUtils.getZkHost(cli); + + echoIfVerbose("Connecting to ZooKeeper at " + zkHost); + solrClientCache.setBasicAuthCredentials( + cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION)); + solrClientCache.getCloudSolrClient(zkHost); + + TupleStream stream; + PushBackStream pushBackStream; + + StreamExpression streamExpression = StreamExpressionParser.parse(expr); + StreamFactory streamFactory = new StreamFactory(); + + // stdin is ONLY available in the local mode, not in the remote mode as it + // requires access to System.in + streamFactory.withFunctionName("stdin", StandardInStream.class); + + // LocalCatStream extends CatStream and disables the Solr cluster specific + // logic about where to read data from. + streamFactory.withFunctionName("cat", LocalCatStream.class); + + streamFactory.withDefaultZkHost(zkHost); + + Lang.register(streamFactory); + + stream = streamFactory.constructStream(streamExpression); + + pushBackStream = new PushBackStream(stream); + + // Now we can run the stream and return the results. + StreamContext streamContext = new StreamContext(); + streamContext.setSolrClientCache(solrClientCache); + + // Output the headers + pushBackStream.setStreamContext(streamContext); + + return pushBackStream; + } + + /** + * Runs a streaming expression on a Solr collection via the /stream end point and returns the + * results to the CLI. Requires a collection to be specified to send the expression to. + * + *

Running remotely allows you to use all the standard Streaming Expression capabilities as the + * expression is running in a Solr environment. + * + * @param cli The CLI invoking the call + * @param expr The streaming expression to be parsed and run remotely + * @return A connection to the streaming expression that receives Tuples as they are emitted from + * Solr /stream. + */ + private PushBackStream doRemoteMode(CommandLine cli, String expr) throws Exception { + + String solrUrl = CLIUtils.normalizeSolrUrl(cli); + if (!cli.hasOption(COLLECTION_OPTION)) { + throw new IllegalStateException( + "You must provide --name COLLECTION with --execution remote parameter."); + } + String collection = cli.getOptionValue(COLLECTION_OPTION); + + if (expr.toLowerCase(Locale.ROOT).contains("stdin(")) { + throw new IllegalStateException( + "The stdin() expression is only usable with --worker local set up."); + } + + final SolrStream solrStream = + new SolrStream(solrUrl + "/solr/" + collection, params("qt", "/stream", "expr", expr)); + + String credentials = cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION); + if (credentials != null) { + String username = credentials.split(":")[0]; + String password = credentials.split(":")[1]; + solrStream.setCredentials(username, password); + } + return new PushBackStream(solrStream); + } + + private static ModifiableSolrParams params(String... params) { + if (params.length % 2 != 0) throw new RuntimeException("Params length should be even"); + ModifiableSolrParams msp = new ModifiableSolrParams(); + for (int i = 0; i < params.length; i += 2) { + msp.add(params[i], params[i + 1]); + } + return msp; + } + + public static class StandardInStream extends TupleStream implements Expressible { + + private BufferedReader reader; + private InputStream inputStream = System.in; + private boolean doClose = false; + + public StandardInStream() {} + + public StandardInStream(StreamExpression expression, StreamFactory factory) + throws IOException {} + + @Override + public List children() { + return null; + } + + public void setInputStream(InputStream inputStream) { + this.inputStream = inputStream; + this.doClose = true; + } + + @Override + public void open() { + reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + } + + @Override + public void close() throws IOException { + if (doClose) { + inputStream.close(); + } + } + + @Override + public Tuple read() throws IOException { + String line = reader.readLine(); + Map map = new HashMap<>(); + Tuple tuple = new Tuple(map); + if (line != null) { + tuple.put("line", line); + tuple.put("file", "cat"); + } else { + tuple.put("EOF", "true"); + } + return tuple; + } + + @Override + public void setStreamContext(StreamContext context) {} + + @Override + public StreamExpression toExpression(StreamFactory factory) { + return null; + } + + @Override + public Explanation toExplanation(StreamFactory factory) { + return null; + } + + @Override + public StreamComparator getStreamSort() { + return null; + } + } + + static String[] getOutputFields(CommandLine cli) { + if (cli.hasOption(FIELDS_OPTION)) { + + String fl = cli.getOptionValue(FIELDS_OPTION); + String[] flArray = fl.split(","); + String[] outputHeaders = new String[flArray.length]; + + for (int i = 0; i < outputHeaders.length; i++) { + outputHeaders[i] = flArray[i].trim(); + } + + return outputHeaders; + + } else { + return null; + } + } + + public static class LocalCatStream extends CatStream { + + public LocalCatStream(StreamExpression expression, StreamFactory factory) throws IOException { + super(expression, factory); + } + + public LocalCatStream(String commaDelimitedFilepaths, int maxLines) { + super(commaDelimitedFilepaths, maxLines); + } + + @Override + public void setStreamContext(StreamContext context) { + // LocalCatStream inherently has no Solr core to pull from the context + } + + @Override + protected List validateAndSetFilepathsInSandbox() { + // The nature of LocalCatStream is that we are not limited to the sandboxed "userfiles" + // directory + // the way the CatStream does. + + final List crawlSeeds = new ArrayList<>(); + for (String crawlRootStr : commaDelimitedFilepaths.split(",")) { + Path crawlRootPath = Paths.get(crawlRootStr).normalize(); + + if (!Files.exists(crawlRootPath)) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "file/directory to stream doesn't exist: " + crawlRootStr); + } + + crawlSeeds.add(new CrawlFile(crawlRootStr, crawlRootPath)); + } + + return crawlSeeds; + } + } + + @SuppressWarnings({"rawtypes"}) + static String[] getHeadersFromFirstTuple(Tuple tuple) { + Set fields = tuple.getFields().keySet(); + String[] outputHeaders = new String[fields.size()]; + int i = -1; + for (Object o : fields) { + outputHeaders[++i] = o.toString(); + } + Arrays.sort(outputHeaders); + return outputHeaders; + } + + @SuppressWarnings({"rawtypes"}) + static String listToString(List values, String internalDelim) { + StringBuilder buf = new StringBuilder(); + for (Object value : values) { + if (buf.length() > 0) { + buf.append(internalDelim); + } + + buf.append(value.toString()); + } + + return buf.toString(); + } + + static String readExpression(LineNumberReader bufferedReader, String[] args) throws IOException { + + StringBuilder exprBuff = new StringBuilder(); + + boolean comment = false; + while (true) { + String line = bufferedReader.readLine(); + if (line == null) { + break; + } + + if (line.trim().indexOf("/*") == 0) { + comment = true; + continue; + } + + if (line.trim().contains("*/")) { + comment = false; + continue; + } + + if (comment || line.trim().startsWith("#") || line.trim().startsWith("//")) { + continue; + } + + // Substitute parameters + + if (line.length() > 0) { + for (int i = 1; i < args.length; i++) { + String arg = args[i]; + line = line.replace("$" + i, arg); + } + } + + exprBuff.append(line); + } + + return exprBuff.toString(); + } +} diff --git a/solr/core/src/java/org/apache/solr/cli/UpdateACLTool.java b/solr/core/src/java/org/apache/solr/cli/UpdateACLTool.java index 996ada0a213..bf54152d2ac 100644 --- a/solr/core/src/java/org/apache/solr/cli/UpdateACLTool.java +++ b/solr/core/src/java/org/apache/solr/cli/UpdateACLTool.java @@ -59,7 +59,7 @@ public Options getOptions() { @Override public void runImpl(CommandLine cli) throws Exception { - String zkHost = SolrCLI.getZkHost(cli); + String zkHost = CLIUtils.getZkHost(cli); String path = cli.getArgs()[0]; if (!ZkController.checkChrootPath(zkHost, true)) { diff --git a/solr/core/src/java/org/apache/solr/cli/ZkCpTool.java b/solr/core/src/java/org/apache/solr/cli/ZkCpTool.java index b2045ef3541..7f901e67cf2 100644 --- a/solr/core/src/java/org/apache/solr/cli/ZkCpTool.java +++ b/solr/core/src/java/org/apache/solr/cli/ZkCpTool.java @@ -124,7 +124,7 @@ public String getHeader() { @Override public void runImpl(CommandLine cli) throws Exception { - String zkHost = SolrCLI.getZkHost(cli); + String zkHost = CLIUtils.getZkHost(cli); echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ..."); String src = cli.getArgs()[0]; diff --git a/solr/core/src/java/org/apache/solr/cli/ZkLsTool.java b/solr/core/src/java/org/apache/solr/cli/ZkLsTool.java index c8f1f55b266..57bb45d4e40 100644 --- a/solr/core/src/java/org/apache/solr/cli/ZkLsTool.java +++ b/solr/core/src/java/org/apache/solr/cli/ZkLsTool.java @@ -57,10 +57,10 @@ public String getUsage() { @Override public void runImpl(CommandLine cli) throws Exception { - String zkHost = SolrCLI.getZkHost(cli); + String zkHost = CLIUtils.getZkHost(cli); String znode = cli.getArgs()[0]; - try (SolrZkClient zkClient = SolrCLI.getSolrZkClient(cli, zkHost)) { + try (SolrZkClient zkClient = CLIUtils.getSolrZkClient(cli, zkHost)) { echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ..."); boolean recursive = cli.hasOption(CommonCLIOptions.RECURSIVE_OPTION); diff --git a/solr/core/src/java/org/apache/solr/cli/ZkMkrootTool.java b/solr/core/src/java/org/apache/solr/cli/ZkMkrootTool.java index f2dc3703835..ace32ad5ce7 100644 --- a/solr/core/src/java/org/apache/solr/cli/ZkMkrootTool.java +++ b/solr/core/src/java/org/apache/solr/cli/ZkMkrootTool.java @@ -76,11 +76,11 @@ public String getHeader() { @Override public void runImpl(CommandLine cli) throws Exception { - String zkHost = SolrCLI.getZkHost(cli); + String zkHost = CLIUtils.getZkHost(cli); String znode = cli.getArgs()[0]; boolean failOnExists = cli.hasOption(FAIL_ON_EXISTS_OPTION); - try (SolrZkClient zkClient = SolrCLI.getSolrZkClient(cli, zkHost)) { + try (SolrZkClient zkClient = CLIUtils.getSolrZkClient(cli, zkHost)) { echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ..."); echo("Creating ZooKeeper path " + znode + " on ZooKeeper at " + zkHost); diff --git a/solr/core/src/java/org/apache/solr/cli/ZkMvTool.java b/solr/core/src/java/org/apache/solr/cli/ZkMvTool.java index 41ca7d78822..dba47d5b824 100644 --- a/solr/core/src/java/org/apache/solr/cli/ZkMvTool.java +++ b/solr/core/src/java/org/apache/solr/cli/ZkMvTool.java @@ -74,9 +74,9 @@ public String getHeader() { @Override public void runImpl(CommandLine cli) throws Exception { - String zkHost = SolrCLI.getZkHost(cli); + String zkHost = CLIUtils.getZkHost(cli); - try (SolrZkClient zkClient = SolrCLI.getSolrZkClient(cli, zkHost)) { + try (SolrZkClient zkClient = CLIUtils.getSolrZkClient(cli, zkHost)) { echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ..."); String src = cli.getArgs()[0]; String dst = cli.getArgs()[1]; diff --git a/solr/core/src/java/org/apache/solr/cli/ZkRmTool.java b/solr/core/src/java/org/apache/solr/cli/ZkRmTool.java index cdb2e600334..777ad6035a5 100644 --- a/solr/core/src/java/org/apache/solr/cli/ZkRmTool.java +++ b/solr/core/src/java/org/apache/solr/cli/ZkRmTool.java @@ -58,7 +58,7 @@ public String getUsage() { @Override public void runImpl(CommandLine cli) throws Exception { - String zkHost = SolrCLI.getZkHost(cli); + String zkHost = CLIUtils.getZkHost(cli); String target = cli.getArgs()[0]; boolean recursive = cli.hasOption(CommonCLIOptions.RECURSIVE_OPTION); @@ -71,7 +71,7 @@ public void runImpl(CommandLine cli) throws Exception { throw new SolrServerException("You may not remove the root ZK node ('/')!"); } echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ..."); - try (SolrZkClient zkClient = SolrCLI.getSolrZkClient(cli, zkHost)) { + try (SolrZkClient zkClient = CLIUtils.getSolrZkClient(cli, zkHost)) { if (!recursive && zkClient.getChildren(znode, null, true).size() != 0) { throw new SolrServerException( "ZooKeeper node " + znode + " has children and recursive has NOT been specified."); diff --git a/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java b/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java index 1da77d8db82..a15195f6228 100644 --- a/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java +++ b/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java @@ -321,8 +321,7 @@ public void writeResults(ResultContext ctx, JavaBinCodec codec) throws IOExcepti /** A list of streams, non-null. */ private List getContentStreams(SolrRequest request) throws IOException { if (request.getMethod() == SolrRequest.METHOD.GET) return List.of(); - if (request instanceof ContentStreamUpdateRequest) { - final ContentStreamUpdateRequest csur = (ContentStreamUpdateRequest) request; + if (request instanceof ContentStreamUpdateRequest csur) { final Collection cs = csur.getContentStreams(); if (cs != null) return new ArrayList<>(cs); } diff --git a/solr/core/src/java/org/apache/solr/cloud/ConfigSetCmds.java b/solr/core/src/java/org/apache/solr/cloud/ConfigSetCmds.java index ee1c01f99d4..52219bbc2ea 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ConfigSetCmds.java +++ b/solr/core/src/java/org/apache/solr/cloud/ConfigSetCmds.java @@ -31,7 +31,6 @@ import java.util.HashMap; import java.util.Map; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.ConfigSetParams; @@ -184,16 +183,18 @@ private static void deleteConfigSet( String configSetName, boolean force, CoreContainer coreContainer) throws IOException { ZkStateReader zkStateReader = coreContainer.getZkController().getZkStateReader(); - for (Map.Entry entry : - zkStateReader.getClusterState().getCollectionsMap().entrySet()) { - String configName = entry.getValue().getConfigName(); - if (configSetName.equals(configName)) - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - "Can not delete ConfigSet as it is currently being used by collection [" - + entry.getKey() - + "]"); - } + zkStateReader + .getClusterState() + .forEachCollection( + state -> { + String configName = state.getConfigName(); + if (configSetName.equals(configName)) + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Can not delete ConfigSet as it is currently being used by collection [" + + state.getName() + + "]"); + }); String propertyPath = ConfigSetProperties.DEFAULT_FILENAME; NamedList properties = diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedClusterStateUpdater.java b/solr/core/src/java/org/apache/solr/cloud/DistributedClusterStateUpdater.java index 7d47dd26bcb..3c7552790c9 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DistributedClusterStateUpdater.java +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedClusterStateUpdater.java @@ -975,7 +975,7 @@ public void computeUpdates(ClusterState clusterState, SolrZkClient client) { final DocCollection docCollection = clusterState.getCollectionOrNull(collectionName); Optional result = docCollection != null - ? NodeMutator.computeCollectionUpdate(nodeName, collectionName, docCollection, client) + ? NodeMutator.computeCollectionUpdate(nodeName, docCollection, client) : Optional.empty(); if (docCollection == null) { diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java index 378b2ff7043..8546bb7adad 100644 --- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java +++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java @@ -423,8 +423,7 @@ public void run() { // Return true whenever the exception thrown by ZkStateWriter is correspond // to a invalid state or 'bad' message (in this case, we should remove that message from queue) private boolean isBadMessage(Exception e) { - if (e instanceof KeeperException) { - KeeperException ke = (KeeperException) e; + if (e instanceof KeeperException ke) { return ke.code() == KeeperException.Code.NONODE || ke.code() == KeeperException.Code.NODEEXISTS; } diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskQueue.java b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskQueue.java index 3092dce185e..c79b1d40ff9 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskQueue.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskQueue.java @@ -318,8 +318,7 @@ public int hashCode() { @Override public boolean equals(Object obj) { if (this == obj) return true; - if (!(obj instanceof QueueEvent)) return false; - QueueEvent other = (QueueEvent) obj; + if (!(obj instanceof QueueEvent other)) return false; return Objects.equals(id, other.id); } diff --git a/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java b/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java index 1821600fc83..6bba8e99319 100644 --- a/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java +++ b/solr/core/src/java/org/apache/solr/cloud/RecoveringCoreTermWatcher.java @@ -76,9 +76,8 @@ public boolean onTermChanged(ShardTerms terms) { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof RecoveringCoreTermWatcher)) return false; + if (!(o instanceof RecoveringCoreTermWatcher that)) return false; - RecoveringCoreTermWatcher that = (RecoveringCoreTermWatcher) o; return coreDescriptor.getName().equals(that.coreDescriptor.getName()); } diff --git a/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java b/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java index 43390e63e9b..06bbbefedbe 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java +++ b/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java @@ -27,6 +27,7 @@ import org.apache.solr.core.SolrCore; import org.apache.solr.handler.IndexFetcher; import org.apache.solr.handler.ReplicationHandler; +import org.apache.solr.handler.admin.api.ReplicationAPIBase; import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.update.CommitUpdateCommand; @@ -89,7 +90,7 @@ public void startReplication(boolean switchTransactionLog) { NamedList followerConfig = new NamedList<>(); followerConfig.add(ReplicationHandler.FETCH_FROM_LEADER, Boolean.TRUE); followerConfig.add(ReplicationHandler.SKIP_COMMIT_ON_LEADER_VERSION_ZERO, Boolean.TRUE); - followerConfig.add(ReplicationHandler.POLL_INTERVAL, pollIntervalStr); + followerConfig.add(ReplicationAPIBase.POLL_INTERVAL, pollIntervalStr); NamedList replicationConfig = new NamedList<>(); replicationConfig.add("follower", followerConfig); diff --git a/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java b/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java index ee7a14733ed..b2f44c9a188 100644 --- a/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java +++ b/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java @@ -16,7 +16,6 @@ */ package org.apache.solr.cloud; -import java.io.File; import java.io.IOException; import java.io.Reader; import java.lang.invoke.MethodHandles; @@ -51,10 +50,10 @@ public class SolrZkServer { private Thread zkThread; // the thread running a zookeeper server, only if zkRun is set - private File dataHome; // o.a.zookeeper.**.QuorumPeerConfig needs a File not a Path + private Path dataHome; // o.a.zookeeper.**.QuorumPeerConfig needs a File not a Path private String confHome; - public SolrZkServer(String zkRun, String zkHost, File dataHome, String confHome, int solrPort) { + public SolrZkServer(String zkRun, String zkHost, Path dataHome, String confHome, int solrPort) { this.zkRun = zkRun; this.zkHost = zkHost; this.dataHome = dataHome; @@ -277,8 +276,8 @@ public static boolean hasServers(Properties props) { return false; } - public void setDataDir(File dataDir) { - this.dataDir = dataDir; + public void setDataDir(Path dataDir) { + this.dataDir = dataDir.toFile(); } /** diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java index 59480caa8e9..5a890121a43 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java @@ -50,7 +50,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Supplier; import java.util.stream.Collectors; import org.apache.curator.framework.api.ACLProvider; import org.apache.solr.client.solrj.SolrClient; @@ -180,8 +179,7 @@ public int hashCode() { @Override public boolean equals(Object obj) { if (this == obj) return true; - if (!(obj instanceof ContextKey)) return false; - ContextKey other = (ContextKey) obj; + if (!(obj instanceof ContextKey other)) return false; return Objects.equals(collection, other.collection) && Objects.equals(coreNodeName, other.coreNodeName); } @@ -199,9 +197,6 @@ public String toString() { public final ZkStateReader zkStateReader; private SolrCloudManager cloudManager; - // only for internal usage - private Http2SolrClient http2SolrClient; - private CloudHttp2SolrClient cloudSolrClient; private final String zkServerAddress; // example: 127.0.0.1:54062/solr @@ -299,16 +294,12 @@ public Object call() throws Exception { * @param zkClientConnectTimeout timeout in ms * @param cloudConfig configuration for this controller. TODO: possibly redundant with * CoreContainer - * @param descriptorsSupplier a supplier of the current core descriptors. used to know which cores - * to re-register on reconnect */ - @SuppressWarnings({"unchecked"}) public ZkController( final CoreContainer cc, String zkServerAddress, int zkClientConnectTimeout, - CloudConfig cloudConfig, - final Supplier> descriptorsSupplier) + CloudConfig cloudConfig) throws InterruptedException, TimeoutException, IOException { if (cc == null) throw new IllegalArgumentException("CoreContainer cannot be null."); @@ -367,8 +358,8 @@ public ZkController( .withUrl(zkServerAddress) .withTimeout(clientTimeout, TimeUnit.MILLISECONDS) .withConnTimeOut(zkClientConnectTimeout, TimeUnit.MILLISECONDS) - .withReconnectListener(() -> onReconnect(descriptorsSupplier)) - .withBeforeConnect(() -> beforeReconnect(descriptorsSupplier)) + .withReconnectListener(this::onReconnect) + .withDisconnectListener((sessionExpired) -> onDisconnect(sessionExpired)) .withAclProvider(zkACLProvider) .withClosedCheck(cc::isShutDown) .withCompressor(compressor) @@ -404,21 +395,33 @@ public ZkController( assert ObjectReleaseTracker.track(this); } - private void beforeReconnect(Supplier> descriptorsSupplier) { + private void onDisconnect(boolean sessionExpired) { try { overseer.close(); } catch (Exception e) { log.error("Error trying to stop any Overseer threads", e); } - closeOutstandingElections(descriptorsSupplier); - markAllAsNotLeader(descriptorsSupplier); + + // Close outstanding leader elections + List descriptors = cc.getCoreDescriptors(); + for (CoreDescriptor descriptor : descriptors) { + closeExistingElectionContext(descriptor, sessionExpired); + } + + // Mark all cores as not leader + for (CoreDescriptor descriptor : descriptors) { + descriptor.getCloudDescriptor().setLeader(false); + descriptor.getCloudDescriptor().setHasRegistered(false); + } } - private void onReconnect(Supplier> descriptorsSupplier) { + private void onReconnect() { // on reconnect, reload cloud info log.info("ZooKeeper session re-connected ... refreshing core states after session expiration."); clearZkCollectionTerms(); try { + // Remove the live node in case it is still there + removeEphemeralLiveNode(); // recreate our watchers first so that they exist even on any problems below zkStateReader.createClusterStateWatchersAndUpdate(); @@ -453,7 +456,7 @@ private void onReconnect(Supplier> descriptorsSupplier) { cc.cancelCoreRecoveries(); try { - registerAllCoresAsDown(descriptorsSupplier, false); + registerAllCoresAsDown(false); } catch (SessionExpiredException e) { // zk has to reconnect and this will all be tried again throw e; @@ -466,26 +469,24 @@ private void onReconnect(Supplier> descriptorsSupplier) { // we have to register as live first to pick up docs in the buffer createEphemeralLiveNode(); - List descriptors = descriptorsSupplier.get(); + List descriptors = cc.getCoreDescriptors(); // re register all descriptors ExecutorService executorService = (cc != null) ? cc.getCoreZkRegisterExecutorService() : null; - if (descriptors != null) { - for (CoreDescriptor descriptor : descriptors) { - // TODO: we need to think carefully about what happens when it was a leader - // that was expired - as well as what to do about leaders/overseers with - // connection loss - try { - // unload solr cores that have been 'failed over' - throwErrorIfReplicaReplaced(descriptor); + for (CoreDescriptor descriptor : descriptors) { + // TODO: we need to think carefully about what happens when it was a leader + // that was expired - as well as what to do about leaders/overseers with + // connection loss + try { + // unload solr cores that have been 'failed over' + throwErrorIfReplicaReplaced(descriptor); - if (executorService != null) { - executorService.submit(new RegisterCoreAsync(descriptor, true, true)); - } else { - register(descriptor.getName(), descriptor, true, true, false); - } - } catch (Exception e) { - log.error("Error registering SolrCore", e); + if (executorService != null) { + executorService.submit(new RegisterCoreAsync(descriptor, true, true)); + } else { + register(descriptor.getName(), descriptor, true, true, false); } + } catch (Exception e) { + log.error("Error registering SolrCore", e); } } @@ -585,75 +586,72 @@ public int getLeaderConflictResolveWait() { return leaderConflictResolveWait; } - private void registerAllCoresAsDown( - final Supplier> registerOnReconnect, boolean updateLastPublished) - throws SessionExpiredException { - List descriptors = registerOnReconnect.get(); + private void registerAllCoresAsDown(boolean updateLastPublished) throws SessionExpiredException { + List descriptors = cc.getCoreDescriptors(); if (isClosed) return; - if (descriptors != null) { - // before registering as live, make sure everyone is in a - // down state - publishNodeAsDown(getNodeName()); - for (CoreDescriptor descriptor : descriptors) { - // if it looks like we are going to be the leader, we don't - // want to wait for the following stuff - CloudDescriptor cloudDesc = descriptor.getCloudDescriptor(); - String collection = cloudDesc.getCollectionName(); - String slice = cloudDesc.getShardId(); - try { - int children = - zkStateReader - .getZkClient() - .getChildren( - ZkStateReader.COLLECTIONS_ZKNODE - + "/" - + collection - + "/leader_elect/" - + slice - + "/election", - null, - true) - .size(); - if (children == 0) { - log.debug( - "looks like we are going to be the leader for collection {} shard {}", - collection, - slice); - continue; - } + // before registering as live, make sure everyone is in a + // down state + publishNodeAsDown(getNodeName()); + for (CoreDescriptor descriptor : descriptors) { + // if it looks like we are going to be the leader, we don't + // want to wait for the following stuff + CloudDescriptor cloudDesc = descriptor.getCloudDescriptor(); + String collection = cloudDesc.getCollectionName(); + String slice = cloudDesc.getShardId(); + try { - } catch (NoNodeException e) { + int children = + zkStateReader + .getZkClient() + .getChildren( + ZkStateReader.COLLECTIONS_ZKNODE + + "/" + + collection + + "/leader_elect/" + + slice + + "/election", + null, + true) + .size(); + if (children == 0) { log.debug( "looks like we are going to be the leader for collection {} shard {}", collection, slice); continue; - } catch (InterruptedException e2) { - Thread.currentThread().interrupt(); - } catch (SessionExpiredException e) { - // zk has to reconnect - throw e; - } catch (KeeperException e) { - log.warn("", e); - Thread.currentThread().interrupt(); } - final String coreZkNodeName = descriptor.getCloudDescriptor().getCoreNodeName(); - try { - log.debug( - "calling waitForLeaderToSeeDownState for coreZkNodeName={} collection={} shard={}", - coreZkNodeName, - collection, - slice); - waitForLeaderToSeeDownState(descriptor, coreZkNodeName); - } catch (Exception e) { - log.warn( - "There was a problem while making a best effort to ensure the leader has seen us as down, this is not unexpected as Zookeeper has just reconnected after a session expiration", - e); - if (isClosed) { - return; - } + } catch (NoNodeException e) { + log.debug( + "looks like we are going to be the leader for collection {} shard {}", + collection, + slice); + continue; + } catch (InterruptedException e2) { + Thread.currentThread().interrupt(); + } catch (SessionExpiredException e) { + // zk has to reconnect + throw e; + } catch (KeeperException e) { + log.warn("", e); + Thread.currentThread().interrupt(); + } + + final String coreZkNodeName = descriptor.getCloudDescriptor().getCoreNodeName(); + try { + log.debug( + "calling waitForLeaderToSeeDownState for coreZkNodeName={} collection={} shard={}", + coreZkNodeName, + collection, + slice); + waitForLeaderToSeeDownState(descriptor, coreZkNodeName); + } catch (Exception e) { + log.warn( + "There was a problem while making a best effort to ensure the leader has seen us as down, this is not unexpected as Zookeeper has just reconnected after a session expiration", + e); + if (isClosed) { + return; } } } @@ -663,16 +661,7 @@ public NodesSysPropsCacher getSysPropsCacher() { return sysPropsCacher; } - private void closeOutstandingElections(final Supplier> registerOnReconnect) { - List descriptors = registerOnReconnect.get(); - if (descriptors != null) { - for (CoreDescriptor descriptor : descriptors) { - closeExistingElectionContext(descriptor); - } - } - } - - private ContextKey closeExistingElectionContext(CoreDescriptor cd) { + private ContextKey closeExistingElectionContext(CoreDescriptor cd, boolean sessionExpired) { // look for old context - if we find it, cancel it String collection = cd.getCloudDescriptor().getCollectionName(); final String coreNodeName = cd.getCloudDescriptor().getCoreNodeName(); @@ -682,22 +671,16 @@ private ContextKey closeExistingElectionContext(CoreDescriptor cd) { if (prevContext != null) { prevContext.close(); - electionContexts.remove(contextKey); + // Only remove the election contexts if the session expired, otherwise the ephemeral nodes + // will still exist + if (sessionExpired) { + electionContexts.remove(contextKey); + } } return contextKey; } - private void markAllAsNotLeader(final Supplier> registerOnReconnect) { - List descriptors = registerOnReconnect.get(); - if (descriptors != null) { - for (CoreDescriptor descriptor : descriptors) { - descriptor.getCloudDescriptor().setLeader(false); - descriptor.getCloudDescriptor().setHasRegistered(false); - } - } - } - public void preClose() { this.isClosed = true; @@ -768,7 +751,6 @@ public void close() { sysPropsCacher.close(); customThreadPool.execute(() -> IOUtils.closeQuietly(cloudManager)); customThreadPool.execute(() -> IOUtils.closeQuietly(cloudSolrClient)); - customThreadPool.execute(() -> IOUtils.closeQuietly(http2SolrClient)); try { try { @@ -864,15 +846,14 @@ public SolrCloudManager getSolrCloudManager() { if (cloudManager != null) { return cloudManager; } - http2SolrClient = + var httpSolrClientBuilder = new Http2SolrClient.Builder() .withHttpClient(cc.getDefaultHttpSolrClient()) .withIdleTimeout(30000, TimeUnit.MILLISECONDS) - .withConnectionTimeout(15000, TimeUnit.MILLISECONDS) - .build(); + .withConnectionTimeout(15000, TimeUnit.MILLISECONDS); cloudSolrClient = new CloudHttp2SolrClient.Builder(new ZkClientClusterStateProvider(zkStateReader)) - .withHttpClient(http2SolrClient) + .withInternalClientBuilder(httpSolrClientBuilder) .build(); cloudManager = new SolrClientCloudManager(cloudSolrClient, cc.getObjectCache()); cloudManager.getClusterStateProvider().connect(); @@ -1209,6 +1190,19 @@ public void removeEphemeralLiveNode() throws KeeperException, InterruptedExcepti } catch (NoNodeException e) { } + cc.nodeRoles + .getRoles() + .forEach( + (role, mode) -> { + try { + zkClient.delete( + NodeRoles.getZNodeForRoleMode(role, mode) + "/" + nodeName, -1, true); + } catch (KeeperException e) { + + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); } public String getNodeName() { @@ -1740,6 +1734,14 @@ public void publish( cd.getCloudDescriptor().setLastPublished(state); } DocCollection coll = zkStateReader.getCollection(collection); + // extra handling for PRS, we need to write the PRS entries from this node directly, + // as overseer does not and should not handle those entries + if (coll != null && coll.isPerReplicaState() && coreNodeName != null) { + PerReplicaStates perReplicaStates = + PerReplicaStatesOps.fetch(coll.getZNode(), zkClient, coll.getPerReplicaStates()); + PerReplicaStatesOps.flipState(coreNodeName, state, perReplicaStates) + .persist(coll.getZNode(), zkClient); + } if (forcePublish || updateStateDotJson(coll, coreNodeName)) { if (distributedClusterStateUpdater.isDistributedStateUpdate()) { distributedClusterStateUpdater.doSingleStateUpdate( @@ -1751,14 +1753,6 @@ public void publish( overseerJobQueue.offer(m); } } - // extra handling for PRS, we need to write the PRS entries from this node directly, - // as overseer does not and should not handle those entries - if (coll != null && coll.isPerReplicaState() && coreNodeName != null) { - PerReplicaStates perReplicaStates = - PerReplicaStatesOps.fetch(coll.getZNode(), zkClient, coll.getPerReplicaStates()); - PerReplicaStatesOps.flipState(coreNodeName, state, perReplicaStates) - .persist(coll.getZNode(), zkClient); - } } finally { MDCLoggingContext.clear(); } @@ -1944,7 +1938,7 @@ public void preRegister(CoreDescriptor cd, boolean publishState) { // before becoming available, make sure we are not live and active // this also gets us our assigned shard id if it was not specified try { - checkStateInZk(cd); + checkStateInZk(cd, null); CloudDescriptor cloudDesc = cd.getCloudDescriptor(); @@ -2001,7 +1995,7 @@ private boolean isPublishAsDownOnStartup(CloudDescriptor cloudDesc) { return !replica.getNodeName().equals(getNodeName()); } - private void checkStateInZk(CoreDescriptor cd) + private void checkStateInZk(CoreDescriptor cd, Replica.State state) throws InterruptedException, NotInClusterStateException { CloudDescriptor cloudDesc = cd.getCloudDescriptor(); String nodeName = cloudDesc.getCoreNodeName(); @@ -2037,6 +2031,16 @@ private void checkStateInZk(CoreDescriptor cd) + ", ignore the exception if the replica was deleted"); return false; } + if (state != null && !state.equals(replica.getState())) { + errorMessage.set( + "coreNodeName " + + coreNodeName + + " does not have the expected state " + + state + + ", found state was: " + + replica.getState()); + return false; + } return true; }); } catch (TimeoutException e) { @@ -2120,10 +2124,15 @@ private ZkCoreNodeProps waitForLeaderToSeeDownState( if (!isLeader && !SKIP_AUTO_RECOVERY) { if (!getShardTerms(collection, shard).canBecomeLeader(myCoreNodeName)) { log.debug( - "Term of replica {} is already less than leader, so not waiting for leader to see down state.", + "Term of replica {} is already less than leader, so not waiting for leader to see down state." + + " Instead, make sure we can see the down state in Zookeeper.", myCoreNodeName); + try { + checkStateInZk(descriptor, Replica.State.DOWN); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } } else { - if (log.isInfoEnabled()) { log.info( "replica={} is making a best effort attempt to wait for leader={} to see it's DOWN state.", @@ -2796,8 +2805,7 @@ public synchronized boolean onStateChanged(DocCollection collectionState) { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof UnloadCoreOnDeletedWatcher)) return false; - UnloadCoreOnDeletedWatcher that = (UnloadCoreOnDeletedWatcher) o; + if (!(o instanceof UnloadCoreOnDeletedWatcher that)) return false; return Objects.equals(coreNodeName, that.coreNodeName) && Objects.equals(shard, that.shard) && Objects.equals(coreName, that.coreName); diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkShardTerms.java b/solr/core/src/java/org/apache/solr/cloud/ZkShardTerms.java index 83f8885325b..e58e0e5ec32 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkShardTerms.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkShardTerms.java @@ -406,21 +406,21 @@ private void registerWatcher() throws KeeperException { Watcher watcher = event -> { // Don't do anything if we are closed - if (isClosed.get()) { + if (isClosed.get() || !event.getState().equals(Watcher.Event.KeeperState.SyncConnected)) { return; } // session events are not change events, and do not remove the watcher if (Watcher.Event.EventType.None == event.getType()) { return; } + // Some events may be missed during registering a watcher, so it is safer to refresh terms + // after registering watcher retryRegisterWatcher(); - // if term node is deleted, refresh cannot possibly succeed - if (Watcher.Event.EventType.NodeDeleted == event.getType()) { - return; + // Only refresh the data if the node was created or its data changed. + if (Watcher.Event.EventType.NodeCreated == event.getType() + || Watcher.Event.EventType.NodeDataChanged == event.getType()) { + refreshTerms(); } - // Some events may be missed during register a watcher, so it is safer to refresh terms - // after registering watcher - refreshTerms(); }; try { // exists operation is faster than getData operation diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java index 1d7d3a4087d..13caeea36ec 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java @@ -17,7 +17,6 @@ package org.apache.solr.cloud.api.collections; import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.CREATE_NODE_SET; -import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP; import java.io.IOException; import java.lang.invoke.MethodHandles; @@ -70,8 +69,7 @@ public static String getCounterNodePath(String collection) { return ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection + "/counter"; } - public static int incAndGetId( - DistribStateManager stateManager, String collection, int defaultValue) { + public static int incAndGetId(DistribStateManager stateManager, String collection) { String path = ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection; try { if (!stateManager.hasData(path)) { @@ -84,8 +82,7 @@ public static int incAndGetId( path += "/counter"; if (!stateManager.hasData(path)) { try { - stateManager.createData( - path, NumberUtils.intToBytes(defaultValue), CreateMode.PERSISTENT); + stateManager.createData(path, NumberUtils.intToBytes(0), CreateMode.PERSISTENT); } catch (AlreadyExistsException e) { // it's okay if another beats us creating the node } @@ -116,7 +113,7 @@ public static int incAndGetId( stateManager.setData(path, bytes, version); return currentId; } catch (BadVersionException e) { - continue; + // Outdated version, try again } catch (IOException | KeeperException e) { throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, @@ -134,16 +131,7 @@ public static int incAndGetId( public static String assignCoreNodeName( DistribStateManager stateManager, DocCollection collection) { - // for backward compatibility; - int defaultValue = defaultCounterValue(collection, false); - String coreNodeName = - "core_node" + incAndGetId(stateManager, collection.getName(), defaultValue); - while (collection.getReplica(coreNodeName) != null) { - // there is wee chance that, the new coreNodeName id not totally unique, - // but this will be guaranteed unique for new collections - coreNodeName = "core_node" + incAndGetId(stateManager, collection.getName(), defaultValue); - } - return coreNodeName; + return "core_node" + incAndGetId(stateManager, collection.getName()); } /** @@ -204,63 +192,16 @@ public static String buildSolrCoreName( replicaNum); } - private static int defaultCounterValue( - DocCollection collection, boolean newCollection, String shard) { - if (newCollection || collection == null) return 0; - - int defaultValue; - if (collection.getSlice(shard) != null && collection.getSlice(shard).getReplicas().isEmpty()) { - return 0; - } else { - defaultValue = collection.getReplicas().size() * 2; - } - - if (collection.getReplicationFactor() != null) { - // numReplicas and replicationFactor * numSlices can be not equals, - // in case of many addReplicas or deleteReplicas are executed - defaultValue = - Math.max(defaultValue, collection.getReplicationFactor() * collection.getSlices().size()); - } - return defaultValue; - } - - private static int defaultCounterValue(DocCollection collection, boolean newCollection) { - if (newCollection) return 0; - int defaultValue = collection.getReplicas().size(); - return defaultValue; - } - public static String buildSolrCoreName( - DistribStateManager stateManager, - String collectionName, - DocCollection collection, - String shard, - Replica.Type type, - boolean newCollection) { - - int defaultValue = defaultCounterValue(collection, newCollection, shard); - int replicaNum = incAndGetId(stateManager, collectionName, defaultValue); - String coreName = buildSolrCoreName(collectionName, shard, type, replicaNum); - while (collection != null && existCoreName(coreName, collection.getSlice(shard))) { - replicaNum = incAndGetId(stateManager, collectionName, defaultValue); - coreName = buildSolrCoreName(collectionName, shard, type, replicaNum); - } - return coreName; + DistribStateManager stateManager, String collectionName, String shard, Replica.Type type) { + + int replicaNum = incAndGetId(stateManager, collectionName); + return buildSolrCoreName(collectionName, shard, type, replicaNum); } public static String buildSolrCoreName( DistribStateManager stateManager, DocCollection collection, String shard, Replica.Type type) { - return buildSolrCoreName(stateManager, collection.getName(), collection, shard, type, false); - } - - private static boolean existCoreName(String coreName, Slice slice) { - if (slice == null) return false; - for (Replica replica : slice.getReplicas()) { - if (coreName.equals(replica.getStr(CORE_NAME_PROP))) { - return true; - } - } - return false; + return buildSolrCoreName(stateManager, collection.getName(), shard, type); } public static List getLiveOrLiveAndCreateNodeSetList( diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java index 5592303580a..2b8a5007bf1 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java @@ -293,10 +293,8 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList resu } // delete related config set iff: it is auto generated AND not related to any other collection - String configSetName = coll.getConfigName(); + String configSetName = coll == null ? null : coll.getConfigName(); if (ConfigSetsHandler.isAutoGeneratedConfigSet(configSetName)) { boolean configSetIsUsedByOtherCollection = false; // make sure the configSet is not shared with other collections // Similar to what happens in: ConfigSetCmds::deleteConfigSet - for (Map.Entry entry : - zkStateReader.getClusterState().getCollectionsMap().entrySet()) { - String otherConfigSetName = entry.getValue().getConfigName(); - if (configSetName.equals(otherConfigSetName)) { - configSetIsUsedByOtherCollection = true; - break; - } - } + configSetIsUsedByOtherCollection = + zkStateReader + .getClusterState() + .collectionStream() + .map(DocCollection::getConfigName) + .anyMatch(configSetName::equals); if (!configSetIsUsedByOtherCollection) { // delete the config set diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java index 95d1eb2ac11..b09f806535a 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java @@ -101,11 +101,11 @@ public void call(ClusterState clusterState, ZkNodeProps message, NamedList sourceSlices = sourceRouter.getSearchSlicesSingle(splitKey, null, sourceCollection); if (sourceSlices.isEmpty()) { diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplicaMigrationUtils.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplicaMigrationUtils.java index 6013dea0c2d..d21aa3a5f95 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplicaMigrationUtils.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplicaMigrationUtils.java @@ -34,9 +34,7 @@ import org.apache.solr.common.SolrCloseableLatch; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.CollectionStateWatcher; -import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CoreAdminParams; @@ -305,30 +303,20 @@ static boolean cleanupReplicas( } static List getReplicasOfNodes(Collection nodeNames, ClusterState state) { - List sourceReplicas = new ArrayList<>(); - for (Map.Entry e : state.getCollectionsMap().entrySet()) { - for (Slice slice : e.getValue().getSlices()) { - for (Replica replica : slice.getReplicas()) { - if (nodeNames.contains(replica.getNodeName())) { - sourceReplicas.add(replica); - } - } - } - } - return sourceReplicas; + return state + .collectionStream() + .flatMap(dc -> dc.getSlices().stream()) + .flatMap(s -> s.getReplicas().stream()) + .filter(r -> nodeNames.contains(r.getNodeName())) + .toList(); } static List getReplicasOfNode(String nodeName, ClusterState state) { - List sourceReplicas = new ArrayList<>(); - for (Map.Entry e : state.getCollectionsMap().entrySet()) { - for (Slice slice : e.getValue().getSlices()) { - for (Replica replica : slice.getReplicas()) { - if (nodeName.equals(replica.getNodeName())) { - sourceReplicas.add(replica); - } - } - } - } - return sourceReplicas; + return state + .collectionStream() + .flatMap(dc -> dc.getSlices().stream()) + .flatMap(s -> s.getReplicas().stream()) + .filter(r -> nodeName.equals(r.getNodeName())) + .toList(); } } diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RoutedAlias.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RoutedAlias.java index d41eed3611d..7dec30d1a8d 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RoutedAlias.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RoutedAlias.java @@ -490,8 +490,7 @@ public Action(RoutedAlias sourceAlias, ActionType actionType, String targetColle @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof Action)) return false; - Action action = (Action) o; + if (!(o instanceof Action action)) return false; return Objects.equals(sourceAlias, action.sourceAlias) && actionType == action.actionType && Objects.equals(targetCollection, action.targetCollection); diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java index 355d03ae7e2..c5ef5649a22 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java @@ -1126,8 +1126,7 @@ public static String fillRanges( } } } else if (splitKey != null) { - if (router instanceof CompositeIdRouter) { - CompositeIdRouter compositeIdRouter = (CompositeIdRouter) router; + if (router instanceof CompositeIdRouter compositeIdRouter) { List tmpSubRanges = compositeIdRouter.partitionRangeByKey(splitKey, range); if (tmpSubRanges.size() == 1) { throw new SolrException( diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/NodeMutator.java b/solr/core/src/java/org/apache/solr/cloud/overseer/NodeMutator.java index f54d2569ef5..48ce2e9fde2 100644 --- a/solr/core/src/java/org/apache/solr/cloud/overseer/NodeMutator.java +++ b/solr/core/src/java/org/apache/solr/cloud/overseer/NodeMutator.java @@ -49,23 +49,12 @@ public List downNode(ClusterState clusterState, ZkNodeProps mess log.debug("DownNode state invoked for node: {}", nodeName); - List zkWriteCommands = new ArrayList<>(); - - Map collections = clusterState.getCollectionsMap(); - for (Map.Entry entry : collections.entrySet()) { - String collectionName = entry.getKey(); - DocCollection docCollection = entry.getValue(); - if (docCollection.isPerReplicaState()) continue; - - Optional zkWriteCommand = - computeCollectionUpdate(nodeName, collectionName, docCollection, zkClient); - - if (zkWriteCommand.isPresent()) { - zkWriteCommands.add(zkWriteCommand.get()); - } - } - - return zkWriteCommands; + return clusterState + .collectionStream() + .filter(entry -> !entry.isPerReplicaState()) + .map(docCollection -> computeCollectionUpdate(nodeName, docCollection, zkClient)) + .flatMap(Optional::stream) + .toList(); } /** @@ -77,7 +66,7 @@ public List downNode(ClusterState clusterState, ZkNodeProps mess * for an update to state.json, depending on the configuration of the collection. */ public static Optional computeCollectionUpdate( - String nodeName, String collectionName, DocCollection docCollection, SolrZkClient client) { + String nodeName, DocCollection docCollection, SolrZkClient client) { boolean needToUpdateCollection = false; List downedReplicas = new ArrayList<>(); final Map slicesCopy = new LinkedHashMap<>(docCollection.getSlicesMap()); @@ -107,13 +96,13 @@ public static Optional computeCollectionUpdate( return Optional.of( new ZkWriteCommand( - collectionName, + docCollection.getName(), docCollection.copyWithSlices(slicesCopy), PerReplicaStatesOps.downReplicas(downedReplicas, prs), false)); } else { return Optional.of( - new ZkWriteCommand(collectionName, docCollection.copyWithSlices(slicesCopy))); + new ZkWriteCommand(docCollection.getName(), docCollection.copyWithSlices(slicesCopy))); } } else { // No update needed for this collection diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/SliceMutator.java b/solr/core/src/java/org/apache/solr/cloud/overseer/SliceMutator.java index d5170c66474..66af10548a8 100644 --- a/solr/core/src/java/org/apache/solr/cloud/overseer/SliceMutator.java +++ b/solr/core/src/java/org/apache/solr/cloud/overseer/SliceMutator.java @@ -62,8 +62,7 @@ public SliceMutator(SolrCloudManager cloudManager) { } static SolrZkClient getZkClient(SolrCloudManager cloudManager) { - if (cloudManager instanceof SolrClientCloudManager) { - SolrClientCloudManager manager = (SolrClientCloudManager) cloudManager; + if (cloudManager instanceof SolrClientCloudManager manager) { return manager.getZkClient(); } else { return null; diff --git a/solr/core/src/java/org/apache/solr/cluster/events/impl/ClusterEventProducerFactory.java b/solr/core/src/java/org/apache/solr/cluster/events/impl/ClusterEventProducerFactory.java index a022f9c20c4..e7401940ad7 100644 --- a/solr/core/src/java/org/apache/solr/cluster/events/impl/ClusterEventProducerFactory.java +++ b/solr/core/src/java/org/apache/solr/cluster/events/impl/ClusterEventProducerFactory.java @@ -133,8 +133,7 @@ public void added(ContainerPluginsRegistry.ApiInfo plugin) { return; } Object instance = plugin.getInstance(); - if (instance instanceof ClusterEventListener) { - ClusterEventListener listener = (ClusterEventListener) instance; + if (instance instanceof ClusterEventListener listener) { clusterEventProducer.registerListener(listener); } else if (instance instanceof ClusterEventProducer) { if (ClusterEventProducer.PLUGIN_NAME.equals(plugin.getInfo().name)) { @@ -162,8 +161,7 @@ public void deleted(ContainerPluginsRegistry.ApiInfo plugin) { return; } Object instance = plugin.getInstance(); - if (instance instanceof ClusterEventListener) { - ClusterEventListener listener = (ClusterEventListener) instance; + if (instance instanceof ClusterEventListener listener) { clusterEventProducer.unregisterListener(listener); } else if (instance instanceof ClusterEventProducer) { if (ClusterEventProducer.PLUGIN_NAME.equals(plugin.getInfo().name)) { diff --git a/solr/core/src/java/org/apache/solr/cluster/events/impl/CollectionsRepairEventListener.java b/solr/core/src/java/org/apache/solr/cluster/events/impl/CollectionsRepairEventListener.java index 05a243f38ee..b2c5f0c99cd 100644 --- a/solr/core/src/java/org/apache/solr/cluster/events/impl/CollectionsRepairEventListener.java +++ b/solr/core/src/java/org/apache/solr/cluster/events/impl/CollectionsRepairEventListener.java @@ -40,7 +40,6 @@ import org.apache.solr.cluster.events.ClusterEvent; import org.apache.solr.cluster.events.ClusterEventListener; import org.apache.solr.cluster.events.NodesDownEvent; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.ReplicaCount; import org.apache.solr.common.cloud.ReplicaPosition; import org.apache.solr.common.util.SolrNamedThreadFactory; @@ -168,38 +167,44 @@ private void runRepair() { // collection / positions Map> newPositions = new HashMap<>(); try { - ClusterState clusterState = solrCloudManager.getClusterState(); - clusterState.forEachCollection( - coll -> { - // shard / number of replicas per type - Map lostReplicas = new HashMap<>(); - coll.forEachReplica( - (shard, replica) -> { - if (reallyLostNodes.contains(replica.getNodeName())) { - lostReplicas - .computeIfAbsent(shard, s -> ReplicaCount.empty()) - .increment(replica.type); - } - }); - Assign.AssignStrategy assignStrategy = Assign.createAssignStrategy(cc); - lostReplicas.forEach( - (shard, types) -> { - Assign.AssignRequest assignRequest = - new Assign.AssignRequestBuilder() - .forCollection(coll.getName()) - .forShard(Collections.singletonList(shard)) - .assignReplicas(types) - .build(); - try { - List positions = - assignStrategy.assign(solrCloudManager, assignRequest); - newPositions.put(coll.getName(), positions); - } catch (Exception e) { - log.warn( - "Exception computing positions for {}/{}: {}", coll.getName(), shard, e); - } - }); - }); + // shard / number of replicas per type + solrCloudManager + .getClusterState() + .collectionStream() + .forEach( + coll -> { + // shard / number of replicas per type + Map lostReplicas = new HashMap<>(); + coll.forEachReplica( + (shard, replica) -> { + if (reallyLostNodes.contains(replica.getNodeName())) { + lostReplicas + .computeIfAbsent(shard, s -> ReplicaCount.empty()) + .increment(replica.type); + } + }); + Assign.AssignStrategy assignStrategy = Assign.createAssignStrategy(cc); + lostReplicas.forEach( + (shard, types) -> { + Assign.AssignRequest assignRequest = + new Assign.AssignRequestBuilder() + .forCollection(coll.getName()) + .forShard(Collections.singletonList(shard)) + .assignReplicas(types) + .build(); + try { + List positions = + assignStrategy.assign(solrCloudManager, assignRequest); + newPositions.put(coll.getName(), positions); + } catch (Exception e) { + log.warn( + "Exception computing positions for {}/{}: {}", + coll.getName(), + shard, + e); + } + }); + }); } catch (IOException e) { log.warn("Exception getting cluster state", e); return; diff --git a/solr/core/src/java/org/apache/solr/cluster/maintenance/InactiveShardRemover.java b/solr/core/src/java/org/apache/solr/cluster/maintenance/InactiveShardRemover.java index 177663d1140..1d951492afd 100644 --- a/solr/core/src/java/org/apache/solr/cluster/maintenance/InactiveShardRemover.java +++ b/solr/core/src/java/org/apache/solr/cluster/maintenance/InactiveShardRemover.java @@ -143,7 +143,8 @@ public void stop() { void deleteInactiveSlices() { final ClusterState clusterState = coreContainer.getZkController().getClusterState(); Collection inactiveSlices = - clusterState.getCollectionsMap().values().stream() + clusterState + .collectionStream() .flatMap(v -> collectInactiveSlices(v).stream()) .collect(Collectors.toSet()); diff --git a/solr/core/src/java/org/apache/solr/cluster/placement/impl/MetricImpl.java b/solr/core/src/java/org/apache/solr/cluster/placement/impl/MetricImpl.java index caee316952c..b84c2892186 100644 --- a/solr/core/src/java/org/apache/solr/cluster/placement/impl/MetricImpl.java +++ b/solr/core/src/java/org/apache/solr/cluster/placement/impl/MetricImpl.java @@ -115,10 +115,9 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (!(o instanceof MetricImpl)) { + if (!(o instanceof MetricImpl that)) { return false; } - MetricImpl that = (MetricImpl) o; return name.equals(that.getName()) && internalName.equals(that.getInternalName()) && converter.equals(that.converter); diff --git a/solr/core/src/java/org/apache/solr/cluster/placement/impl/NodeMetricImpl.java b/solr/core/src/java/org/apache/solr/cluster/placement/impl/NodeMetricImpl.java index b76013d1903..cdf1fcb93fc 100644 --- a/solr/core/src/java/org/apache/solr/cluster/placement/impl/NodeMetricImpl.java +++ b/solr/core/src/java/org/apache/solr/cluster/placement/impl/NodeMetricImpl.java @@ -88,13 +88,12 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (!(o instanceof NodeMetricImpl)) { + if (!(o instanceof NodeMetricImpl that)) { return false; } if (!super.equals(o)) { return false; } - NodeMetricImpl that = (NodeMetricImpl) o; return registry == that.registry; } diff --git a/solr/core/src/java/org/apache/solr/cluster/placement/impl/SimpleClusterAbstractionsImpl.java b/solr/core/src/java/org/apache/solr/cluster/placement/impl/SimpleClusterAbstractionsImpl.java index cb86dc304e5..b518c37c5a3 100644 --- a/solr/core/src/java/org/apache/solr/cluster/placement/impl/SimpleClusterAbstractionsImpl.java +++ b/solr/core/src/java/org/apache/solr/cluster/placement/impl/SimpleClusterAbstractionsImpl.java @@ -93,7 +93,8 @@ public SolrCollection getCollection(String collectionName) { @Override public Iterator iterator() { - return clusterState.getCollectionsMap().values().stream() + return clusterState + .collectionStream() .map(SolrCollectionImpl::fromDocCollection) .collect(Collectors.toSet()) .iterator(); @@ -138,10 +139,9 @@ public boolean equals(Object obj) { if (obj == this) { return true; } - if (!(obj instanceof NodeImpl)) { + if (!(obj instanceof NodeImpl other)) { return false; } - NodeImpl other = (NodeImpl) obj; return Objects.equals(this.nodeName, other.nodeName); } @@ -306,10 +306,9 @@ public boolean equals(Object obj) { if (obj == this) { return true; } - if (!(obj instanceof ShardImpl)) { + if (!(obj instanceof ShardImpl other)) { return false; } - ShardImpl other = (ShardImpl) obj; return Objects.equals(this.shardName, other.shardName) && Objects.equals(this.collection, other.collection) && Objects.equals(this.shardState, other.shardState) @@ -466,10 +465,9 @@ public boolean equals(Object obj) { if (obj == this) { return true; } - if (!(obj instanceof ReplicaImpl)) { + if (!(obj instanceof ReplicaImpl other)) { return false; } - ReplicaImpl other = (ReplicaImpl) obj; return Objects.equals(this.replicaName, other.replicaName) && Objects.equals(this.coreName, other.coreName) && Objects.equals(this.shard, other.shard) diff --git a/solr/core/src/java/org/apache/solr/cluster/placement/plugins/OrderedNodePlacementPlugin.java b/solr/core/src/java/org/apache/solr/cluster/placement/plugins/OrderedNodePlacementPlugin.java index 86ca79526b3..a353b9dd5e2 100644 --- a/solr/core/src/java/org/apache/solr/cluster/placement/plugins/OrderedNodePlacementPlugin.java +++ b/solr/core/src/java/org/apache/solr/cluster/placement/plugins/OrderedNodePlacementPlugin.java @@ -555,10 +555,9 @@ public int hashCode() { @Override public boolean equals(Object o) { - if (!(o instanceof WeightedNode)) { + if (!(o instanceof WeightedNode on)) { return false; } else { - WeightedNode on = (WeightedNode) o; if (this.node == null) { return on.node == null; } else { diff --git a/solr/core/src/java/org/apache/solr/core/ClusterSingletons.java b/solr/core/src/java/org/apache/solr/core/ClusterSingletons.java index 0c71c853400..f8f41a58e29 100644 --- a/solr/core/src/java/org/apache/solr/core/ClusterSingletons.java +++ b/solr/core/src/java/org/apache/solr/core/ClusterSingletons.java @@ -69,8 +69,7 @@ public void added(ContainerPluginsRegistry.ApiInfo plugin) { } // register new api Object instance = plugin.getInstance(); - if (instance instanceof ClusterSingleton) { - ClusterSingleton singleton = (ClusterSingleton) instance; + if (instance instanceof ClusterSingleton singleton) { singletonMap.put(singleton.getName(), singleton); // check to see if we should immediately start this singleton if (isReady() && runSingletons.get()) { @@ -89,8 +88,7 @@ public void deleted(ContainerPluginsRegistry.ApiInfo plugin) { return; } Object instance = plugin.getInstance(); - if (instance instanceof ClusterSingleton) { - ClusterSingleton singleton = (ClusterSingleton) instance; + if (instance instanceof ClusterSingleton singleton) { singleton.stop(); singletonMap.remove(singleton.getName()); } diff --git a/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java b/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java index 403501c19f8..811ea865cd6 100644 --- a/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java +++ b/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java @@ -193,8 +193,7 @@ private static Class isEditable(boolean isXpath, List hierarchy, List if (hierarchy != null) hierarchy.add(part); if (obj == null) return null; if (i == parts.size() - 1) { - if (obj instanceof Map) { - Map map = (Map) obj; + if (obj instanceof Map map) { Object o = map.get(part); return checkType(o, isXpath, isAttr); } @@ -209,8 +208,7 @@ private static Class isEditable(boolean isXpath, List hierarchy, List new Class[] {String.class, Boolean.class, Integer.class, Float.class}; private static Class checkType(Object o, boolean isXpath, boolean isAttr) { - if (o instanceof Long) { - Long aLong = (Long) o; + if (o instanceof Long aLong) { int ten = aLong.intValue() / 10; int one = aLong.intValue() % 10; if (isXpath && isAttr && one != 0) return null; diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java index 53160cd75ee..a9f9b417abf 100644 --- a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java +++ b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java @@ -272,7 +272,7 @@ public final ConfigSet loadConfigSet(CoreDescriptor dcore) { NamedList properties = loadConfigSetProperties(dcore, coreLoader); boolean trusted = isConfigSetTrusted(coreLoader); - SolrConfig solrConfig = createSolrConfig(dcore, coreLoader, trusted); + SolrConfig solrConfig = createSolrConfig(dcore, coreLoader); return new ConfigSet( configSetName(dcore), solrConfig, @@ -314,13 +314,12 @@ public ConfigSetService(SolrResourceLoader loader, boolean shareSchema) { * * @param cd the core's CoreDescriptor * @param loader the core's resource loader - * @param isTrusted is the configset trusted? * @return a SolrConfig object */ - protected SolrConfig createSolrConfig( - CoreDescriptor cd, SolrResourceLoader loader, boolean isTrusted) throws IOException { + protected SolrConfig createSolrConfig(CoreDescriptor cd, SolrResourceLoader loader) + throws IOException { return SolrConfig.readFromResourceLoader( - loader, cd.getConfigName(), isTrusted, cd.getSubstitutableProperties()); + loader, cd.getConfigName(), cd.getSubstitutableProperties()); } /** diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index a92c73ef496..c709f4e6a21 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -59,7 +59,6 @@ import org.apache.http.client.CredentialsProvider; import org.apache.http.config.Lookup; import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.IndexWriter; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; @@ -95,6 +94,7 @@ import org.apache.solr.common.cloud.Replica.State; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.CollectionUtil; import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.IOUtils; @@ -124,7 +124,7 @@ import org.apache.solr.handler.admin.SecurityConfHandlerLocal; import org.apache.solr.handler.admin.SecurityConfHandlerZk; import org.apache.solr.handler.admin.ZookeeperInfoHandler; -import org.apache.solr.handler.admin.ZookeeperReadAPI; +import org.apache.solr.handler.admin.ZookeeperRead; import org.apache.solr.handler.admin.ZookeeperStatusHandler; import org.apache.solr.handler.api.V2ApiUtils; import org.apache.solr.handler.component.ShardHandlerFactory; @@ -138,6 +138,8 @@ import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricsContext; import org.apache.solr.pkg.SolrPackageLoader; +import org.apache.solr.request.LocalSolrQueryRequest; +import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.search.CacheConfig; @@ -153,10 +155,10 @@ import org.apache.solr.security.PublicKeyHandler; import org.apache.solr.security.SecurityPluginHolder; import org.apache.solr.security.SolrNodeKeyPair; +import org.apache.solr.update.CommitUpdateCommand; import org.apache.solr.update.SolrCoreState; import org.apache.solr.update.UpdateShardHandler; import org.apache.solr.util.OrderedExecutor; -import org.apache.solr.util.RefCounted; import org.apache.solr.util.StartupLoggingUtils; import org.apache.solr.util.stats.MetricUtils; import org.apache.zookeeper.KeeperException; @@ -592,12 +594,14 @@ private synchronized void initializeAuthenticationPlugin( } private void setupHttpClientForAuthPlugin(Object authcPlugin) { - if (authcPlugin instanceof HttpClientBuilderPlugin) { + if (authcPlugin instanceof HttpClientBuilderPlugin builderPlugin) { // Setup HttpClient for internode communication - HttpClientBuilderPlugin builderPlugin = ((HttpClientBuilderPlugin) authcPlugin); SolrHttpClientBuilder builder = builderPlugin.getHttpClientBuilder(HttpClientUtil.getHttpClientBuilder()); + // The Hadoop Auth Plugin was removed in SOLR-17540, however leaving the below reference + // for future readers, as there may be an option to simplify this logic. + // // this caused plugins like KerberosPlugin to register its intercepts, but this intercept // logic is also handled by the pki authentication code when it decides to let the plugin // handle auth via its intercept - so you would end up with two intercepts @@ -822,8 +826,7 @@ private void loadInternal() { shardHandlerFactory = ShardHandlerFactory.newInstance(cfg.getShardHandlerFactoryPluginInfo(), loader); - if (shardHandlerFactory instanceof SolrMetricProducer) { - SolrMetricProducer metricProducer = (SolrMetricProducer) shardHandlerFactory; + if (shardHandlerFactory instanceof SolrMetricProducer metricProducer) { metricProducer.initializeMetrics(solrMetricsContext, "httpShardHandler"); } @@ -870,7 +873,7 @@ private void loadInternal() { packageLoader = new SolrPackageLoader(this); registerV2ApiIfEnabled(packageLoader.getPackageAPI().editAPI); registerV2ApiIfEnabled(packageLoader.getPackageAPI().readAPI); - registerV2ApiIfEnabled(ZookeeperReadAPI.class); + registerV2ApiIfEnabled(ZookeeperRead.class); } MDCLoggingContext.setNode(this); @@ -1127,8 +1130,7 @@ private void loadInternal() { .forEach( handlerName -> { SolrRequestHandler handler = containerHandlers.get(handlerName); - if (handler instanceof ClusterSingleton) { - ClusterSingleton singleton = (ClusterSingleton) handler; + if (handler instanceof ClusterSingleton singleton) { clusterSingletons.getSingletons().put(singleton.getName(), singleton); } }); @@ -2058,19 +2060,8 @@ public void reload(String name, UUID coreId) { // force commit on old core if the new one is readOnly and prevent any new updates if (newCore.readOnly) { - RefCounted iwRef = core.getSolrCoreState().getIndexWriter(null); - if (iwRef != null) { - IndexWriter iw = iwRef.get(); - // switch old core to readOnly - core.readOnly = true; - try { - if (iw != null) { - iw.commit(); - } - } finally { - iwRef.decref(); - } - } + SolrQueryRequest req = new LocalSolrQueryRequest(core, new ModifiableSolrParams()); + core.getUpdateHandler().commit(CommitUpdateCommand.closeOnCommit(req, false)); } if (docCollection != null) { diff --git a/solr/core/src/java/org/apache/solr/core/CoreSorter.java b/solr/core/src/java/org/apache/solr/core/CoreSorter.java index b62167c5ec7..a21e64948e9 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreSorter.java +++ b/solr/core/src/java/org/apache/solr/core/CoreSorter.java @@ -157,8 +157,7 @@ public String toString() { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof CountsForEachShard)) return false; - CountsForEachShard that = (CountsForEachShard) o; + if (!(o instanceof CountsForEachShard that)) return false; return totalReplicasInDownNodes == that.totalReplicasInDownNodes && myReplicas == that.myReplicas && totalReplicasInLiveNodes == that.totalReplicasInLiveNodes; diff --git a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java index e1d22572bb5..aebbd64b7a1 100644 --- a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java @@ -339,6 +339,8 @@ public String getDataHome(CoreDescriptor cd) throws IOException { public void cleanupOldIndexDirectories( final String dataDirPath, final String currentIndexDirPath, boolean afterCoreReload) { + + // TODO SOLR-8282 move to PATH File dataDir = new File(dataDirPath); if (!dataDir.isDirectory()) { log.debug( diff --git a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java index da01cc57620..56f17c395d1 100644 --- a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java +++ b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java @@ -38,6 +38,7 @@ import org.apache.solr.common.cloud.ZkMaintenanceUtils; import org.apache.solr.common.util.Utils; import org.apache.solr.util.FileTypeMagicUtil; +import org.apache.solr.util.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -150,19 +151,30 @@ public void uploadFileToConfig( throws IOException { if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fileName)) { log.warn("Not including uploading file to config, as it is a forbidden type: {}", fileName); - } else { - if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) { - Path filePath = getConfigDir(configName).resolve(normalizePathToOsSeparator(fileName)); - if (!Files.exists(filePath) || overwriteOnExists) { - Files.write(filePath, data); - } - } else { - String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data); - log.warn( - "Not including uploading file {}, as it matched the MAGIC signature of a forbidden mime type {}", - fileName, - mimeType); - } + return; + } + if (FileTypeMagicUtil.isFileForbiddenInConfigset(data)) { + String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data); + log.warn( + "Not including uploading file {}, as it matched the MAGIC signature of a forbidden mime type {}", + fileName, + mimeType); + return; + } + final var configsetBasePath = getConfigDir(configName); + final var configsetFilePath = configsetBasePath.resolve(normalizePathToOsSeparator(fileName)); + if (!FileUtils.isPathAChildOfParent( + configsetBasePath, configsetFilePath)) { // See SOLR-17543 for context + log.warn( + "Not uploading file [{}], as it resolves to a location [{}] outside of the configset root directory [{}]", + fileName, + configsetFilePath, + configsetBasePath); + return; + } + + if (overwriteOnExists || !Files.exists(configsetFilePath)) { + Files.write(configsetFilePath, data); } } diff --git a/solr/core/src/java/org/apache/solr/core/HttpSolrClientProvider.java b/solr/core/src/java/org/apache/solr/core/HttpSolrClientProvider.java index 2bf25a896f6..e9631b26d1f 100644 --- a/solr/core/src/java/org/apache/solr/core/HttpSolrClientProvider.java +++ b/solr/core/src/java/org/apache/solr/core/HttpSolrClientProvider.java @@ -16,7 +16,6 @@ */ package org.apache.solr.core; -import java.util.List; import java.util.concurrent.TimeUnit; import org.apache.solr.client.solrj.impl.Http2SolrClient; import org.apache.solr.common.util.IOUtils; @@ -37,22 +36,24 @@ final class HttpSolrClientProvider implements AutoCloseable { private final Http2SolrClient httpSolrClient; + private final Http2SolrClient.Builder httpSolrClientBuilder; + private final InstrumentedHttpListenerFactory trackHttpSolrMetrics; HttpSolrClientProvider(UpdateShardHandlerConfig cfg, SolrMetricsContext parentContext) { trackHttpSolrMetrics = new InstrumentedHttpListenerFactory(getNameStrategy(cfg)); initializeMetrics(parentContext); - Http2SolrClient.Builder httpClientBuilder = - new Http2SolrClient.Builder().withListenerFactory(List.of(trackHttpSolrMetrics)); + this.httpSolrClientBuilder = + new Http2SolrClient.Builder().addListenerFactory(trackHttpSolrMetrics); if (cfg != null) { - httpClientBuilder + httpSolrClientBuilder .withConnectionTimeout(cfg.getDistributedConnectionTimeout(), TimeUnit.MILLISECONDS) .withIdleTimeout(cfg.getDistributedSocketTimeout(), TimeUnit.MILLISECONDS) .withMaxConnectionsPerHost(cfg.getMaxUpdateConnectionsPerHost()); } - httpSolrClient = httpClientBuilder.build(); + httpSolrClient = httpSolrClientBuilder.build(); } private InstrumentedHttpListenerFactory.NameStrategy getNameStrategy( @@ -76,7 +77,7 @@ Http2SolrClient getSolrClient() { } void setSecurityBuilder(HttpClientBuilderPlugin builder) { - builder.setup(httpSolrClient); + builder.setup(httpSolrClientBuilder, httpSolrClient); } @Override diff --git a/solr/core/src/java/org/apache/solr/core/OverlaidConfigNode.java b/solr/core/src/java/org/apache/solr/core/OverlaidConfigNode.java index 30561311dfc..c6fab13fef7 100644 --- a/solr/core/src/java/org/apache/solr/core/OverlaidConfigNode.java +++ b/solr/core/src/java/org/apache/solr/core/OverlaidConfigNode.java @@ -19,9 +19,9 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.function.Function; import java.util.function.Predicate; -import org.apache.solr.cluster.api.SimpleMap; import org.apache.solr.common.ConfigNode; /** A config node impl which has an overlay */ @@ -80,7 +80,7 @@ public String name() { } @Override - public SimpleMap attributes() { + public Map attributes() { return delegate.attributes(); } diff --git a/solr/core/src/java/org/apache/solr/core/PluginBag.java b/solr/core/src/java/org/apache/solr/core/PluginBag.java index 629c480d5ce..7624a4c13dd 100644 --- a/solr/core/src/java/org/apache/solr/core/PluginBag.java +++ b/solr/core/src/java/org/apache/solr/core/PluginBag.java @@ -239,8 +239,7 @@ public PluginHolder put(String name, PluginHolder plugin) { if (loadV2ApisIfPresent) { if (plugin.isLoaded()) { T inst = plugin.get(); - if (inst instanceof ApiSupport) { - ApiSupport apiSupport = (ApiSupport) inst; + if (inst instanceof ApiSupport apiSupport) { if (registerApi == null) registerApi = apiSupport.registerV2(); if (disableHandler == null) disableHandler = !apiSupport.registerV1(); @@ -366,8 +365,7 @@ public boolean isLoaded(String name) { private void registerMBean(Object inst, SolrCore core, String pluginKey) { if (core == null) return; - if (inst instanceof SolrInfoBean) { - SolrInfoBean mBean = (SolrInfoBean) inst; + if (inst instanceof SolrInfoBean mBean) { String name = (inst instanceof SolrRequestHandler) ? pluginKey : mBean.getName(); core.registerInfoBean(name, mBean); } diff --git a/solr/core/src/java/org/apache/solr/core/PluginInfo.java b/solr/core/src/java/org/apache/solr/core/PluginInfo.java index 5c24ee9a8f4..2e1991973df 100644 --- a/solr/core/src/java/org/apache/solr/core/PluginInfo.java +++ b/solr/core/src/java/org/apache/solr/core/PluginInfo.java @@ -118,7 +118,7 @@ public PluginInfo(ConfigNode node, String err, boolean requireName, boolean requ className = cName.className; pkgName = cName.pkg; initArgs = DOMUtil.childNodesToNamedList(node); - attributes = node.attributes().asMap(); + attributes = node.attributes(); children = loadSubPlugins(node); isFromSolrConfig = true; } @@ -142,8 +142,7 @@ public PluginInfo(String type, Map map) { for (Map.Entry entry : map.entrySet()) { if (NAME.equals(entry.getKey()) || CLASS_NAME.equals(entry.getKey())) continue; Object value = entry.getValue(); - if (value instanceof List) { - List list = (List) value; + if (value instanceof List list) { if (!list.isEmpty() && list.get(0) instanceof Map) { // this is a subcomponent for (Object o : list) { if (o instanceof Map) o = new NamedList<>((Map) o); @@ -234,8 +233,7 @@ public Map toMap(Map map) { Object old = m.get(child.name); if (old == null) { m.put(child.name, child.toMap(new LinkedHashMap<>())); - } else if (old instanceof List) { - List list = (List) old; + } else if (old instanceof List list) { list.add(child.toMap(new LinkedHashMap<>())); } else { ArrayList l = new ArrayList(); diff --git a/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java b/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java index a612b27809a..dd7b679b685 100644 --- a/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java +++ b/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java @@ -82,13 +82,11 @@ public void close() {} for (int i = 0; i < values.size(); i++) { Object o = values.getVal(i); SolrDocumentFetcher docFetcher = null; - if (o instanceof ResultContext) { - ResultContext ctx = (ResultContext) o; + if (o instanceof ResultContext ctx) { o = ctx.getDocList(); docFetcher = ctx.getDocFetcher(); } - if (o instanceof DocList) { - DocList docs = (DocList) o; + if (o instanceof DocList docs) { if (docFetcher == null) { docFetcher = newSearcher.getDocFetcher(); } diff --git a/solr/core/src/java/org/apache/solr/core/RequestParams.java b/solr/core/src/java/org/apache/solr/core/RequestParams.java index e5937d0856d..e480de4bfb5 100644 --- a/solr/core/src/java/org/apache/solr/core/RequestParams.java +++ b/solr/core/src/java/org/apache/solr/core/RequestParams.java @@ -52,8 +52,7 @@ public RequestParams(Map data, int znodeVersion) { Map paramsets = (Map) data.get(NAME); if (paramsets != null) { for (Map.Entry e : paramsets.entrySet()) { - if (e.getValue() instanceof Map) { - Map value = (Map) e.getValue(); + if (e.getValue() instanceof Map value) { this.paramsets.put((String) e.getKey(), createParamSet(value, 0l)); } } @@ -87,8 +86,7 @@ private static Map getMapCopy(Map value) { } } else if (entry.getValue() == null) { copy.put(entry.getKey(), null); - } else if (entry.getValue() instanceof List) { - List l = (List) entry.getValue(); + } else if (entry.getValue() instanceof List l) { String[] sarr = new String[l.size()]; for (int i = 0; i < l.size(); i++) { if (l.get(i) != null) sarr[i] = String.valueOf(l.get(i)); @@ -138,8 +136,7 @@ public RequestParams setParams(String name, ParamSet paramSet) { public static RequestParams getFreshRequestParams( SolrResourceLoader loader, RequestParams requestParams) { - if (loader instanceof ZkSolrResourceLoader) { - ZkSolrResourceLoader resourceLoader = (ZkSolrResourceLoader) loader; + if (loader instanceof ZkSolrResourceLoader resourceLoader) { try { Stat stat = resourceLoader diff --git a/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java b/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java index 9c52f912784..a7d9305b3c3 100644 --- a/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java +++ b/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java @@ -126,8 +126,7 @@ public DocValuesFormat getDocValuesFormatForField(String field) { public KnnVectorsFormat getKnnVectorsFormatForField(String field) { final SchemaField schemaField = core.getLatestSchema().getFieldOrNull(field); FieldType fieldType = (schemaField == null ? null : schemaField.getType()); - if (fieldType instanceof DenseVectorField) { - DenseVectorField vectorType = (DenseVectorField) fieldType; + if (fieldType instanceof DenseVectorField vectorType) { String knnAlgorithm = vectorType.getKnnAlgorithm(); if (DenseVectorField.HNSW_ALGORITHM.equals(knnAlgorithm)) { int maxConn = vectorType.getHnswMaxConn(); diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java index 173a44b9ba2..e7adaf8d2f6 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java +++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java @@ -17,7 +17,6 @@ package org.apache.solr.core; import static org.apache.solr.common.params.CommonParams.NAME; -import static org.apache.solr.common.params.CommonParams.PATH; import static org.apache.solr.core.ConfigOverlay.ZNODEVER; import static org.apache.solr.core.SolrConfig.PluginOpts.LAZY; import static org.apache.solr.core.SolrConfig.PluginOpts.MULTI_OK; @@ -31,7 +30,6 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.lang.invoke.MethodHandles; -import java.net.MalformedURLException; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; @@ -96,7 +94,7 @@ /** * Provides a static reference to a Config object modeling the main configuration data for a Solr - * instance -- typically found in "solrconfig.xml". + * core -- typically found in "solrconfig.xml". */ public class SolrConfig implements MapSerializable { @@ -143,16 +141,13 @@ public enum PluginOpts { * @param name the configuration name used by the loader if the stream is null */ public SolrConfig(Path instanceDir, String name) throws IOException { - this(new SolrResourceLoader(instanceDir), name, true, null); + this(new SolrResourceLoader(instanceDir), name, null); } public static SolrConfig readFromResourceLoader( - SolrResourceLoader loader, - String name, - boolean isConfigsetTrusted, - Properties substitutableProperties) { + SolrResourceLoader loader, String name, Properties substitutableProperties) { try { - return new SolrConfig(loader, name, isConfigsetTrusted, substitutableProperties); + return new SolrConfig(loader, name, substitutableProperties); } catch (Exception e) { String resource; if (loader instanceof ZkSolrResourceLoader) { @@ -173,14 +168,11 @@ private class ResourceProvider implements Function { ResourceProvider(InputStream in) throws IOException { this.in = in; - if (in instanceof ZkSolrResourceLoader.ZkByteArrayInputStream) { - ZkSolrResourceLoader.ZkByteArrayInputStream zkin = - (ZkSolrResourceLoader.ZkByteArrayInputStream) in; + if (in instanceof ZkSolrResourceLoader.ZkByteArrayInputStream zkin) { zkVersion = zkin.getStat().getVersion(); hash = Objects.hash(zkin.getStat().getCtime(), zkVersion, overlay.getVersion()); this.fileName = zkin.fileName; - } else if (in instanceof SolrResourceLoader.SolrFileInputStream) { - SolrResourceLoader.SolrFileInputStream sfin = (SolrResourceLoader.SolrFileInputStream) in; + } else if (in instanceof SolrResourceLoader.SolrFileInputStream sfin) { zkVersion = (int) sfin.getLastModified(); hash = Objects.hash(sfin.getLastModified(), overlay.getVersion()); } @@ -199,15 +191,9 @@ public InputStream apply(String s) { * * @param loader the resource loader * @param name the configuration name - * @param isConfigsetTrusted false if configset was uploaded using unsecured configset upload API, - * true otherwise * @param substitutableProperties optional properties to substitute into the XML */ - private SolrConfig( - SolrResourceLoader loader, - String name, - boolean isConfigsetTrusted, - Properties substitutableProperties) { + private SolrConfig(SolrResourceLoader loader, String name, Properties substitutableProperties) { this.resourceLoader = loader; this.resourceName = name; this.substituteProperties = substitutableProperties; @@ -240,7 +226,7 @@ private SolrConfig( rootDataHashCode = this.root.txt().hashCode(); getRequestParams(); - initLibs(loader, isConfigsetTrusted); + initLibs(loader); String val = root.child( IndexSchema.LUCENE_MATCH_VERSION_PARAM, @@ -937,11 +923,10 @@ public PluginInfo getPluginInfo(String type) { SolrException.ErrorCode.SERVER_ERROR, "Multiple plugins configured for type: " + type); } - private void initLibs(SolrResourceLoader loader, boolean isConfigsetTrusted) { + private void initLibs(SolrResourceLoader loader) { // TODO Want to remove SolrResourceLoader.getInstancePath; it can be on a Standalone subclass. // For Zk subclass, it's needed for the time being as well. We could remove that one if we - // remove two things in SolrCloud: (1) instancePath/lib and (2) solrconfig lib directives with - // relative paths. Can wait till 9.0. + // remove "instancePath/lib" in SolrCloud. Can wait till 9.0. Path instancePath = loader.getInstancePath(); List urls = new ArrayList<>(); @@ -953,48 +938,15 @@ private void initLibs(SolrResourceLoader loader, boolean isConfigsetTrusted) { log.warn("Couldn't add files from {} to classpath: {}", libPath, e); } } - - List nodes = root.getAll("lib"); - if (nodes != null && nodes.size() > 0) { - if (!isConfigsetTrusted) { - throw new SolrException( - ErrorCode.UNAUTHORIZED, - "The configset for this collection was uploaded without any authentication in place," - + " and use of is not available for collections with untrusted configsets. To use this component, re-upload the configset" - + " after enabling authentication and authorization."); - } - - for (int i = 0; i < nodes.size(); i++) { - ConfigNode node = nodes.get(i); - String baseDir = node.attr("dir"); - String path = node.attr(PATH); - if (null != baseDir) { - // :TODO: add support for a simpler 'glob' mutually exclusive of regex - Path dir = instancePath.resolve(baseDir); - String regex = node.attr("regex"); - try { - if (regex == null) urls.addAll(SolrResourceLoader.getURLs(dir)); - else urls.addAll(SolrResourceLoader.getFilteredURLs(dir, regex)); - } catch (IOException e) { - log.warn("Couldn't add files from {} filtered by {} to classpath: {}", dir, regex, e); - } - } else if (null != path) { - final Path dir = instancePath.resolve(path); - try { - urls.add(dir.toUri().toURL()); - } catch (MalformedURLException e) { - log.warn("Couldn't add file {} to classpath: {}", dir, e); - } - } else { - throw new RuntimeException("lib: missing mandatory attributes: 'dir' or 'path'"); - } - } - } - if (!urls.isEmpty()) { loader.addToClassLoader(urls); loader.reloadLuceneSPI(); } + + List nodes = root.getAll("lib"); + if (nodes != null && nodes.size() > 0) { + log.warn(" entries no longer supported in solrconfig.xml; ignoring..."); + } } public int getMultipartUploadLimitKB() { diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index f56cb846767..3611b943feb 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -106,9 +106,9 @@ import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager.SnapshotMetaData; import org.apache.solr.handler.IndexFetcher; -import org.apache.solr.handler.ReplicationHandler; import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.handler.SolrConfigHandler; +import org.apache.solr.handler.admin.api.ReplicationAPIBase; import org.apache.solr.handler.api.V2ApiUtils; import org.apache.solr.handler.component.HighlightComponent; import org.apache.solr.handler.component.SearchComponent; @@ -949,8 +949,7 @@ public static T createInstance( } catch (Exception e) { // The JVM likes to wrap our helpful SolrExceptions in things like // "InvocationTargetException" that have no useful getMessage - if (null != e.getCause() && e.getCause() instanceof SolrException) { - SolrException inner = (SolrException) e.getCause(); + if (null != e.getCause() && e.getCause() instanceof SolrException inner) { throw inner; } @@ -995,8 +994,7 @@ private UpdateHandler createReloadedUpdateHandler( } catch (Exception e) { // The JVM likes to wrap our helpful SolrExceptions in things like // "InvocationTargetException" that have no useful getMessage - if (null != e.getCause() && e.getCause() instanceof SolrException) { - SolrException inner = (SolrException) e.getCause(); + if (null != e.getCause() && e.getCause() instanceof SolrException inner) { throw inner; } @@ -1356,6 +1354,7 @@ public void initializeMetrics(SolrMetricsContext parentContext, String scope) { Category.CORE.toString()); } + // TODO SOLR-8282 move to PATH // initialize disk total / free metrics Path dataDirPath = Paths.get(dataDir); File dataDirFile = dataDirPath.toFile(); @@ -1840,13 +1839,15 @@ private void doClose() { } // Close the snapshots meta-data directory. - Directory snapshotsDir = snapshotMgr.getSnapshotsDir(); - try { - this.directoryFactory.release(snapshotsDir); - } catch (Throwable e) { - log.error("Exception releasing snapshotsDir {}", snapshotsDir, e); - if (e instanceof Error) { - throw (Error) e; + if (snapshotMgr != null) { + Directory snapshotsDir = snapshotMgr.getSnapshotsDir(); + try { + this.directoryFactory.release(snapshotsDir); + } catch (Throwable e) { + log.error("Exception releasing snapshotsDir {}", snapshotsDir, e); + if (e instanceof Error) { + throw (Error) e; + } } } @@ -3018,7 +3019,7 @@ public PluginBag getResponseWriters() { m.put("schema.xml", new SchemaXmlResponseWriter()); m.put("smile", new SmileResponseWriter()); m.put(PROMETHEUS_METRICS_WT, new PrometheusResponseWriter()); - m.put(ReplicationHandler.FILE_STREAM, getFileStreamWriter()); + m.put(ReplicationAPIBase.FILE_STREAM, getFileStreamWriter()); DEFAULT_RESPONSE_WRITERS = Collections.unmodifiableMap(m); try { m.put( @@ -3037,7 +3038,7 @@ private static BinaryResponseWriter getFileStreamWriter() { @Override public void write(OutputStream out, SolrQueryRequest req, SolrQueryResponse response) throws IOException { - RawWriter rawWriter = (RawWriter) response.getValues().get(ReplicationHandler.FILE_STREAM); + RawWriter rawWriter = (RawWriter) response.getValues().get(ReplicationAPIBase.FILE_STREAM); if (rawWriter != null) { rawWriter.write(out); if (rawWriter instanceof Closeable) ((Closeable) rawWriter).close(); @@ -3046,7 +3047,7 @@ public void write(OutputStream out, SolrQueryRequest req, SolrQueryResponse resp @Override public String getContentType(SolrQueryRequest request, SolrQueryResponse response) { - RawWriter rawWriter = (RawWriter) response.getValues().get(ReplicationHandler.FILE_STREAM); + RawWriter rawWriter = (RawWriter) response.getValues().get(ReplicationAPIBase.FILE_STREAM); if (rawWriter != null) { return rawWriter.getContentType(); } else { @@ -3392,8 +3393,7 @@ public boolean removeConfListener(Runnable runnable) { * some data so that events are triggered. */ private void registerConfListener() { - if (!(resourceLoader instanceof ZkSolrResourceLoader)) return; - final ZkSolrResourceLoader zkSolrResourceLoader = (ZkSolrResourceLoader) resourceLoader; + if (!(resourceLoader instanceof ZkSolrResourceLoader zkSolrResourceLoader)) return; if (zkSolrResourceLoader != null) zkSolrResourceLoader .getZkController() @@ -3413,8 +3413,7 @@ public static Runnable getConfListener(SolrCore core, ZkSolrResourceLoader zkSol zkSolrResourceLoader.getConfigSetZkPath() + "/" + core.getSolrConfig().getName(); String schemaRes = null; if (core.getLatestSchema().isMutable() - && core.getLatestSchema() instanceof ManagedIndexSchema) { - ManagedIndexSchema mis = (ManagedIndexSchema) core.getLatestSchema(); + && core.getLatestSchema() instanceof ManagedIndexSchema mis) { schemaRes = mis.getResourceName(); } final String managedSchemaResourcePath = diff --git a/solr/core/src/java/org/apache/solr/core/SolrDeletionPolicy.java b/solr/core/src/java/org/apache/solr/core/SolrDeletionPolicy.java index 2b8232deef0..18043a389ad 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrDeletionPolicy.java +++ b/solr/core/src/java/org/apache/solr/core/SolrDeletionPolicy.java @@ -16,9 +16,9 @@ */ package org.apache.solr.core; -import java.io.File; import java.io.IOException; import java.lang.invoke.MethodHandles; +import java.nio.file.Path; import java.util.List; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexDeletionPolicy; @@ -113,8 +113,7 @@ public final String toString() { protected void appendDetails(StringBuilder sb, IndexCommit c) { Directory dir = c.getDirectory(); - if (dir instanceof FSDirectory) { - FSDirectory fsd = (FSDirectory) dir; + if (dir instanceof FSDirectory fsd) { sb.append("dir=").append(fsd.getDirectory()); } else { sb.append("dir=").append(dir); @@ -198,10 +197,9 @@ private String getId(IndexCommit commit) { // For anything persistent, make something that will // be the same, regardless of the Directory instance. - if (dir instanceof FSDirectory) { - FSDirectory fsd = (FSDirectory) dir; - File fdir = fsd.getDirectory().toFile(); - sb.append(fdir.getPath()); + if (dir instanceof FSDirectory fsd) { + Path fdir = fsd.getDirectory(); + sb.append(fdir.toString()); } else { sb.append(dir); } diff --git a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java index b19d65320ad..3525fd2cc95 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java +++ b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java @@ -31,6 +31,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Properties; import java.util.Set; import java.util.function.Consumer; @@ -131,10 +132,10 @@ public static NodeConfig fromConfig( // since it is arranged as a separate section it is placed here Map coreAdminHandlerActions = readNodeListAsNamedList(root.get("coreAdminHandlerActions"), "") - .asMap() + .asShallowMap() .entrySet() .stream() - .collect(Collectors.toMap(item -> item.getKey(), item -> item.getValue().toString())); + .collect(Collectors.toMap(Entry::getKey, item -> item.getValue().toString())); UpdateShardHandlerConfig updateConfig; if (deprecatedUpdateConfig == null) { @@ -733,7 +734,7 @@ private static MetricsConfig getMetricsConfig(ConfigNode metrics) { ConfigNode caching = metrics.get("solr/metrics/caching"); if (caching != null) { Object threadsCachingIntervalSeconds = - DOMUtil.childNodesToNamedList(caching).get("threadsIntervalSeconds", null); + DOMUtil.childNodesToNamedList(caching).get("threadsIntervalSeconds"); builder.setCacheConfig( new MetricsConfig.CacheConfig( threadsCachingIntervalSeconds == null @@ -760,8 +761,7 @@ private static Map getCachesConfig( } private static Object decodeNullValue(Object o) { - if (o instanceof String) { // check if it's a JSON object - String str = (String) o; + if (o instanceof String str) { // check if it's a JSON object if (!str.isBlank() && (str.startsWith("{") || str.startsWith("["))) { try { o = Utils.fromJSONString((String) o); diff --git a/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java index 2e53828ca78..5d02de67aa9 100644 --- a/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java @@ -16,7 +16,6 @@ */ package org.apache.solr.core; -import java.io.File; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.nio.file.AtomicMoveNotSupportedException; @@ -78,7 +77,7 @@ protected LockFactory createLockFactory(String rawLockType) throws IOException { @Override public String normalize(String path) throws IOException { - return super.normalize(new File(path).getCanonicalPath()); + return super.normalize(Path.of(path).toAbsolutePath().normalize().toString()); } @Override diff --git a/solr/core/src/java/org/apache/solr/core/ZkContainer.java b/solr/core/src/java/org/apache/solr/core/ZkContainer.java index 6dd7b2d571d..a8c00bc7cd9 100644 --- a/solr/core/src/java/org/apache/solr/core/ZkContainer.java +++ b/solr/core/src/java/org/apache/solr/core/ZkContainer.java @@ -19,16 +19,13 @@ import static org.apache.solr.common.cloud.ZkStateReader.HTTPS; import static org.apache.solr.common.cloud.ZkStateReader.HTTPS_PORT_PROP; -import java.io.File; import java.io.IOException; import java.lang.invoke.MethodHandles; +import java.nio.file.Path; import java.nio.file.Paths; -import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeoutException; import java.util.function.Predicate; -import java.util.function.Supplier; -import java.util.stream.Collectors; import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; import org.apache.solr.cloud.SolrZkServer; import org.apache.solr.cloud.ZkController; @@ -99,7 +96,7 @@ public void initZooKeeper(final CoreContainer cc, CloudConfig config) { new SolrZkServer( stripChroot(zkRun), stripChroot(config.getZkHost()), - new File(zkDataHome), + Path.of(zkDataHome), zkConfHome, config.getSolrHostPort()); zkServer.parseConfig(); @@ -132,15 +129,8 @@ public void initZooKeeper(final CoreContainer cc, CloudConfig config) { "A chroot was specified in ZkHost but the znode doesn't exist. " + zookeeperHost); } - Supplier> descriptorsSupplier = - () -> - cc.getCores().stream() - .map(SolrCore::getCoreDescriptor) - .collect(Collectors.toList()); - ZkController zkController = - new ZkController( - cc, zookeeperHost, zkClientConnectTimeout, config, descriptorsSupplier); + new ZkController(cc, zookeeperHost, zkClientConnectTimeout, config); if (zkRun != null) { if (StrUtils.isNotNullOrEmpty(System.getProperty(HTTPS_PORT_PROP))) { @@ -184,7 +174,7 @@ public SolrMetricsContext getSolrMetricsContext() { } private String stripChroot(String zkRun) { - if (zkRun == null || zkRun.trim().length() == 0 || zkRun.lastIndexOf('/') < 0) return zkRun; + if (zkRun == null || zkRun.trim().isEmpty() || zkRun.lastIndexOf('/') < 0) return zkRun; return zkRun.substring(0, zkRun.lastIndexOf('/')); } diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupId.java b/solr/core/src/java/org/apache/solr/core/backup/BackupId.java index 3effa11b4e8..e5e335609cd 100644 --- a/solr/core/src/java/org/apache/solr/core/backup/BackupId.java +++ b/solr/core/src/java/org/apache/solr/core/backup/BackupId.java @@ -62,8 +62,7 @@ public int compareTo(BackupId o) { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof BackupId)) return false; - BackupId backupId = (BackupId) o; + if (!(o instanceof BackupId backupId)) return false; return id == backupId.id; } diff --git a/solr/core/src/java/org/apache/solr/core/backup/Checksum.java b/solr/core/src/java/org/apache/solr/core/backup/Checksum.java index 888c6489b1d..1d57d72ff7d 100644 --- a/solr/core/src/java/org/apache/solr/core/backup/Checksum.java +++ b/solr/core/src/java/org/apache/solr/core/backup/Checksum.java @@ -32,9 +32,8 @@ public Checksum(long checksum, long size) { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof Checksum)) return false; - Checksum checksum = (Checksum) o; - return size == checksum.size && this.checksum == checksum.checksum; + if (!(o instanceof Checksum cs)) return false; + return size == cs.size && this.checksum == cs.checksum; } @Override diff --git a/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepositoryFactory.java b/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepositoryFactory.java index c838ebd4185..d699d2005d5 100644 --- a/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepositoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepositoryFactory.java @@ -76,8 +76,7 @@ public BackupRepository newInstance(SolrResourceLoader loader, String name) { BackupRepository backupRepository = loader.newInstance(repo.className, BackupRepository.class); backupRepository.init(repo.initArgs); - if (backupRepository instanceof DelegatingBackupRepository) { - DelegatingBackupRepository delegatingRepo = (DelegatingBackupRepository) backupRepository; + if (backupRepository instanceof DelegatingBackupRepository delegatingRepo) { String delegateName = (String) repo.initArgs.get(PARAM_DELEGATE_REPOSITORY_NAME); if (delegateName == null) { throw new SolrException( diff --git a/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java b/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java index 14e8e1e0c34..ae16148a1e6 100644 --- a/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java +++ b/solr/core/src/java/org/apache/solr/filestore/FileStoreAPI.java @@ -57,8 +57,7 @@ public int hashCode() { @Override public boolean equals(Object that) { - if (that instanceof MetaData) { - MetaData metaData = (MetaData) that; + if (that instanceof MetaData metaData) { return Objects.equals(sha512, metaData.sha512) && Objects.equals(signatures, metaData.signatures) && Objects.equals(otherAttribs, metaData.otherAttribs); diff --git a/solr/core/src/java/org/apache/solr/filestore/NodeFileStore.java b/solr/core/src/java/org/apache/solr/filestore/NodeFileStore.java index 2b12274ad26..e6c61419437 100644 --- a/solr/core/src/java/org/apache/solr/filestore/NodeFileStore.java +++ b/solr/core/src/java/org/apache/solr/filestore/NodeFileStore.java @@ -17,7 +17,7 @@ package org.apache.solr.filestore; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM; +import static org.apache.solr.handler.admin.api.ReplicationAPIBase.FILE_STREAM; import static org.apache.solr.response.RawResponseWriter.CONTENT; import static org.apache.solr.security.PermissionNameProvider.Name.FILESTORE_READ_PERM; diff --git a/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java b/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java index faa79850ab6..ed3bc83ccd2 100644 --- a/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java +++ b/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java @@ -305,8 +305,7 @@ public void reflect(Class attClass, String key, Object valu k = ATTRIBUTE_MAPPING.get(k); } - if (value instanceof BytesRef) { - final BytesRef p = (BytesRef) value; + if (value instanceof BytesRef p) { value = p.toString(); } diff --git a/solr/core/src/java/org/apache/solr/handler/BlobHandler.java b/solr/core/src/java/org/apache/solr/handler/BlobHandler.java index 003a9412741..540cc03f9ce 100644 --- a/solr/core/src/java/org/apache/solr/handler/BlobHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/BlobHandler.java @@ -58,6 +58,7 @@ import org.apache.solr.core.PluginInfo; import org.apache.solr.core.SolrCore; import org.apache.solr.handler.admin.api.GetBlobInfoAPI; +import org.apache.solr.handler.admin.api.ReplicationAPIBase; import org.apache.solr.handler.admin.api.UploadBlobAPI; import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; @@ -194,7 +195,7 @@ public void handleRequestBody(final SolrQueryRequest req, SolrQueryResponse rsp) return; } } - if (ReplicationHandler.FILE_STREAM.equals(req.getParams().get(CommonParams.WT))) { + if (ReplicationAPIBase.FILE_STREAM.equals(req.getParams().get(CommonParams.WT))) { if (blobName == null) { throw new SolrException( SolrException.ErrorCode.NOT_FOUND, @@ -211,7 +212,7 @@ public void handleRequestBody(final SolrQueryRequest req, SolrQueryResponse rsp) new Sort(new SortField("version", SortField.Type.LONG, true))); if (docs.totalHits.value > 0) { rsp.add( - ReplicationHandler.FILE_STREAM, + ReplicationAPIBase.FILE_STREAM, new SolrCore.RawWriter() { @Override diff --git a/solr/core/src/java/org/apache/solr/handler/CatStream.java b/solr/core/src/java/org/apache/solr/handler/CatStream.java index 70ee2b65242..fdffe29e6c0 100644 --- a/solr/core/src/java/org/apache/solr/handler/CatStream.java +++ b/solr/core/src/java/org/apache/solr/handler/CatStream.java @@ -49,7 +49,7 @@ public class CatStream extends TupleStream implements Expressible { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private final String commaDelimitedFilepaths; + protected final String commaDelimitedFilepaths; private final int maxLines; // -1 for no max private StreamContext context; @@ -163,7 +163,7 @@ public Explanation toExplanation(StreamFactory factory) throws IOException { .withExpression(toExpression(factory).toString()); } - private List validateAndSetFilepathsInSandbox() { + protected List validateAndSetFilepathsInSandbox() { final List crawlSeeds = new ArrayList<>(); for (String crawlRootStr : commaDelimitedFilepaths.split(",")) { Path crawlRootPath = chroot.resolve(crawlRootStr).normalize(); diff --git a/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java b/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java index e161d55e5b3..4807e19aca2 100644 --- a/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java +++ b/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java @@ -23,7 +23,6 @@ import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.REQUESTID; import static org.apache.solr.common.params.CollectionParams.ACTION; import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDROLE; -import static org.apache.solr.common.params.CollectionParams.CollectionAction.CLUSTERPROP; import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETESTATUS; import static org.apache.solr.common.params.CollectionParams.CollectionAction.OVERSEERSTATUS; import static org.apache.solr.common.params.CollectionParams.CollectionAction.REMOVEROLE; @@ -43,7 +42,6 @@ import org.apache.solr.api.EndPoint; import org.apache.solr.api.PayloadObj; import org.apache.solr.client.solrj.cloud.DistribStateManager; -import org.apache.solr.client.solrj.request.beans.ClusterPropPayload; import org.apache.solr.client.solrj.request.beans.RateLimiterPayload; import org.apache.solr.common.SolrException; import org.apache.solr.common.annotation.JsonProperty; @@ -275,26 +273,6 @@ public void removeRole(PayloadObj obj) throws Exception { collectionsHandler.handleRequestBody(wrapParams(obj.getRequest(), m), obj.getResponse()); } - @Command(name = "set-obj-property") - public void setObjProperty(PayloadObj obj) { - // Not using the object directly here because the API differentiate between {name:null} and {} - Map m = obj.getDataMap(); - ClusterProperties clusterProperties = - new ClusterProperties(getCoreContainer().getZkController().getZkClient()); - try { - clusterProperties.setClusterProperties(m); - } catch (Exception e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error in API", e); - } - } - - @Command(name = "set-property") - public void setProperty(PayloadObj> obj) throws Exception { - Map m = obj.getDataMap(); - m.put("action", CLUSTERPROP.toString()); - collectionsHandler.handleRequestBody(wrapParams(obj.getRequest(), m), obj.getResponse()); - } - @Command(name = "set-ratelimiter") public void setRateLimiters(PayloadObj payLoad) { RateLimiterPayload rateLimiterConfig = payLoad.get(); diff --git a/solr/core/src/java/org/apache/solr/handler/ExportHandler.java b/solr/core/src/java/org/apache/solr/handler/ExportHandler.java index 85a99dfaea9..8fa1ba64e38 100644 --- a/solr/core/src/java/org/apache/solr/handler/ExportHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/ExportHandler.java @@ -31,6 +31,7 @@ import org.apache.solr.common.params.SolrParams; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrCore; +import org.apache.solr.handler.admin.api.ReplicationAPIBase; import org.apache.solr.handler.component.SearchHandler; import org.apache.solr.handler.export.ExportWriter; import org.apache.solr.handler.export.ExportWriterStream; @@ -125,10 +126,10 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw } String wt = req.getParams().get(CommonParams.WT, JSON); if ("xsort".equals(wt)) wt = JSON; - Map map = Map.of(CommonParams.WT, ReplicationHandler.FILE_STREAM); + Map map = Map.of(CommonParams.WT, ReplicationAPIBase.FILE_STREAM); req.setParams(SolrParams.wrapDefaults(new MapSolrParams(map), req.getParams())); rsp.add( - ReplicationHandler.FILE_STREAM, + ReplicationAPIBase.FILE_STREAM, new ExportWriter( req, rsp, wt, initialStreamContext, solrMetricsContext, writerMetricsPath)); } diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java index 66acda5c185..a5a467386db 100644 --- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java +++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java @@ -19,27 +19,23 @@ import static org.apache.solr.common.params.CommonParams.JAVABIN; import static org.apache.solr.common.params.CommonParams.NAME; import static org.apache.solr.handler.ReplicationHandler.ALIAS; -import static org.apache.solr.handler.ReplicationHandler.CHECKSUM; import static org.apache.solr.handler.ReplicationHandler.CMD_DETAILS; import static org.apache.solr.handler.ReplicationHandler.CMD_GET_FILE; import static org.apache.solr.handler.ReplicationHandler.CMD_GET_FILE_LIST; import static org.apache.solr.handler.ReplicationHandler.CMD_INDEX_VERSION; import static org.apache.solr.handler.ReplicationHandler.COMMAND; -import static org.apache.solr.handler.ReplicationHandler.COMPRESSION; import static org.apache.solr.handler.ReplicationHandler.CONF_FILES; -import static org.apache.solr.handler.ReplicationHandler.CONF_FILE_SHORT; -import static org.apache.solr.handler.ReplicationHandler.EXTERNAL; import static org.apache.solr.handler.ReplicationHandler.FETCH_FROM_LEADER; -import static org.apache.solr.handler.ReplicationHandler.FILE; -import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM; -import static org.apache.solr.handler.ReplicationHandler.GENERATION; -import static org.apache.solr.handler.ReplicationHandler.INTERNAL; import static org.apache.solr.handler.ReplicationHandler.LEADER_URL; -import static org.apache.solr.handler.ReplicationHandler.LEGACY_LEADER_URL; -import static org.apache.solr.handler.ReplicationHandler.LEGACY_SKIP_COMMIT_ON_LEADER_VERSION_ZERO; -import static org.apache.solr.handler.ReplicationHandler.OFFSET; import static org.apache.solr.handler.ReplicationHandler.SIZE; import static org.apache.solr.handler.ReplicationHandler.SKIP_COMMIT_ON_LEADER_VERSION_ZERO; +import static org.apache.solr.handler.admin.api.ReplicationAPIBase.CHECKSUM; +import static org.apache.solr.handler.admin.api.ReplicationAPIBase.COMPRESSION; +import static org.apache.solr.handler.admin.api.ReplicationAPIBase.CONF_FILE_SHORT; +import static org.apache.solr.handler.admin.api.ReplicationAPIBase.FILE; +import static org.apache.solr.handler.admin.api.ReplicationAPIBase.FILE_STREAM; +import static org.apache.solr.handler.admin.api.ReplicationAPIBase.GENERATION; +import static org.apache.solr.handler.admin.api.ReplicationAPIBase.OFFSET; import java.io.File; import java.io.FileNotFoundException; @@ -55,11 +51,9 @@ import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; import java.nio.file.FileStore; -import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; -import java.nio.file.StandardCopyOption; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; @@ -95,6 +89,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; +import org.apache.solr.client.api.model.FileMetaData; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.Http2SolrClient; import org.apache.solr.client.solrj.impl.HttpClientUtil; @@ -120,15 +115,13 @@ import org.apache.solr.core.DirectoryFactory.DirContext; import org.apache.solr.core.IndexDeletionPolicyWrapper; import org.apache.solr.core.SolrCore; -import org.apache.solr.handler.ReplicationHandler.FileInfo; -import org.apache.solr.handler.admin.api.CoreReplicationAPI; +import org.apache.solr.handler.admin.api.ReplicationAPIBase; import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.security.AllowListUrlChecker; import org.apache.solr.update.CommitUpdateCommand; import org.apache.solr.update.UpdateShardHandler; -import org.apache.solr.util.FileUtils; import org.apache.solr.util.IndexOutputOutputStream; import org.apache.solr.util.RTimer; import org.apache.solr.util.RefCounted; @@ -191,8 +184,6 @@ public class IndexFetcher { private Integer soTimeout; - private boolean downloadTlogFiles = false; - private boolean skipCommitOnLeaderVersionZero = true; private boolean clearLocalIndexFirst = false; @@ -213,7 +204,7 @@ public static class IndexFetchResult { new IndexFetchResult("Local index commit is already in sync with peer", true, null); public static final IndexFetchResult INDEX_FETCH_FAILURE = - new IndexFetchResult("Fetching lastest index is failed", false, null); + new IndexFetchResult("Fetching latest index is failed", false, null); public static final IndexFetchResult INDEX_FETCH_SUCCESS = new IndexFetchResult("Fetching latest index is successful", true, null); public static final IndexFetchResult LOCK_OBTAIN_FAILED = @@ -228,8 +219,6 @@ public static class IndexFetchResult { public static final IndexFetchResult PEER_INDEX_COMMIT_DELETED = new IndexFetchResult( "No files to download because IndexCommit in peer was deleted", false, null); - public static final IndexFetchResult LOCAL_ACTIVITY_DURING_REPLICATION = - new IndexFetchResult("Local index modification during replication", false, null); public static final IndexFetchResult EXPECTING_NON_LEADER = new IndexFetchResult("Replicating from leader but I'm the shard leader", false, null); public static final IndexFetchResult LEADER_IS_NOT_ACTIVE = @@ -281,17 +270,11 @@ public IndexFetcher( if (fetchFromLeader != null && fetchFromLeader instanceof Boolean) { this.fetchFromLeader = (boolean) fetchFromLeader; } - Object skipCommitOnLeaderVersionZero = - ReplicationHandler.getObjectWithBackwardCompatibility( - initArgs, - SKIP_COMMIT_ON_LEADER_VERSION_ZERO, - LEGACY_SKIP_COMMIT_ON_LEADER_VERSION_ZERO); + Object skipCommitOnLeaderVersionZero = initArgs.get(SKIP_COMMIT_ON_LEADER_VERSION_ZERO); if (skipCommitOnLeaderVersionZero != null && skipCommitOnLeaderVersionZero instanceof Boolean) { this.skipCommitOnLeaderVersionZero = (boolean) skipCommitOnLeaderVersionZero; } - String leaderUrl = - ReplicationHandler.getObjectWithBackwardCompatibility( - initArgs, LEADER_URL, LEGACY_LEADER_URL); + String leaderUrl = (String) initArgs.get(LEADER_URL); if (leaderUrl == null && !this.fetchFromLeader) throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "'leaderUrl' is required for a follower"); @@ -303,16 +286,11 @@ public IndexFetcher( this.replicationHandler = handler; String compress = (String) initArgs.get(COMPRESSION); - useInternalCompression = INTERNAL.equals(compress); - useExternalCompression = EXTERNAL.equals(compress); + useInternalCompression = ReplicationHandler.INTERNAL.equals(compress); + useExternalCompression = ReplicationHandler.EXTERNAL.equals(compress); connTimeout = getParameter(initArgs, HttpClientUtil.PROP_CONNECTION_TIMEOUT, 30000, null); + soTimeout = getParameter(initArgs, HttpClientUtil.PROP_SO_TIMEOUT, 120000, null); - // allow a leader override for tests - you specify this in /replication follower section of - // solrconfig and some test don't want to define this - soTimeout = Integer.getInteger("solr.indexfetcher.sotimeout", -1); - if (soTimeout == -1) { - soTimeout = getParameter(initArgs, HttpClientUtil.PROP_SO_TIMEOUT, 120000, null); - } String httpBasicAuthUser = (String) initArgs.get(HttpClientUtil.PROP_BASIC_AUTH_USER); String httpBasicAuthPassword = (String) initArgs.get(HttpClientUtil.PROP_BASIC_AUTH_PASS); solrClient = @@ -417,7 +395,7 @@ IndexFetchResult fetchLatestIndex(boolean forceReplication) } /** - * This command downloads all the necessary files from leader to install a index commit point. + * This command downloads all the necessary files from leader to install an index commit point. * Only changed files are downloaded. It also downloads the conf files (if they are modified). * * @param forceReplication force a replication in all cases @@ -437,7 +415,7 @@ IndexFetchResult fetchLatestIndex(boolean forceReplication, boolean forceCoreRel Directory indexDir = null; String indexDirPath; boolean deleteTmpIdxDir = true; - File tmpTlogDir = null; + Path tmpTlogDir = null; if (!solrCore.getSolrCoreState().getLastReplicateIndexSuccess()) { // if the last replication was not a success, we force a full replication @@ -537,6 +515,12 @@ IndexFetchResult fetchLatestIndex(boolean forceReplication, boolean forceCoreRel IndexDeletionPolicyWrapper.getCommitTimestamp(commit)); // nowarn } + // Leader's version is 0 and generation is 0 - not open for replication + if (latestVersion == 0L && latestGeneration == 0L) { + log.info("Leader's version is 0 and generation is 0 - not open for replication"); + return IndexFetchResult.LEADER_IS_NOT_ACTIVE; + } + if (latestVersion == 0L) { if (IndexDeletionPolicyWrapper.getCommitTimestamp(commit) != 0L) { // since we won't get the files for an empty index, @@ -685,7 +669,7 @@ IndexFetchResult fetchLatestIndex(boolean forceReplication, boolean forceCoreRel latestGeneration); final long timeTakenSeconds = getReplicationTimeElapsed(); final Long bytesDownloadedPerSecond = - (timeTakenSeconds != 0 ? Long.valueOf(bytesDownloaded / timeTakenSeconds) : null); + (timeTakenSeconds != 0 ? bytesDownloaded / timeTakenSeconds : null); log.info( "Total time taken for download (fullCopy={},bytesDownloaded={}) : {} secs ({} bytes/sec) to {}", isFullCopyNeeded, @@ -812,7 +796,7 @@ private void cleanup( Directory tmpIndexDir, Directory indexDir, boolean deleteTmpIdxDir, - File tmpTlogDir, + Path tmpTlogDir, boolean successfulInstall) throws IOException { try { @@ -862,7 +846,9 @@ private void cleanup( log.error("Error releasing indexDir", e); } try { - if (tmpTlogDir != null) delTree(tmpTlogDir); + if (tmpTlogDir != null) { + delTree(tmpTlogDir); + } } catch (Exception e) { log.error("Error deleting tmpTlogDir", e); } @@ -1080,6 +1066,7 @@ private void downloadConfFiles( confFilesDownloaded = Collections.synchronizedList(new ArrayList<>()); Path tmpConfPath = solrCore.getResourceLoader().getConfigPath().resolve("conf." + getDateAsStr(new Date())); + // TODO SOLR-8282 move to PATH File tmpconfDir = tmpConfPath.toFile(); try { boolean status = tmpconfDir.mkdirs(); @@ -1091,7 +1078,8 @@ private void downloadConfFiles( for (Map file : confFilesToDownload) { String saveAs = (String) (file.get(ALIAS) == null ? file.get(NAME) : file.get(ALIAS)); localFileFetcher = - new LocalFsFileFetcher(tmpconfDir, file, saveAs, CONF_FILE_SHORT, latestGeneration); + new LocalFsFileFetcher( + tmpconfDir.toPath(), file, saveAs, CONF_FILE_SHORT, latestGeneration); currentFile = file; localFileFetcher.fetchFile(); confFilesDownloaded.add(new HashMap<>(file)); @@ -1101,7 +1089,7 @@ private void downloadConfFiles( terminateAndWaitFsyncService(); copyTmpConfFiles2Conf(tmpConfPath); } finally { - delTree(tmpconfDir); + delTree(tmpconfDir.toPath()); } } @@ -1168,6 +1156,7 @@ private long downloadIndexFiles( alwaysDownload); } if (!compareResult.equal || downloadCompleteIndex || alwaysDownload) { + // TODO SOLR-8282 move to PATH File localFile = new File(indexDirPath, filename); if (downloadCompleteIndex && doDifferentialCopy @@ -1215,15 +1204,15 @@ private long downloadIndexFiles( private static Long getUsableSpace(String dir) { try { - File file = new File(dir); - if (!file.exists()) { - file = file.getParentFile(); + Path file = Path.of(dir); + if (Files.notExists(file)) { + file = file.getParent(); // this is not a disk directory. so just pretend that there is enough space - if (!file.exists()) { + if (Files.notExists(file)) { return Long.MAX_VALUE; } } - FileStore fileStore = Files.getFileStore(file.toPath()); + FileStore fileStore = Files.getFileStore(file); return fileStore.getUsableSpace(); } catch (IOException e) { throw new SolrException(ErrorCode.SERVER_ERROR, "Could not free disk space", e); @@ -1383,7 +1372,7 @@ private static boolean slowFileExists(Directory dir, String fileName) throws IOE * All the files which are common between leader and follower must have same size and same * checksum else we assume they are not compatible (stale). * - * @return true if the index stale and we need to download a fresh copy, false otherwise. + * @return true if the index is stale, and we need to download a fresh copy, false otherwise. * @throws IOException if low level io error */ private boolean isIndexStale(Directory dir) throws IOException { @@ -1502,7 +1491,7 @@ private void copyTmpConfFiles2Conf(Path tmpconfDir) { ErrorCode.SERVER_ERROR, "Unable to mkdirs: " + oldPath.getParent(), e); } if (Files.exists(oldPath)) { - File oldFile = oldPath.toFile(); // TODO drop this + File oldFile = oldPath.toFile(); // TODO SOLR-8282 move to PATH File backupFile = new File(oldFile.getPath() + "." + getDateAsStr(new Date(oldFile.lastModified()))); if (!backupFile.getParentFile().exists()) { @@ -1528,54 +1517,11 @@ private void copyTmpConfFiles2Conf(Path tmpconfDir) { } } - /** - * The tlog files are moved from the tmp dir to the tlog dir as an atomic filesystem operation. A - * backup of the old directory is maintained. If the directory move fails, it will try to revert - * back the original tlog directory. - */ - private boolean copyTmpTlogFiles2Tlog(File tmpTlogDir) { - Path tlogDir = - FileSystems.getDefault().getPath(solrCore.getUpdateHandler().getUpdateLog().getTlogDir()); - Path backupTlogDir = - FileSystems.getDefault() - .getPath(tlogDir.getParent().toAbsolutePath().toString(), tmpTlogDir.getName()); - - try { - Files.move(tlogDir, backupTlogDir, StandardCopyOption.ATOMIC_MOVE); - } catch (IOException e) { - log.error("Unable to rename: {} to: {}", tlogDir, backupTlogDir, e); - return false; - } - - Path src = - FileSystems.getDefault() - .getPath(backupTlogDir.toAbsolutePath().toString(), tmpTlogDir.getName()); - try { - Files.move(src, tlogDir, StandardCopyOption.ATOMIC_MOVE); - } catch (IOException e) { - log.error("Unable to rename: {} to: {}", src, tlogDir, e); - - // In case of error, try to revert back the original tlog directory - try { - Files.move(backupTlogDir, tlogDir, StandardCopyOption.ATOMIC_MOVE); - } catch (IOException e2) { - // bad, we were not able to revert back the original tlog directory - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Unable to rename: " + backupTlogDir + " to: " + tlogDir); - } - - return false; - } - - return true; - } - private String getDateAsStr(Date d) { return new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).format(d); } - private final Map confFileInfoCache = new HashMap<>(); + private final Map confFileInfoCache = new HashMap<>(); /** * The local conf files are compared with the conf files in the leader. If they are same (by @@ -1599,10 +1545,10 @@ private Collection> getModifiedConfFiles( names.add(name, null); } // get the details of the local conf files with the same alias/name - List localFilesInfo = + List localFilesInfo = replicationHandler.getConfFileInfoFromCache(names, confFileInfoCache); // compare their size/checksum to see if - for (CoreReplicationAPI.FileMetaData fileInfo : localFilesInfo) { + for (FileMetaData fileInfo : localFilesInfo) { String name = fileInfo.name; Map m = nameVsFile.get(name); if (m == null) continue; // the file is not even present locally (so must be downloaded) @@ -1613,26 +1559,9 @@ private Collection> getModifiedConfFiles( return nameVsFile.isEmpty() ? Collections.emptyList() : nameVsFile.values(); } - /** - * This simulates File.delete exception-wise, since this class has some strange behavior with it. - * The only difference is it returns null on success, throws SecurityException on - * SecurityException, otherwise returns Throwable preventing deletion (instead of false), for - * additional information. - */ - static Throwable delete(File file) { + static boolean delTree(Path dir) { try { - Files.delete(file.toPath()); - return null; - } catch (SecurityException e) { - throw e; - } catch (Throwable other) { - return other; - } - } - - static boolean delTree(File dir) { - try { - org.apache.lucene.util.IOUtils.rm(dir.toPath()); + org.apache.lucene.util.IOUtils.rm(dir); return true; } catch (IOException e) { log.warn("Unable to delete directory : {}", dir, e); @@ -1723,7 +1652,7 @@ private interface FileInterface { * The class acts as a client for ReplicationHandler.FileStream. It understands the protocol of * wt=filestream * - * @see org.apache.solr.handler.ReplicationHandler.DirectoryFileStream + *

see org.apache.solr.handler.admin.api.ReplicationAPIBase.DirectoryFileStream */ private class FileFetcher { private final FileInterface file; @@ -1745,12 +1674,11 @@ private class FileFetcher { Map fileDetails, String saveAs, String solrParamOutput, - long latestGen) - throws IOException { + long latestGen) { this.file = file; this.fileName = (String) fileDetails.get(NAME); this.size = (Long) fileDetails.get(SIZE); - buf = new byte[(int) Math.min(this.size, ReplicationHandler.PACKET_SZ)]; + buf = new byte[(int) Math.min(this.size, ReplicationAPIBase.PACKET_SZ)]; this.solrParamOutput = solrParamOutput; this.saveAs = saveAs; indexGen = latestGen; @@ -1796,7 +1724,7 @@ private void fetch() throws Exception { } } finally { cleanup(); - // if cleanup succeeds . The file is downloaded fully. do an fsync + // if cleanup succeeds, and the file is downloaded fully, then do a fsync. fsyncService.execute( () -> { try { @@ -1900,8 +1828,8 @@ private int fetchPackets(FastInputStream fis) throws Exception { } /** - * The webcontainer flushes the data only after it fills the buffer size. So, all data has to be - * read as readFully() other wise it fails. So read everything as bytes and then extract an + * The web container flushes the data only after it fills the buffer size. So, all data has to + * be read as readFully() otherwise it fails. So read everything as bytes and then extract an * integer out of it */ private int readInt(byte[] b) { @@ -1968,7 +1896,7 @@ private FastInputStream getStream() throws IOException { } // wt=filestream this is a custom protocol params.set(CommonParams.WT, FILE_STREAM); - // This happen if there is a failure there is a retry. the offset= ensures + // This happens if there is a failure there is a retry. the offset= ensures // that the server starts from the offset if (bytesDownloaded > 0) { params.set(OFFSET, Long.toString(bytesDownloaded)); @@ -2047,7 +1975,7 @@ public void delete() throws Exception { } } - private class DirectoryFileFetcher extends FileFetcher { + protected class DirectoryFileFetcher extends FileFetcher { DirectoryFileFetcher( Directory tmpIndexDir, Map fileDetails, @@ -2061,33 +1989,32 @@ private class DirectoryFileFetcher extends FileFetcher { } private static class LocalFsFile implements FileInterface { - private File copy2Dir; FileChannel fileChannel; private FileOutputStream fileOutputStream; - File file; - - LocalFsFile(File dir, String saveAs) throws IOException { - this.copy2Dir = dir; + Path file; - this.file = new File(copy2Dir, saveAs); + LocalFsFile(Path dir, String saveAs) throws IOException { + this.file = dir.resolve(saveAs); - File parentDir = this.file.getParentFile(); - if (!parentDir.exists()) { - if (!parentDir.mkdirs()) { + Path parentDir = this.file.getParent(); + if (Files.notExists(parentDir)) { + try { + Files.createDirectories(parentDir); + } catch (Exception e) { throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Failed to create (sub)directory for file: " + saveAs); } } - this.fileOutputStream = new FileOutputStream(file); + this.fileOutputStream = new FileOutputStream(file.toFile()); this.fileChannel = this.fileOutputStream.getChannel(); } @Override public void sync() throws IOException { - FileUtils.sync(file); + org.apache.lucene.util.IOUtils.fsync(file, false); } @Override @@ -2103,13 +2030,13 @@ public void close() throws Exception { @Override public void delete() throws Exception { - Files.delete(file.toPath()); + Files.delete(file); } } - private class LocalFsFileFetcher extends FileFetcher { + protected class LocalFsFileFetcher extends FileFetcher { LocalFsFileFetcher( - File dir, + Path dir, Map fileDetails, String saveAs, String solrParamOutput, diff --git a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java index e2a9572c7ab..e3a5d451cf9 100644 --- a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java @@ -382,8 +382,7 @@ private BooleanQuery getBoostedQuery(Query mltquery) { for (BooleanClause clause : boostedQuery) { Query q = clause.getQuery(); float originalBoost = 1f; - if (q instanceof BoostQuery) { - BoostQuery bq = (BoostQuery) q; + if (q instanceof BoostQuery bq) { q = bq.getQuery(); originalBoost = bq.getBoost(); } @@ -481,8 +480,7 @@ public List getInterestingTerms(BooleanQuery boostedMLTQuery, i } Query q = o.getQuery(); float boost = 1f; - if (q instanceof BoostQuery) { - BoostQuery bq = (BoostQuery) q; + if (q instanceof BoostQuery bq) { q = bq.getQuery(); boost = bq.getBoost(); } diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java index 04e75065d3a..8b126b5cc06 100644 --- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java @@ -17,23 +17,31 @@ package org.apache.solr.handler; import static org.apache.solr.common.params.CommonParams.NAME; +import static org.apache.solr.handler.admin.api.ReplicationAPIBase.CHECKSUM; +import static org.apache.solr.handler.admin.api.ReplicationAPIBase.COMPRESSION; +import static org.apache.solr.handler.admin.api.ReplicationAPIBase.CONF_FILE_SHORT; +import static org.apache.solr.handler.admin.api.ReplicationAPIBase.FILE; +import static org.apache.solr.handler.admin.api.ReplicationAPIBase.GENERATION; +import static org.apache.solr.handler.admin.api.ReplicationAPIBase.INTERVAL_ERR_MSG; +import static org.apache.solr.handler.admin.api.ReplicationAPIBase.INTERVAL_PATTERN; +import static org.apache.solr.handler.admin.api.ReplicationAPIBase.LEN; +import static org.apache.solr.handler.admin.api.ReplicationAPIBase.MAX_WRITE_PER_SECOND; +import static org.apache.solr.handler.admin.api.ReplicationAPIBase.OFFSET; +import static org.apache.solr.handler.admin.api.ReplicationAPIBase.STATUS; +import static org.apache.solr.handler.admin.api.ReplicationAPIBase.TLOG_FILE; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.io.OutputStream; import java.lang.invoke.MethodHandles; import java.net.URI; -import java.nio.ByteBuffer; -import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -52,11 +60,8 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.zip.Adler32; import java.util.zip.Checksum; -import java.util.zip.DeflaterOutputStream; -import org.apache.commons.io.output.CloseShieldOutputStream; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexDeletionPolicy; @@ -64,17 +69,16 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.RateLimiter; import org.apache.solr.api.JerseyResource; +import org.apache.solr.client.api.model.FileMetaData; +import org.apache.solr.client.api.model.IndexVersionResponse; import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.ExecutorUtil; -import org.apache.solr.common.util.FastOutputStream; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.SolrNamedThreadFactory; @@ -91,11 +95,11 @@ import org.apache.solr.core.backup.repository.LocalFileSystemRepository; import org.apache.solr.handler.IndexFetcher.IndexFetchResult; import org.apache.solr.handler.ReplicationHandler.ReplicationHandlerConfig; -import org.apache.solr.handler.admin.api.CoreReplicationAPI; +import org.apache.solr.handler.admin.api.CoreReplication; +import org.apache.solr.handler.admin.api.ReplicationAPIBase; import org.apache.solr.handler.admin.api.SnapshotBackupAPI; import org.apache.solr.handler.api.V2ApiUtils; import org.apache.solr.jersey.APIConfigProvider; -import org.apache.solr.jersey.APIConfigProvider.APIConfig; import org.apache.solr.metrics.MetricsMap; import org.apache.solr.metrics.SolrMetricsContext; import org.apache.solr.request.SolrQueryRequest; @@ -148,8 +152,6 @@ public class ReplicationHandler extends RequestHandlerBase private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); SolrCore core; - private volatile boolean closed = false; - @Override public Name getPermissionName(AuthorizationContext request) { return Name.READ_PERM; @@ -222,8 +224,6 @@ public String toString() { private volatile long executorStartTime; - private int numTimesReplicated = 0; - private final Map confFileInfoCache = new HashMap<>(); private Long reserveCommitDuration = readIntervalMs("00:00:10"); @@ -272,9 +272,9 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw final SolrJerseyResponse indexVersionResponse = getIndexVersionResponse(); V2ApiUtils.squashIntoSolrResponseWithoutHeader(rsp, indexVersionResponse); } else if (command.equals(CMD_GET_FILE)) { - getFileStream(solrParams, rsp); + getFileStream(solrParams, rsp, req); } else if (command.equals(CMD_GET_FILE_LIST)) { - final CoreReplicationAPI coreReplicationAPI = new CoreReplicationAPI(core, req, rsp); + final CoreReplication coreReplicationAPI = new CoreReplication(core, req, rsp); V2ApiUtils.squashIntoSolrResponseWithoutHeader( rsp, coreReplicationAPI.fetchFileList(Long.parseLong(solrParams.required().get(GENERATION)))); @@ -301,8 +301,7 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw } else if (command.equals(CMD_SHOW_COMMITS)) { populateCommitInfo(rsp); } else if (command.equals(CMD_DETAILS)) { - getReplicationDetails( - rsp, getBoolWithBackwardCompatibility(solrParams, "follower", "slave", true)); + getReplicationDetails(rsp, solrParams.getBool("follower", true)); } else if (CMD_ENABLE_REPL.equalsIgnoreCase(command)) { replicationEnabled.set(true); rsp.add(STATUS, OK_STATUS); @@ -312,37 +311,44 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw } } - static boolean getBoolWithBackwardCompatibility( - SolrParams params, String preferredKey, String alternativeKey, boolean defaultValue) { - Boolean value = params.getBool(preferredKey); - if (value != null) { - return value; + /** + * This method adds an Object of FileStream to the response . The FileStream implements a custom + * protocol which is understood by IndexFetcher.FileFetcher + * + * @see IndexFetcher.LocalFsFileFetcher + * @see IndexFetcher.DirectoryFileFetcher + */ + private void getFileStream(SolrParams solrParams, SolrQueryResponse rsp, SolrQueryRequest req) { + final CoreReplication coreReplicationAPI = new CoreReplication(core, req, rsp); + String fileName; + String dirType; + + if (solrParams.get(CONF_FILE_SHORT) != null) { + fileName = solrParams.get(CONF_FILE_SHORT); + dirType = CONF_FILE_SHORT; + } else if (solrParams.get(TLOG_FILE) != null) { + fileName = solrParams.get(TLOG_FILE); + dirType = TLOG_FILE; + } else if (solrParams.get(FILE) != null) { + fileName = solrParams.get(FILE); + dirType = FILE; + } else { + reportErrorOnResponse( + rsp, + "Missing file parameter", + new SolrException(SolrException.ErrorCode.BAD_REQUEST, "File not specified in request")); + return; } - return params.getBool(alternativeKey, defaultValue); - } - @SuppressWarnings("unchecked") - static T getObjectWithBackwardCompatibility( - SolrParams params, String preferredKey, String alternativeKey, T defaultValue) { - Object value = params.get(preferredKey); - if (value != null) { - return (T) value; - } - value = params.get(alternativeKey); - if (value != null) { - return (T) value; - } - return defaultValue; - } - - @SuppressWarnings("unchecked") - public static T getObjectWithBackwardCompatibility( - NamedList params, String preferredKey, String alternativeKey) { - Object value = params.get(preferredKey); - if (value != null) { - return (T) value; - } - return (T) params.get(alternativeKey); + coreReplicationAPI.fetchFile( + fileName, + dirType, + solrParams.get(OFFSET), + solrParams.get(LEN), + Boolean.parseBoolean(solrParams.get(COMPRESSION)), + solrParams.getBool(CHECKSUM, false), + solrParams.getDouble(MAX_WRITE_PER_SECOND, Double.MAX_VALUE), + solrParams.getLong(GENERATION)); } private void reportErrorOnResponse(SolrQueryResponse response, String message, Exception e) { @@ -377,8 +383,7 @@ private void deleteSnapshot(ModifiableSolrParams params, SolrQueryResponse rsp) private void fetchIndex(SolrParams solrParams, SolrQueryResponse rsp) throws InterruptedException { - String leaderUrl = - getObjectWithBackwardCompatibility(solrParams, LEADER_URL, LEGACY_LEADER_URL, null); + String leaderUrl = solrParams.get(LEADER_URL, null); if (!isFollower && leaderUrl == null) { reportErrorOnResponse(rsp, "No follower configured or no 'leaderUrl' specified", null); return; @@ -445,11 +450,7 @@ static Long getCheckSum(Checksum checksum, Path f) { private volatile IndexFetcher currentIndexFetcher; public IndexFetchResult doFetch(SolrParams solrParams, boolean forceReplication) { - String leaderUrl = - solrParams == null - ? null - : ReplicationHandler.getObjectWithBackwardCompatibility( - solrParams, LEADER_URL, LEGACY_LEADER_URL, null); + String leaderUrl = solrParams == null ? null : solrParams.get(LEADER_URL, null); if (!indexFetchLock.tryLock()) return IndexFetchResult.LOCK_OBTAIN_FAILED; if (core.getCoreContainer().isShutDown()) { log.warn("I was asked to replicate but CoreContainer is shutting down"); @@ -673,33 +674,10 @@ public static void doSnapShoot( snapShooter.createSnapAsync(numberToKeep, result); } - /** - * This method adds an Object of FileStream to the response . The FileStream implements a custom - * protocol which is understood by IndexFetcher.FileFetcher - * - * @see IndexFetcher.LocalFsFileFetcher - * @see IndexFetcher.DirectoryFileFetcher - */ - private void getFileStream(SolrParams solrParams, SolrQueryResponse rsp) { - ModifiableSolrParams rawParams = new ModifiableSolrParams(solrParams); - rawParams.set(CommonParams.WT, FILE_STREAM); - - String cfileName = solrParams.get(CONF_FILE_SHORT); - String tlogFileName = solrParams.get(TLOG_FILE); - if (cfileName != null) { - rsp.add(FILE_STREAM, new LocalFsConfFileStream(solrParams)); - } else if (tlogFileName != null) { - rsp.add(FILE_STREAM, new LocalFsTlogFileStream(solrParams)); - } else { - rsp.add(FILE_STREAM, new DirectoryFileStream(solrParams)); - } - rsp.add(STATUS, OK_STATUS); - } - - public CoreReplicationAPI.IndexVersionResponse getIndexVersionResponse() throws IOException { + public IndexVersionResponse getIndexVersionResponse() throws IOException { IndexCommit commitPoint = indexCommitPoint; // make a copy so it won't change - CoreReplicationAPI.IndexVersionResponse rsp = new CoreReplicationAPI.IndexVersionResponse(); + IndexVersionResponse rsp = new IndexVersionResponse(); if (commitPoint == null) { // if this handler is 'lazy', we may not have tracked the last commit // because our commit listener is registered on inform @@ -735,9 +713,9 @@ public CoreReplicationAPI.IndexVersionResponse getIndexVersionResponse() throws *

The local conf files information is cached so that everytime it does not have to compute the * checksum. The cache is refreshed only if the lastModified of the file changes */ - public List getConfFileInfoFromCache( + public List getConfFileInfoFromCache( NamedList nameAndAlias, final Map confFileInfoCache) { - List confFiles = new ArrayList<>(); + List confFiles = new ArrayList<>(); synchronized (confFileInfoCache) { Checksum checksum = null; for (int i = 0; i < nameAndAlias.size(); i++) { @@ -758,7 +736,7 @@ public List getConfFileInfoFromCache( info = new FileInfo(lastModified, cf, size, getCheckSum(checksum, f)); confFileInfoCache.put(cf, info); } - CoreReplicationAPI.FileMetaData m = info.fileMetaData; + FileMetaData m = info.fileMetaData; if (nameAndAlias.getVal(i) != null) m.alias = nameAndAlias.getVal(i); confFiles.add(m); } @@ -768,11 +746,11 @@ public List getConfFileInfoFromCache( static class FileInfo { long lastmodified; - CoreReplicationAPI.FileMetaData fileMetaData; + FileMetaData fileMetaData; public FileInfo(long lasmodified, String name, long size, long checksum) { this.lastmodified = lasmodified; - this.fileMetaData = new CoreReplicationAPI.FileMetaData(size, name, checksum); + this.fileMetaData = new FileMetaData(size, name, checksum); } } @@ -816,14 +794,6 @@ private Date getNextScheduledExecTime() { return nextTime; } - int getTimesReplicatedSinceStartup() { - return numTimesReplicated; - } - - void setTimesReplicatedSinceStartup() { - numTimesReplicated++; - } - @Override public Category getCategory() { return Category.REPLICATION; @@ -907,7 +877,7 @@ public void initializeMetrics(SolrMetricsContext parentContext, String scope) { if (fetcher != null) { map.put(LEADER_URL, fetcher.getLeaderCoreUrl()); if (getPollInterval() != null) { - map.put(POLL_INTERVAL, getPollInterval()); + map.put(ReplicationAPIBase.POLL_INTERVAL, getPollInterval()); } map.put("isPollingDisabled", isPollingDisabled()); map.put("isReplicating", isReplicating()); @@ -989,7 +959,7 @@ private NamedList getReplicationDetails( } follower.add(LEADER_URL, fetcher.getLeaderCoreUrl()); if (getPollInterval() != null) { - follower.add(POLL_INTERVAL, getPollInterval()); + follower.add(ReplicationAPIBase.POLL_INTERVAL, getPollInterval()); } Date nextScheduled = getNextScheduledExecTime(); if (nextScheduled != null && !isPollingDisabled()) { @@ -1059,7 +1029,7 @@ private NamedList getReplicationDetails( follower.add("replicationStartTime", replicationStartTimeStamp.toString()); } long elapsed = fetcher.getReplicationTimeElapsed(); - follower.add("timeElapsed", String.valueOf(elapsed) + "s"); + follower.add("timeElapsed", elapsed + "s"); if (bytesDownloaded > 0) estimatedTimeRemaining = @@ -1124,13 +1094,13 @@ private Object formatVal(String key, Properties props, Class clzz) { if (s == null || s.trim().length() == 0) return null; if (clzz == Date.class) { try { - Long l = Long.parseLong(s); + long l = Long.parseLong(s); return new Date(l).toString(); } catch (NumberFormatException e) { return null; } } else if (clzz == List.class) { - String ss[] = s.split(","); + String[] ss = s.split(","); List l = new ArrayList<>(); for (String s1 : ss) { l.add(new Date(Long.parseLong(s1)).toString()); @@ -1258,14 +1228,14 @@ public void inform(SolrCore core) { } else { replicationHandlerConfig.numberBackupsToKeep = 0; } - NamedList follower = getObjectWithBackwardCompatibility(initArgs, "follower", "slave"); + NamedList follower = (NamedList) initArgs.get("follower"); boolean enableFollower = isEnabled(follower); if (enableFollower) { currentIndexFetcher = pollingIndexFetcher = new IndexFetcher(follower, this, core); - setupPolling((String) follower.get(POLL_INTERVAL)); + setupPolling((String) follower.get(ReplicationAPIBase.POLL_INTERVAL)); isFollower = true; } - NamedList leader = getObjectWithBackwardCompatibility(initArgs, "leader", "master"); + NamedList leader = (NamedList) initArgs.get("leader"); boolean enableLeader = isEnabled(leader); if (enableLeader || (enableFollower && !currentIndexFetcher.fetchFromLeader)) { @@ -1288,11 +1258,11 @@ public void inform(SolrCore core) { if (enableLeader) { includeConfFiles = (String) leader.get(CONF_FILES); if (includeConfFiles != null && includeConfFiles.trim().length() > 0) { - List files = Arrays.asList(includeConfFiles.split(",")); + String[] files = includeConfFiles.split(","); for (String file : files) { if (file.trim().length() == 0) continue; String[] strs = file.trim().split(":"); - // if there is an alias add it or it is null + // if there is an alias add it, or it is null confFileNameAlias.add(strs[0], strs.length > 1 ? strs[1] : null); } log.info("Replication enabled for following config files: {}", includeConfFiles); @@ -1313,8 +1283,7 @@ public void inform(SolrCore core) { if (replicateOnOptimize) { IndexDeletionPolicyWrapper wrapper = core.getDeletionPolicy(); IndexDeletionPolicy policy = wrapper == null ? null : wrapper.getWrappedDeletionPolicy(); - if (policy instanceof SolrDeletionPolicy) { - SolrDeletionPolicy solrPolicy = (SolrDeletionPolicy) policy; + if (policy instanceof SolrDeletionPolicy solrPolicy) { if (solrPolicy.getMaxOptimizedCommitsToKeep() < 1) { solrPolicy.setMaxOptimizedCommitsToKeep(1); } @@ -1364,7 +1333,7 @@ public void inform(SolrCore core) { } } - // ensure the writer is init'd so that we have a list of commit points + // ensure the writer is initialized so that we have a list of commit points RefCounted iw = core.getUpdateHandler().getSolrCoreState().getIndexWriter(core); iw.decref(); @@ -1389,7 +1358,7 @@ public void inform(SolrCore core) { @Override public Collection> getJerseyResources() { - return List.of(CoreReplicationAPI.class, SnapshotBackupAPI.class); + return List.of(CoreReplication.class, SnapshotBackupAPI.class); } @Override @@ -1512,257 +1481,11 @@ public void postSoftCommit() {} }; } - /** This class is used to read and send files in the lucene index */ - private class DirectoryFileStream implements SolrCore.RawWriter { - protected SolrParams params; - - protected FastOutputStream fos; - - protected Long indexGen; - protected IndexDeletionPolicyWrapper delPolicy; - - protected String fileName; - protected String cfileName; - protected String tlogFileName; - protected String sOffset; - protected String sLen; - protected final boolean compress; - protected boolean useChecksum; - - protected long offset = -1; - protected int len = -1; - - protected Checksum checksum; - - private RateLimiter rateLimiter; - - byte[] buf; - - public DirectoryFileStream(SolrParams solrParams) { - params = solrParams; - delPolicy = core.getDeletionPolicy(); - - fileName = validateFilenameOrError(params.get(FILE)); - cfileName = validateFilenameOrError(params.get(CONF_FILE_SHORT)); - tlogFileName = validateFilenameOrError(params.get(TLOG_FILE)); - - sOffset = params.get(OFFSET); - sLen = params.get(LEN); - compress = Boolean.parseBoolean(params.get(COMPRESSION)); - useChecksum = params.getBool(CHECKSUM, false); - indexGen = params.getLong(GENERATION); - if (useChecksum) { - checksum = new Adler32(); - } - // No throttle if MAX_WRITE_PER_SECOND is not specified - double maxWriteMBPerSec = params.getDouble(MAX_WRITE_PER_SECOND, Double.MAX_VALUE); - rateLimiter = new RateLimiter.SimpleRateLimiter(maxWriteMBPerSec); - } - - // Throw exception on directory traversal attempts - protected String validateFilenameOrError(String fileName) { - if (fileName != null) { - Path filePath = Paths.get(fileName); - filePath.forEach( - subpath -> { - if ("..".equals(subpath.toString())) { - throw new SolrException(ErrorCode.FORBIDDEN, "File name cannot contain .."); - } - }); - if (filePath.isAbsolute()) { - throw new SolrException(ErrorCode.FORBIDDEN, "File name must be relative"); - } - return fileName; - } else return null; - } - - protected void initWrite() throws IOException { - if (sOffset != null) offset = Long.parseLong(sOffset); - if (sLen != null) len = Integer.parseInt(sLen); - if (fileName == null && cfileName == null && tlogFileName == null) { - // no filename do nothing - writeNothingAndFlush(); - } - buf = new byte[(len == -1 || len > PACKET_SZ) ? PACKET_SZ : len]; - - // reserve commit point till write is complete - if (indexGen != null) { - delPolicy.saveCommitPoint(indexGen); - } - } - - protected void createOutputStream(OutputStream out) { - // DeflaterOutputStream requires a close call, but don't close the request outputstream - out = new CloseShieldOutputStream(out); - if (compress) { - fos = new FastOutputStream(new DeflaterOutputStream(out)); - } else { - fos = new FastOutputStream(out); - } - } - - protected void extendReserveAndReleaseCommitPoint() { - if (indexGen != null) { - // Reserve the commit point for another 10s for the next file to be to fetched. - // We need to keep extending the commit reservation between requests so that the replica can - // fetch all the files correctly. - delPolicy.setReserveDuration(indexGen, reserveCommitDuration); - - // release the commit point as the write is complete - delPolicy.releaseCommitPoint(indexGen); - } - } - - @Override - public void write(OutputStream out) throws IOException { - createOutputStream(out); - - IndexInput in = null; - try { - initWrite(); - - Directory dir = core.withSearcher(searcher -> searcher.getIndexReader().directory()); - in = dir.openInput(fileName, IOContext.READONCE); - // if offset is mentioned move the pointer to that point - if (offset != -1) in.seek(offset); - - long filelen = dir.fileLength(fileName); - long maxBytesBeforePause = 0; - - while (true) { - offset = offset == -1 ? 0 : offset; - int read = (int) Math.min(buf.length, filelen - offset); - in.readBytes(buf, 0, read); - - fos.writeInt(read); - if (useChecksum) { - checksum.reset(); - checksum.update(buf, 0, read); - fos.writeLong(checksum.getValue()); - } - fos.write(buf, 0, read); - fos.flush(); - log.debug("Wrote {} bytes for file {}", offset + read, fileName); // nowarn - - // Pause if necessary - maxBytesBeforePause += read; - if (maxBytesBeforePause >= rateLimiter.getMinPauseCheckBytes()) { - rateLimiter.pause(maxBytesBeforePause); - maxBytesBeforePause = 0; - } - if (read != buf.length) { - writeNothingAndFlush(); - // we close because DeflaterOutputStream requires a close call, but the request - // outputstream is protected - fos.close(); - break; - } - offset += read; - in.seek(offset); - } - } catch (IOException e) { - log.warn("Exception while writing response for params: {}", params, e); - } finally { - if (in != null) { - in.close(); - } - extendReserveAndReleaseCommitPoint(); - } - } - - /** Used to write a marker for EOF */ - protected void writeNothingAndFlush() throws IOException { - fos.writeInt(0); - fos.flush(); - } - } - - /** This is used to write files in the conf directory. */ - private abstract class LocalFsFileStream extends DirectoryFileStream { - - private Path file; - - public LocalFsFileStream(SolrParams solrParams) { - super(solrParams); - this.file = this.initFile(); - } - - protected abstract Path initFile(); - - @Override - public void write(OutputStream out) throws IOException { - createOutputStream(out); - try { - initWrite(); - - if (Files.isReadable(file)) { - try (SeekableByteChannel channel = Files.newByteChannel(file)) { - // if offset is mentioned move the pointer to that point - if (offset != -1) channel.position(offset); - ByteBuffer bb = ByteBuffer.wrap(buf); - - while (true) { - bb.clear(); - long bytesRead = channel.read(bb); - if (bytesRead <= 0) { - writeNothingAndFlush(); - // we close because DeflaterOutputStream requires a close call, but the request - // outputstream is protected - fos.close(); - break; - } - fos.writeInt((int) bytesRead); - if (useChecksum) { - checksum.reset(); - checksum.update(buf, 0, (int) bytesRead); - fos.writeLong(checksum.getValue()); - } - fos.write(buf, 0, (int) bytesRead); - fos.flush(); - } - } - } else { - writeNothingAndFlush(); - } - } catch (IOException e) { - log.warn("Exception while writing response for params: {}", params, e); - } finally { - extendReserveAndReleaseCommitPoint(); - } - } - } - - private class LocalFsTlogFileStream extends LocalFsFileStream { - - public LocalFsTlogFileStream(SolrParams solrParams) { - super(solrParams); - } - - @Override - protected Path initFile() { - // if it is a tlog file read from tlog directory - return Path.of(core.getUpdateHandler().getUpdateLog().getTlogDir(), tlogFileName); - } - } - - private class LocalFsConfFileStream extends LocalFsFileStream { - - public LocalFsConfFileStream(SolrParams solrParams) { - super(solrParams); - } - - @Override - protected Path initFile() { - // if it is a conf file read from config directory - return core.getResourceLoader().getConfigPath().resolve(cfileName); - } - } - - private static Long readIntervalMs(String interval) { + private Long readIntervalMs(String interval) { return TimeUnit.MILLISECONDS.convert(readIntervalNs(interval), TimeUnit.NANOSECONDS); } - private static Long readIntervalNs(String interval) { + private Long readIntervalNs(String interval) { if (interval == null) return null; int result = 0; Matcher m = INTERVAL_PATTERN.matcher(interval.trim()); @@ -1792,30 +1515,15 @@ private static Long readIntervalNs(String interval) { public static final String LEADER_URL = "leaderUrl"; - /** - * @deprecated Only used for backwards compatibility. Use {@link #LEADER_URL} - */ - @Deprecated public static final String LEGACY_LEADER_URL = "masterUrl"; - public static final String FETCH_FROM_LEADER = "fetchFromLeader"; - // in case of TLOG replica, if leaderVersion = zero, don't do commit - // otherwise updates from current tlog won't copied over properly to the new tlog, leading to data - // loss - // don't commit on leader version zero for PULL replicas as PULL should only get its index - // state from leader + // In case of TLOG replica, if leaderVersion = zero, don't do commit + // otherwise updates from current tlog won't be copied over properly to the new tlog, + // leading to data loss. + // Don't commit on leader version zero for PULL replicas as PULL should only get its index + // state from leader. public static final String SKIP_COMMIT_ON_LEADER_VERSION_ZERO = "skipCommitOnLeaderVersionZero"; - /** - * @deprecated Only used for backwards compatibility. Use {@link - * #SKIP_COMMIT_ON_LEADER_VERSION_ZERO} - */ - @Deprecated - public static final String LEGACY_SKIP_COMMIT_ON_LEADER_VERSION_ZERO = - "skipCommitOnMasterVersionZero"; - - public static final String STATUS = "status"; - public static final String MESSAGE = "message"; public static final String COMMAND = "command"; @@ -1850,47 +1558,16 @@ private static Long readIntervalNs(String interval) { public static final String CMD_DELETE_BACKUP = "deletebackup"; - public static final String GENERATION = "generation"; - - public static final String OFFSET = "offset"; - - public static final String LEN = "len"; - - public static final String FILE = "file"; - public static final String SIZE = "size"; - public static final String MAX_WRITE_PER_SECOND = "maxWriteMBPerSec"; - - public static final String CONF_FILE_SHORT = "cf"; - - public static final String TLOG_FILE = "tlogFile"; - - public static final String CHECKSUM = "checksum"; - public static final String ALIAS = "alias"; - public static final String CONF_CHECKSUM = "confchecksum"; - public static final String CONF_FILES = "confFiles"; public static final String REPLICATE_AFTER = "replicateAfter"; - public static final String FILE_STREAM = "filestream"; - - public static final String POLL_INTERVAL = "pollInterval"; - - public static final String INTERVAL_ERR_MSG = - "The " + POLL_INTERVAL + " must be in this format 'HH:mm:ss'"; - - private static final Pattern INTERVAL_PATTERN = Pattern.compile("(\\d*?):(\\d*?):(\\d*)"); - - public static final int PACKET_SZ = 1024 * 1024; // 1MB - public static final String RESERVE = "commitReserveDuration"; - public static final String COMPRESSION = "compression"; - public static final String EXTERNAL = "external"; public static final String INTERNAL = "internal"; @@ -1908,14 +1585,14 @@ private static Long readIntervalNs(String interval) { /** * Boolean param for tests that can be specified when using {@link #CMD_FETCH_INDEX} to force the * current request to block until the fetch is complete. NOTE: This param is not advised - * for non-test code, since the duration of the fetch for non-trivial indexes will likeley cause + * for non-test code, since the duration of the fetch for non-trivial indexes will likely cause * the request to time out. * * @lucene.internal */ public static final String WAIT = "wait"; - public static class ReplicationHandlerConfig implements APIConfig { + public static class ReplicationHandlerConfig implements APIConfigProvider.APIConfig { private int numberBackupsToKeep = 0; // zero: do not delete old backups diff --git a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java index 9f438fc4605..43559d170cf 100644 --- a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java +++ b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java @@ -47,6 +47,7 @@ import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.search.CpuAllowedLimit; import org.apache.solr.search.QueryLimits; +import org.apache.solr.search.QueryLimitsExceededException; import org.apache.solr.search.SyntaxError; import org.apache.solr.security.PermissionNameProvider; import org.apache.solr.update.processor.DistributedUpdateProcessor; @@ -241,6 +242,8 @@ public void handleRequest(SolrQueryRequest req, SolrQueryResponse rsp) { metrics.numTimeouts.mark(); rsp.setHttpCaching(false); } + } catch (QueryLimitsExceededException e) { + rsp.setPartialResults(req); } catch (Exception e) { Exception normalized = normalizeReceivedException(req, e); processErrorMetricsOnException(normalized, metrics); @@ -282,8 +285,7 @@ public void handleRequest(SolrQueryRequest req, SolrQueryResponse rsp) { public static void processErrorMetricsOnException(Exception e, HandlerMetrics metrics) { boolean isClientError = false; - if (e instanceof SolrException) { - final SolrException se = (SolrException) e; + if (e instanceof SolrException se) { if (se.code() == SolrException.ErrorCode.CONFLICT.code) { return; } else if (se.code() >= 400 && se.code() < 500) { diff --git a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java index 6d8e7c319bf..c81fa8eb572 100644 --- a/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java @@ -42,7 +42,7 @@ import org.apache.solr.common.util.StrUtils; import org.apache.solr.core.SolrCore; import org.apache.solr.handler.admin.api.GetSchema; -import org.apache.solr.handler.admin.api.GetSchemaFieldAPI; +import org.apache.solr.handler.admin.api.GetSchemaField; import org.apache.solr.handler.admin.api.SchemaBulkModifyAPI; import org.apache.solr.handler.api.V2ApiUtils; import org.apache.solr.request.SolrQueryRequest; @@ -178,12 +178,12 @@ private void handleGET(SolrQueryRequest req, SolrQueryResponse rsp) { if (parts.size() > 2) { V2ApiUtils.squashIntoSolrResponseWithoutHeader( rsp, - new GetSchemaFieldAPI(req.getCore().getLatestSchema(), req.getParams()) + new GetSchemaField(req.getCore().getLatestSchema(), req.getParams()) .getFieldInfo(parts.get(2))); } else { V2ApiUtils.squashIntoSolrResponseWithoutHeader( rsp, - new GetSchemaFieldAPI(req.getCore().getLatestSchema(), req.getParams()) + new GetSchemaField(req.getCore().getLatestSchema(), req.getParams()) .listSchemaFields()); } return; @@ -192,7 +192,7 @@ private void handleGET(SolrQueryRequest req, SolrQueryResponse rsp) { { V2ApiUtils.squashIntoSolrResponseWithoutHeader( rsp, - new GetSchemaFieldAPI(req.getCore().getLatestSchema(), req.getParams()) + new GetSchemaField(req.getCore().getLatestSchema(), req.getParams()) .listCopyFields()); return; } @@ -201,12 +201,12 @@ private void handleGET(SolrQueryRequest req, SolrQueryResponse rsp) { if (parts.size() > 2) { V2ApiUtils.squashIntoSolrResponseWithoutHeader( rsp, - new GetSchemaFieldAPI(req.getCore().getLatestSchema(), req.getParams()) + new GetSchemaField(req.getCore().getLatestSchema(), req.getParams()) .getDynamicFieldInfo(parts.get(2))); } else { V2ApiUtils.squashIntoSolrResponseWithoutHeader( rsp, - new GetSchemaFieldAPI(req.getCore().getLatestSchema(), req.getParams()) + new GetSchemaField(req.getCore().getLatestSchema(), req.getParams()) .listDynamicFields()); } return; @@ -216,12 +216,12 @@ private void handleGET(SolrQueryRequest req, SolrQueryResponse rsp) { if (parts.size() > 2) { V2ApiUtils.squashIntoSolrResponseWithoutHeader( rsp, - new GetSchemaFieldAPI(req.getCore().getLatestSchema(), req.getParams()) + new GetSchemaField(req.getCore().getLatestSchema(), req.getParams()) .getFieldTypeInfo(parts.get(2))); } else { V2ApiUtils.squashIntoSolrResponseWithoutHeader( rsp, - new GetSchemaFieldAPI(req.getCore().getLatestSchema(), req.getParams()) + new GetSchemaField(req.getCore().getLatestSchema(), req.getParams()) .listSchemaFieldTypes()); } return; @@ -294,7 +294,7 @@ public Collection getApis() { @Override public Collection> getJerseyResources() { - return List.of(GetSchema.class, GetSchemaFieldAPI.class); + return List.of(GetSchema.class, GetSchemaField.class); } @Override diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java index 8a8a0da4425..4842534259d 100644 --- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java @@ -501,8 +501,7 @@ private void handleParams(ArrayList ops, RequestParams params) } SolrResourceLoader loader = req.getCore().getResourceLoader(); - if (loader instanceof ZkSolrResourceLoader) { - ZkSolrResourceLoader zkLoader = (ZkSolrResourceLoader) loader; + if (loader instanceof ZkSolrResourceLoader zkLoader) { if (ops.isEmpty()) { ZkController.touchConfDir(zkLoader); } else { diff --git a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java index 756a7956483..4f80731614d 100644 --- a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java @@ -240,8 +240,7 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw rsp.add("explanation", tupleStream.toExplanation(this.streamFactory)); } - if (tupleStream instanceof DaemonStream) { - DaemonStream daemonStream = (DaemonStream) tupleStream; + if (tupleStream instanceof DaemonStream daemonStream) { if (daemons.containsKey(daemonStream.getId())) { daemons.remove(daemonStream.getId()).close(); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java b/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java index 6c5998d17a2..18c8843f916 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java @@ -24,8 +24,9 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; -import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.Aliases; import org.apache.solr.common.cloud.ClusterState; @@ -43,6 +44,7 @@ import org.apache.zookeeper.KeeperException; public class ClusterStatus { + private final ZkStateReader zkStateReader; private final SolrParams solrParams; private final String collection; // maybe null @@ -178,79 +180,74 @@ private void fetchClusterStatusForCollOrAlias( String routeKey = solrParams.get(ShardParams._ROUTE_); String shard = solrParams.get(ZkStateReader.SHARD_ID_PROP); - Map collectionsMap = null; + Stream collectionStream; if (collection == null) { - collectionsMap = clusterState.getCollectionsMap(); + collectionStream = clusterState.collectionStream(); } else { - collectionsMap = - Collections.singletonMap(collection, clusterState.getCollectionOrNull(collection)); - } - - boolean isAlias = aliasVsCollections.containsKey(collection); - boolean didNotFindCollection = collectionsMap.get(collection) == null; - - if (didNotFindCollection && isAlias) { - // In this case this.collection is an alias name not a collection - // get all collections and filter out collections not in the alias - // clusterState.getCollectionsMap() should be replaced with an inexpensive call - collectionsMap = - clusterState.getCollectionsMap().entrySet().stream() - .filter((entry) -> aliasVsCollections.get(collection).contains(entry.getKey())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - NamedList collectionProps = new SimpleOrderedMap<>(); - - for (Map.Entry entry : collectionsMap.entrySet()) { - Map collectionStatus; - String name = entry.getKey(); - DocCollection clusterStateCollection = entry.getValue(); - if (clusterStateCollection == null) { - if (collection != null) { + DocCollection collState = clusterState.getCollectionOrNull(collection); + if (collState != null) { + collectionStream = Stream.of(collState); + } else { // couldn't find collection + // hopefully an alias... + if (!aliasVsCollections.containsKey(collection)) { // not an alias either SolrException solrException = new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "Collection: " + name + " not found"); + SolrException.ErrorCode.BAD_REQUEST, "Collection: " + collection + " not found"); solrException.setMetadata("CLUSTERSTATUS", "NOT_FOUND"); throw solrException; - } else { - // collection might have got deleted at the same time - continue; } + // In this case this.collection is an alias name not a collection + // Resolve them (not recursively but maybe should?). + collectionStream = + aliasVsCollections.get(collection).stream() + .map(clusterState::getCollectionOrNull) + .filter(Objects::nonNull); } + } - Set requestedShards = new HashSet<>(); - if (routeKey != null) { - DocRouter router = clusterStateCollection.getRouter(); - Collection slices = router.getSearchSlices(routeKey, null, clusterStateCollection); - for (Slice slice : slices) { - requestedShards.add(slice.getName()); - } - } - if (shard != null) { - String[] paramShards = shard.split(","); - requestedShards.addAll(Arrays.asList(paramShards)); - } + // TODO use an Iterable to stream the data to the client instead of gathering it all in mem - byte[] bytes = Utils.toJSON(clusterStateCollection); - @SuppressWarnings("unchecked") - Map docCollection = (Map) Utils.fromJSON(bytes); - collectionStatus = getCollectionStatus(docCollection, name, requestedShards); + NamedList collectionProps = new SimpleOrderedMap<>(); - collectionStatus.put("znodeVersion", clusterStateCollection.getZNodeVersion()); - collectionStatus.put( - "creationTimeMillis", clusterStateCollection.getCreationTime().toEpochMilli()); + collectionStream.forEach( + clusterStateCollection -> { + Map collectionStatus; + String name = clusterStateCollection.getName(); + + Set requestedShards = new HashSet<>(); + if (routeKey != null) { + DocRouter router = clusterStateCollection.getRouter(); + Collection slices = + router.getSearchSlices(routeKey, null, clusterStateCollection); + for (Slice slice : slices) { + requestedShards.add(slice.getName()); + } + } + if (shard != null) { + String[] paramShards = shard.split(","); + requestedShards.addAll(Arrays.asList(paramShards)); + } - if (collectionVsAliases.containsKey(name) && !collectionVsAliases.get(name).isEmpty()) { - collectionStatus.put("aliases", collectionVsAliases.get(name)); - } - String configName = clusterStateCollection.getConfigName(); - collectionStatus.put("configName", configName); - if (solrParams.getBool("prs", false) && clusterStateCollection.isPerReplicaState()) { - PerReplicaStates prs = clusterStateCollection.getPerReplicaStates(); - collectionStatus.put("PRS", prs); - } - collectionProps.add(name, collectionStatus); - } + byte[] bytes = Utils.toJSON(clusterStateCollection); + @SuppressWarnings("unchecked") + Map docCollection = (Map) Utils.fromJSON(bytes); + collectionStatus = getCollectionStatus(docCollection, name, requestedShards); + + collectionStatus.put("znodeVersion", clusterStateCollection.getZNodeVersion()); + collectionStatus.put( + "creationTimeMillis", clusterStateCollection.getCreationTime().toEpochMilli()); + + if (collectionVsAliases.containsKey(name) && !collectionVsAliases.get(name).isEmpty()) { + collectionStatus.put("aliases", collectionVsAliases.get(name)); + } + String configName = clusterStateCollection.getConfigName(); + collectionStatus.put("configName", configName); + if (solrParams.getBool("prs", false) && clusterStateCollection.isPerReplicaState()) { + PerReplicaStates prs = clusterStateCollection.getPerReplicaStates(); + collectionStatus.put("PRS", prs); + } + collectionProps.add(name, collectionStatus); + }); // now we need to walk the collectionProps tree to cross-check replica state with live nodes crossCheckReplicaStateWithLiveNodes(liveNodes, collectionProps); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index 7ee6da6a0c1..df6ba086d06 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -119,10 +119,13 @@ import org.apache.solr.api.Api; import org.apache.solr.api.JerseyResource; import org.apache.solr.client.api.model.AddReplicaPropertyRequestBody; +import org.apache.solr.client.api.model.CreateAliasRequestBody; import org.apache.solr.client.api.model.CreateCollectionSnapshotRequestBody; import org.apache.solr.client.api.model.CreateCollectionSnapshotResponse; import org.apache.solr.client.api.model.InstallShardDataRequestBody; +import org.apache.solr.client.api.model.ListCollectionSnapshotsResponse; import org.apache.solr.client.api.model.ReplaceNodeRequestBody; +import org.apache.solr.client.api.model.SetClusterPropertyRequestBody; import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.UpdateAliasPropertiesRequestBody; import org.apache.solr.client.api.model.UpdateCollectionPropertyRequestBody; @@ -140,7 +143,6 @@ import org.apache.solr.cloud.api.collections.ReindexCollectionCmd; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.cloud.ClusterProperties; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Replica.State; @@ -170,9 +172,10 @@ import org.apache.solr.handler.admin.api.AliasProperty; import org.apache.solr.handler.admin.api.BalanceReplicas; import org.apache.solr.handler.admin.api.BalanceShardUnique; +import org.apache.solr.handler.admin.api.ClusterProperty; import org.apache.solr.handler.admin.api.CollectionProperty; import org.apache.solr.handler.admin.api.CollectionStatusAPI; -import org.apache.solr.handler.admin.api.CreateAliasAPI; +import org.apache.solr.handler.admin.api.CreateAlias; import org.apache.solr.handler.admin.api.CreateCollection; import org.apache.solr.handler.admin.api.CreateCollectionBackup; import org.apache.solr.handler.admin.api.CreateCollectionSnapshot; @@ -190,7 +193,7 @@ import org.apache.solr.handler.admin.api.InstallShardData; import org.apache.solr.handler.admin.api.ListAliases; import org.apache.solr.handler.admin.api.ListCollectionBackups; -import org.apache.solr.handler.admin.api.ListCollectionSnapshotsAPI; +import org.apache.solr.handler.admin.api.ListCollectionSnapshots; import org.apache.solr.handler.admin.api.ListCollections; import org.apache.solr.handler.admin.api.MigrateDocsAPI; import org.apache.solr.handler.admin.api.MigrateReplicas; @@ -200,7 +203,7 @@ import org.apache.solr.handler.admin.api.ReloadCollectionAPI; import org.apache.solr.handler.admin.api.RenameCollection; import org.apache.solr.handler.admin.api.ReplaceNode; -import org.apache.solr.handler.admin.api.RestoreCollectionAPI; +import org.apache.solr.handler.admin.api.RestoreCollection; import org.apache.solr.handler.admin.api.SplitShardAPI; import org.apache.solr.handler.admin.api.SyncShard; import org.apache.solr.handler.api.V2ApiUtils; @@ -614,10 +617,9 @@ public enum CollectionOperation implements CollectionOp { CREATEALIAS_OP( CREATEALIAS, (req, rsp, h) -> { - final CreateAliasAPI.CreateAliasRequestBody reqBody = - CreateAliasAPI.createFromSolrParams(req.getParams()); + final CreateAliasRequestBody reqBody = CreateAlias.createFromSolrParams(req.getParams()); final SolrJerseyResponse response = - new CreateAliasAPI(h.coreContainer, req, rsp).createAlias(reqBody); + new CreateAlias(h.coreContainer, req, rsp).createAlias(reqBody); V2ApiUtils.squashIntoSolrResponseWithoutHeader(rsp, response); return null; }), @@ -770,11 +772,12 @@ public enum CollectionOperation implements CollectionOp { CLUSTERPROP_OP( CLUSTERPROP, (req, rsp, h) -> { + ClusterProperty clusterProperty = new ClusterProperty(req.getCoreContainer(), req, rsp); + SetClusterPropertyRequestBody setClusterPropertyRequestBody = + new SetClusterPropertyRequestBody(); String name = req.getParams().required().get(NAME); - String val = req.getParams().get(VALUE_LONG); - ClusterProperties cp = - new ClusterProperties(h.coreContainer.getZkController().getZkClient()); - cp.setClusterProperty(name, val); + setClusterPropertyRequestBody.value = req.getParams().get(VALUE_LONG); + clusterProperty.createOrUpdateClusterProperty(name, setClusterPropertyRequestBody); return null; }), COLLECTIONPROP_OP( @@ -1063,7 +1066,7 @@ public Map execute( RESTORE_OP( RESTORE, (req, rsp, h) -> { - final var response = RestoreCollectionAPI.invokeFromV1Params(req, rsp, h.coreContainer); + final var response = RestoreCollection.invokeFromV1Params(req, rsp, h.coreContainer); V2ApiUtils.squashIntoSolrResponseWithoutHeader(rsp, response); return null; }), @@ -1148,15 +1151,16 @@ public Map execute( (req, rsp, h) -> { req.getParams().required().check(COLLECTION_PROP); - final ListCollectionSnapshotsAPI listCollectionSnapshotsAPI = - new ListCollectionSnapshotsAPI(h.coreContainer, req, rsp); + final ListCollectionSnapshots listCollectionSnapshotsAPI = + new ListCollectionSnapshots(h.coreContainer, req, rsp); - final ListCollectionSnapshotsAPI.ListSnapshotsResponse response = + final ListCollectionSnapshotsResponse response = listCollectionSnapshotsAPI.listSnapshots(req.getParams().get(COLLECTION_PROP)); NamedList snapshots = new NamedList<>(); - for (CollectionSnapshotMetaData meta : response.snapshots.values()) { - snapshots.add(meta.getName(), meta.toNamedList()); + for (Object meta : response.snapshots.values()) { + final var metaTyped = (CollectionSnapshotMetaData) meta; + snapshots.add(metaTyped.getName(), metaTyped.toNamedList()); } rsp.add(SolrSnapshotManager.SNAPSHOTS_INFO, snapshots); @@ -1356,7 +1360,7 @@ public Collection> getJerseyResources() { CreateReplica.class, AddReplicaProperty.class, BalanceShardUnique.class, - CreateAliasAPI.class, + CreateAlias.class, CreateCollection.class, CreateCollectionBackup.class, CreateShard.class, @@ -1375,15 +1379,16 @@ public Collection> getJerseyResources() { ReplaceNode.class, MigrateReplicas.class, BalanceReplicas.class, - RestoreCollectionAPI.class, + RestoreCollection.class, SyncShard.class, CollectionProperty.class, DeleteNode.class, ListAliases.class, AliasProperty.class, - ListCollectionSnapshotsAPI.class, + ListCollectionSnapshots.class, CreateCollectionSnapshot.class, - DeleteCollectionSnapshot.class); + DeleteCollectionSnapshot.class, + ClusterProperty.class); } @Override diff --git a/solr/core/src/java/org/apache/solr/handler/admin/HealthCheckHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/HealthCheckHandler.java index a3724aa626c..b174e177f1a 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/HealthCheckHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/HealthCheckHandler.java @@ -20,7 +20,7 @@ import static org.apache.solr.common.params.CommonParams.FAILURE; import static org.apache.solr.common.params.CommonParams.OK; import static org.apache.solr.common.params.CommonParams.STATUS; -import static org.apache.solr.handler.ReplicationHandler.GENERATION; +import static org.apache.solr.handler.admin.api.ReplicationAPIBase.GENERATION; import java.lang.invoke.MethodHandles; import java.util.ArrayList; @@ -243,9 +243,8 @@ private boolean isWithinGenerationLag( IndexFetcher indexFetcher = null; try { // may not be the best way to get leader's replicableCommit - NamedList follower = - ReplicationHandler.getObjectWithBackwardCompatibility( - replicationHandler.getInitArgs(), "follower", "slave"); + NamedList follower = (NamedList) replicationHandler.getInitArgs().get("follower"); + indexFetcher = new IndexFetcher(follower, replicationHandler, core); NamedList replicableCommitOnLeader = indexFetcher.getLatestVersion(); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/IndexSizeEstimator.java b/solr/core/src/java/org/apache/solr/handler/admin/IndexSizeEstimator.java index 87d4764acba..f5e6bcf8c24 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/IndexSizeEstimator.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/IndexSizeEstimator.java @@ -276,13 +276,11 @@ public Estimate estimate() throws Exception { private void convert(Map result) { for (Map.Entry entry : result.entrySet()) { Object value = entry.getValue(); - if (value instanceof ItemPriorityQueue) { - ItemPriorityQueue queue = (ItemPriorityQueue) value; + if (value instanceof ItemPriorityQueue queue) { Map map = new LinkedHashMap<>(); queue.toMap(map); entry.setValue(map); - } else if (value instanceof MapWriterSummaryStatistics) { - MapWriterSummaryStatistics stats = (MapWriterSummaryStatistics) value; + } else if (value instanceof MapWriterSummaryStatistics stats) { Map map = new LinkedHashMap<>(); stats.toMap(map); entry.setValue(map); @@ -308,8 +306,7 @@ private void estimateSummary(Map details, Map su ((Map) perField) .forEach( (k, val) -> { - if (val instanceof SummaryStatistics) { - SummaryStatistics stats = (SummaryStatistics) val; + if (val instanceof SummaryStatistics stats) { if (k.startsWith("lengths")) { AtomicLong total = (AtomicLong) @@ -596,8 +593,7 @@ private void estimateStoredFields(Map result) throws IOException LeafReader leafReader = context.reader(); EstimatingVisitor visitor = new EstimatingVisitor(stats, topN, maxLength, samplingStep); Bits liveDocs = leafReader.getLiveDocs(); - if (leafReader instanceof CodecReader) { - CodecReader codecReader = (CodecReader) leafReader; + if (leafReader instanceof CodecReader codecReader) { StoredFieldsReader storedFieldsReader = codecReader.getFieldsReader(); // this instance may be faster for a full sequential pass StoredFieldsReader mergeInstance = storedFieldsReader.getMergeInstance(); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/LoggingHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/LoggingHandler.java index 67dbe69cd9c..7593bb7cbdd 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/LoggingHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/LoggingHandler.java @@ -22,12 +22,15 @@ import java.util.List; import org.apache.solr.api.Api; import org.apache.solr.api.JerseyResource; +import org.apache.solr.client.api.model.LogLevelChange; +import org.apache.solr.client.api.model.LoggingResponse; +import org.apache.solr.client.api.model.SetThresholdRequestBody; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.params.SolrParams; import org.apache.solr.core.CoreContainer; import org.apache.solr.handler.RequestHandlerBase; -import org.apache.solr.handler.admin.api.NodeLoggingAPI; +import org.apache.solr.handler.admin.api.NodeLogging; import org.apache.solr.handler.api.V2ApiUtils; import org.apache.solr.logging.LogWatcher; import org.apache.solr.request.SolrQueryRequest; @@ -54,25 +57,23 @@ public LoggingHandler(CoreContainer cc) { @Override public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { - final NodeLoggingAPI loggingApi = new NodeLoggingAPI(cc); + final NodeLogging loggingApi = new NodeLogging(cc); SolrParams params = req.getParams(); if (params.get("threshold") != null) { squashV2Response( rsp, - loggingApi.setMessageThreshold( - new NodeLoggingAPI.SetThresholdRequestBody(params.get("threshold")))); + loggingApi.setMessageThreshold(new SetThresholdRequestBody(params.get("threshold")))); } // Write something at each level if (params.get("test") != null) { - NodeLoggingAPI.writeLogsForTesting(); + NodeLogging.writeLogsForTesting(); } String[] set = params.getParams("set"); if (set != null) { - final List changes = - NodeLoggingAPI.LogLevelChange.createRequestBodyFromV1Params(set); + final List changes = NodeLogging.parseLogLevelChanges(set); squashV2Response(rsp, loggingApi.modifyLocalLogLevel(changes)); } @@ -95,7 +96,7 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw } } - private void squashV2Response(SolrQueryResponse rsp, NodeLoggingAPI.LoggingResponse response) { + private void squashV2Response(SolrQueryResponse rsp, LoggingResponse response) { V2ApiUtils.squashIntoSolrResponseWithoutHeader(rsp, response); } @@ -118,7 +119,7 @@ public Collection getApis() { @Override public Collection> getJerseyResources() { - return List.of(NodeLoggingAPI.class); + return List.of(NodeLogging.class); } @Override diff --git a/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java index c99c6f9afea..f06ec4369b1 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java @@ -547,9 +547,7 @@ private static SimpleOrderedMap getSimilarityInfo(Similarity similarity) private static SimpleOrderedMap getAnalyzerInfo(Analyzer analyzer) { SimpleOrderedMap aninfo = new SimpleOrderedMap<>(); aninfo.add("className", analyzer.getClass().getName()); - if (analyzer instanceof TokenizerChain) { - - TokenizerChain tchain = (TokenizerChain) analyzer; + if (analyzer instanceof TokenizerChain tchain) { CharFilterFactory[] cfiltfacs = tchain.getCharFilterFactories(); if (0 < cfiltfacs.length) { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/SegmentsInfoRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/SegmentsInfoRequestHandler.java index 5acd633d11b..93cdf071a1c 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/SegmentsInfoRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/SegmentsInfoRequestHandler.java @@ -237,8 +237,7 @@ private SimpleOrderedMap getSegmentInfo( for (LeafReaderContext lrc : leafContexts) { LeafReader leafReader = lrc.reader(); leafReader = FilterLeafReader.unwrap(leafReader); - if (leafReader instanceof SegmentReader) { - SegmentReader sr = (SegmentReader) leafReader; + if (leafReader instanceof SegmentReader sr) { if (sr.getSegmentInfo().info.equals(segmentCommitInfo.info)) { seg = sr; break; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java index e2d8b7715b8..1ec536f3c22 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java @@ -201,7 +201,7 @@ private void showFromFileSystem(SolrQueryRequest req, SolrQueryResponse rsp) { if (admin == null) { // exception already recorded return; } - + // TODO SOLR-8282 move to PATH File adminFile = admin.toFile(); // Make sure the file exists, is readable and is not a hidden file if (!adminFile.exists()) { @@ -252,7 +252,7 @@ private void showFromFileSystem(SolrQueryRequest req, SolrQueryResponse rsp) { params.set(CommonParams.WT, "raw"); req.setParams(params); - ContentStreamBase content = new ContentStreamBase.FileStream(adminFile); + ContentStreamBase content = new ContentStreamBase.FileStream(adminFile.toPath()); content.setContentType(getSafeContentType(req.getParams().get(USE_CONTENT_TYPE))); rsp.add(RawResponseWriter.CONTENT, content); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/SplitOp.java b/solr/core/src/java/org/apache/solr/handler/admin/SplitOp.java index fd29d31fcd0..e244f3e2543 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/SplitOp.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/SplitOp.java @@ -158,8 +158,7 @@ public void execute(CoreAdminHandler.CallInfo it) throws Exception { } Object routerObj = collection.get(CollectionStateProps.DOC_ROUTER); // for back-compat with Solr 4.4 - if (routerObj instanceof Map) { - Map routerProps = (Map) routerObj; + if (routerObj instanceof Map routerProps) { routeFieldName = (String) routerProps.get("field"); } } @@ -275,8 +274,7 @@ private void handleGetRanges(CoreAdminHandler.CallInfo it, String coreName) thro Object routerObj = collection.get(CollectionStateProps.DOC_ROUTER); // for back-compat with Solr 4.4 - if (routerObj instanceof Map) { - Map routerProps = (Map) routerObj; + if (routerObj instanceof Map routerProps) { routeFieldName = (String) routerProps.get("field"); } if (routeFieldName == null) { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java index 6f30e06b340..cb5218bfa71 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java @@ -19,13 +19,13 @@ import static org.apache.solr.common.params.CommonParams.NAME; import com.codahale.metrics.Gauge; -import java.io.File; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.lang.management.ManagementFactory; import java.lang.management.OperatingSystemMXBean; import java.lang.management.RuntimeMXBean; import java.net.InetAddress; +import java.nio.file.Path; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; import java.util.ArrayList; @@ -193,7 +193,7 @@ private SimpleOrderedMap getCoreInfo(SolrCore core, IndexSchema schema) // Solr Home SimpleOrderedMap dirs = new SimpleOrderedMap<>(); - dirs.add("cwd", new File(System.getProperty("user.dir")).getAbsolutePath()); + dirs.add("cwd", Path.of(System.getProperty("user.dir")).toAbsolutePath().toString()); dirs.add("instance", core.getInstancePath().toString()); try { dirs.add("data", core.getDirectoryFactory().normalize(core.getDataDir())); @@ -336,8 +336,7 @@ public SimpleOrderedMap getSecurityInfo(SolrQueryRequest req) { // Mapped roles for this principal @SuppressWarnings("resource") AuthorizationPlugin auth = cc == null ? null : cc.getAuthorizationPlugin(); - if (auth instanceof RuleBasedAuthorizationPluginBase) { - RuleBasedAuthorizationPluginBase rbap = (RuleBasedAuthorizationPluginBase) auth; + if (auth instanceof RuleBasedAuthorizationPluginBase rbap) { Set roles = rbap.getUserRoles(req.getUserPrincipal()); info.add("roles", roles); if (roles == null) { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperReadAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperRead.java similarity index 51% rename from solr/core/src/java/org/apache/solr/handler/admin/ZookeeperReadAPI.java rename to solr/core/src/java/org/apache/solr/handler/admin/ZookeeperRead.java index c3e2c4c6a93..8ded06a7a01 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperReadAPI.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperRead.java @@ -17,112 +17,77 @@ package org.apache.solr.handler.admin; -import static org.apache.solr.response.RawResponseWriter.CONTENT; import static org.apache.solr.security.PermissionNameProvider.Name.SECURITY_READ_PERM; import static org.apache.solr.security.PermissionNameProvider.Name.ZK_READ_PERM; -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.swagger.v3.oas.annotations.Parameter; import jakarta.inject.Inject; -import jakarta.ws.rs.GET; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.PathParam; -import jakarta.ws.rs.Produces; -import jakarta.ws.rs.QueryParam; -import jakarta.ws.rs.core.MediaType; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.endpoint.ZooKeeperReadApis; +import org.apache.solr.client.api.model.ZooKeeperFileResponse; +import org.apache.solr.client.api.model.ZooKeeperListChildrenResponse; +import org.apache.solr.client.api.model.ZooKeeperStat; import org.apache.solr.client.solrj.impl.BinaryResponseParser; import org.apache.solr.client.solrj.impl.XMLResponseParser; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.util.ContentStream; import org.apache.solr.common.util.ContentStreamBase; import org.apache.solr.core.CoreContainer; import org.apache.solr.handler.admin.api.AdminAPIBase; -import org.apache.solr.jersey.ExperimentalResponse; -import org.apache.solr.jersey.JacksonReflectMapWriter; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.RawResponseWriter; import org.apache.solr.response.SolrQueryResponse; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.data.Stat; /** - * Exposes the content of the Zookeeper This is an expert feature that exposes the data inside the - * back end zookeeper.This API may change or be removed in future versions. This is not a public - * API. The data that is returned is not guaranteed to remain same across releases, as the data - * stored in Zookeeper may change from time to time. + * v2 API definition exposing read-content in Zookeeper. + * + *

This is an expert feature that exposes the data inside the back end zookeeper.This API may + * change or be removed in future versions. This is not a public API. The data that is returned is + * not guaranteed to remain same across releases, as the data stored in Zookeeper may change from + * time to time. * * @lucene.experimental */ -@Path("/cluster/zookeeper/") -public class ZookeeperReadAPI extends AdminAPIBase { +public class ZookeeperRead extends AdminAPIBase implements ZooKeeperReadApis { + + private static final String EMPTY = "empty"; + @Inject - public ZookeeperReadAPI( - CoreContainer coreContainer, SolrQueryRequest req, SolrQueryResponse rsp) { + public ZookeeperRead(CoreContainer coreContainer, SolrQueryRequest req, SolrQueryResponse rsp) { super(coreContainer, req, rsp); } /** Request contents of a znode, except security.json */ - @GET - @Path("/data{zkPath:.+}") - @Produces({RawResponseWriter.CONTENT_TYPE, MediaType.APPLICATION_JSON}) + @Override @PermissionName(ZK_READ_PERM) - public ZooKeeperFileResponse readNode( - @Parameter(description = "The path of the node to read from ZooKeeper") @PathParam("zkPath") - String zkPath) { + public ZooKeeperFileResponse readNode(String zkPath) { zkPath = sanitizeZkPath(zkPath); return readNodeAndAddToResponse(zkPath); } /** Request contents of the security.json node */ - @GET - @Path("/data/security.json") - @Produces({RawResponseWriter.CONTENT_TYPE, MediaType.APPLICATION_JSON}) + @Override @PermissionName(SECURITY_READ_PERM) public ZooKeeperFileResponse readSecurityJsonNode() { return readNodeAndAddToResponse("/security.json"); } - private String sanitizeZkPath(String zkPath) { - if (zkPath == null || zkPath.isEmpty()) { - return "/"; - } else if (zkPath.length() > 1 && zkPath.endsWith("/")) { - return zkPath.substring(0, zkPath.length() - 1); - } - - return zkPath; - } - /** List the children of a certain zookeeper znode */ - @GET - @Path("/children{zkPath:.*}") - @Produces({"application/json", "application/javabin"}) + @Override @PermissionName(ZK_READ_PERM) - public ListZkChildrenResponse listNodes( - @Parameter(description = "The path of the ZooKeeper node to stat and list children of") - @PathParam("zkPath") - String zkPath, - @Parameter( - description = - "Controls whether stat information for child nodes is included in the response. 'true' by default.") - @QueryParam("children") - Boolean includeChildren) + public ZooKeeperListChildrenResponse listNodes(String zkPath, Boolean includeChildren) throws Exception { - final ListZkChildrenResponse listResponse = - instantiateJerseyResponse(ListZkChildrenResponse.class); + final ZooKeeperListChildrenResponse listResponse = + instantiateJerseyResponse(ZooKeeperListChildrenResponse.class); zkPath = sanitizeZkPath(zkPath); try { Stat stat = coreContainer.getZkController().getZkClient().exists(zkPath, null, true); - listResponse.stat = new AnnotatedStat(stat); + listResponse.stat = createAnnotatedStatFrom(stat); if (includeChildren != null && !includeChildren.booleanValue()) { return listResponse; } @@ -140,9 +105,9 @@ public ListZkChildrenResponse listNodes( } } - final Map childStats = new HashMap<>(); + final Map childStats = new HashMap<>(); for (Map.Entry e : stats.entrySet()) { - childStats.put(e.getKey(), new AnnotatedStat(e.getValue())); + childStats.put(e.getKey(), createAnnotatedStatFrom(e.getValue())); } listResponse.unknownFields.put(zkPath, childStats); @@ -152,6 +117,16 @@ public ListZkChildrenResponse listNodes( } } + private String sanitizeZkPath(String zkPath) { + if (zkPath == null || zkPath.isEmpty()) { + return "/"; + } else if (zkPath.length() > 1 && zkPath.endsWith("/")) { + return zkPath.substring(0, zkPath.length() - 1); + } + + return zkPath; + } + /** Simple mime type guessing based on first character of the response */ private String guessMime(byte firstByte) { switch (firstByte) { @@ -193,85 +168,20 @@ private byte[] readPathFromZookeeper(String path) { return d; } - public static class ListZkChildrenResponse extends ExperimentalResponse { - @JsonProperty("stat") - public AnnotatedStat stat; - - // TODO Currently the list response (when child information is fetched) consists primarily of an - // object with only one key - the name of the root node - with separate objects under there for - // each child. The additional nesting under the root node doesn't serve much purpose afaict - // and should be removed. - private Map> unknownFields = new HashMap<>(); - - @JsonAnyGetter - public Map> unknownProperties() { - return unknownFields; - } - - @JsonAnySetter - public void setUnknownProperty(String field, Map value) { - unknownFields.put(field, value); - } - } - - public static class AnnotatedStat implements JacksonReflectMapWriter { - @JsonProperty("version") - public int version; - - @JsonProperty("aversion") - public int aversion; - - @JsonProperty("children") - public int children; - - @JsonProperty("ctime") - public long ctime; - - @JsonProperty("cversion") - public int cversion; - - @JsonProperty("czxid") - public long czxid; - - @JsonProperty("ephemeralOwner") - public long ephemeralOwner; - - @JsonProperty("mtime") - public long mtime; - - @JsonProperty("mzxid") - public long mzxid; - - @JsonProperty("pzxid") - public long pzxid; - - @JsonProperty("dataLength") - public int dataLength; - - public AnnotatedStat(Stat stat) { - this.version = stat.getVersion(); - this.aversion = stat.getAversion(); - this.children = stat.getNumChildren(); - this.ctime = stat.getCtime(); - this.cversion = stat.getCversion(); - this.czxid = stat.getCzxid(); - this.ephemeralOwner = stat.getEphemeralOwner(); - this.mtime = stat.getMtime(); - this.mzxid = stat.getMzxid(); - this.pzxid = stat.getPzxid(); - this.dataLength = stat.getDataLength(); - } - - public AnnotatedStat() {} - } - - private static final String EMPTY = "empty"; - - public static class ZooKeeperFileResponse extends SolrJerseyResponse { - @JsonProperty(CONTENT) // A flag value that RawResponseWriter handles specially - public ContentStream output; - - @JsonProperty("zkData") - public String zkData; + public static ZooKeeperStat createAnnotatedStatFrom(Stat stat) { + final var annotatedStat = new ZooKeeperStat(); + annotatedStat.version = stat.getVersion(); + annotatedStat.aversion = stat.getAversion(); + annotatedStat.children = stat.getNumChildren(); + annotatedStat.ctime = stat.getCtime(); + annotatedStat.cversion = stat.getCversion(); + annotatedStat.czxid = stat.getCzxid(); + annotatedStat.ephemeralOwner = stat.getEphemeralOwner(); + annotatedStat.mtime = stat.getMtime(); + annotatedStat.mzxid = stat.getMzxid(); + annotatedStat.pzxid = stat.getPzxid(); + annotatedStat.dataLength = stat.getDataLength(); + + return annotatedStat; } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/ClusterProperty.java b/solr/core/src/java/org/apache/solr/handler/admin/api/ClusterProperty.java new file mode 100644 index 00000000000..efce79f7e84 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/ClusterProperty.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin.api; + +import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; +import static org.apache.solr.security.PermissionNameProvider.Name.COLL_READ_PERM; + +import jakarta.inject.Inject; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Map; +import org.apache.solr.client.api.endpoint.ClusterPropertyApis; +import org.apache.solr.client.api.model.ClusterPropertyDetails; +import org.apache.solr.client.api.model.GetClusterPropertyResponse; +import org.apache.solr.client.api.model.ListClusterPropertiesResponse; +import org.apache.solr.client.api.model.SetClusterPropertyRequestBody; +import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ClusterProperties; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.jersey.PermissionName; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; + +public class ClusterProperty extends AdminAPIBase implements ClusterPropertyApis { + protected final ClusterProperties clusterProperties; + + @Inject + public ClusterProperty( + CoreContainer coreContainer, + SolrQueryRequest solrQueryRequest, + SolrQueryResponse solrQueryResponse) { + super(coreContainer, solrQueryRequest, solrQueryResponse); + this.clusterProperties = + new ClusterProperties( + fetchAndValidateZooKeeperAwareCoreContainer().getZkController().getZkClient()); + } + + /** + * V2 API for listing cluster properties. + * + *

This API (GET /api/cluster/properties) has no v1 equivalent. + */ + @Override + @PermissionName(COLL_READ_PERM) + public ListClusterPropertiesResponse listClusterProperties() { + ListClusterPropertiesResponse response = + instantiateJerseyResponse(ListClusterPropertiesResponse.class); + + try { + response.clusterProperties = + new ArrayList<>(clusterProperties.getClusterProperties().keySet()); + } catch (IOException e) { + throw new RuntimeException(e); + } + + return response; + } + + /** + * V2 API for returning the value of a cluster property. + * + *

This API (GET /api/cluster/properties/{propertyName}) has no v1 equivalent. + */ + @Override + @PermissionName(COLL_READ_PERM) + public SolrJerseyResponse getClusterProperty(String propertyName) { + GetClusterPropertyResponse response = + instantiateJerseyResponse(GetClusterPropertyResponse.class); + + try { + Object value = clusterProperties.getClusterProperties().get(propertyName); + if (value != null) { + response.clusterProperty = new ClusterPropertyDetails(); + response.clusterProperty.name = propertyName; + response.clusterProperty.value = value; + } else { + throw new SolrException( + SolrException.ErrorCode.NOT_FOUND, "No such cluster property [" + propertyName + "]"); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + + return response; + } + + /** + * V2 API for setting the value of a single new or existing cluster property. + * + *

This API (PUT /api/cluster/properties/{propertyName} with an object listing the value) is + * equivalent to the v1 GET + * /solr/admin/collections?action=CLUSTERPROP&name={propertyName}&val={propertyValue} API. + */ + @Override + @PermissionName(COLL_EDIT_PERM) + public SolrJerseyResponse createOrUpdateClusterProperty( + String propertyName, SetClusterPropertyRequestBody requestBody) throws IOException { + SolrJerseyResponse response = instantiateJerseyResponse(SolrJerseyResponse.class); + clusterProperties.setClusterProperty(propertyName, requestBody.value); + return response; + } + + /** + * V2 API for setting the value of nested cluster properties. + * + *

This API (PUT /api/cluster/properties with an object listing those properties) has no v1 + * equivalent. + */ + @Override + @PermissionName(COLL_EDIT_PERM) + public SolrJerseyResponse createOrUpdateNestedClusterProperty( + Map propertyValuesByName) { + SolrJerseyResponse response = instantiateJerseyResponse(SolrJerseyResponse.class); + try { + clusterProperties.setClusterProperties(propertyValuesByName); + } catch (Exception e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error in API", e); + } + return response; + } + + /** + * V2 API for deleting a cluster property. + * + *

This API (DELETE /api/cluster/properties/{propertyName}) is equivalent to the v1 GET + * /solr/admin/collections?action=CLUSTERPROP&name={propertyName} API. + */ + @PermissionName(COLL_EDIT_PERM) + @Override + public SolrJerseyResponse deleteClusterProperty(String propertyName) { + final var response = instantiateJerseyResponse(SolrJerseyResponse.class); + + try { + clusterProperties.setClusterProperty(propertyName, null); + } catch (IOException e) { + throw new RuntimeException(e); + } + + return response; + } +} diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CoreReplication.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CoreReplication.java new file mode 100644 index 00000000000..1389c6e780f --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CoreReplication.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin.api; + +import static org.apache.solr.security.PermissionNameProvider.Name.CORE_READ_PERM; + +import jakarta.inject.Inject; +import jakarta.ws.rs.core.StreamingOutput; +import java.io.IOException; +import org.apache.solr.client.api.endpoint.ReplicationApis; +import org.apache.solr.client.api.model.FileListResponse; +import org.apache.solr.client.api.model.IndexVersionResponse; +import org.apache.solr.common.SolrException; +import org.apache.solr.core.SolrCore; +import org.apache.solr.jersey.PermissionName; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; + +/** + * V2 API implementation of {@link ReplicationApis} + * + *

These APIs are analogous to the v1 /coreName/replication APIs. + */ +public class CoreReplication extends ReplicationAPIBase implements ReplicationApis { + + @Inject + public CoreReplication(SolrCore solrCore, SolrQueryRequest req, SolrQueryResponse rsp) { + super(solrCore, req, rsp); + } + + @Override + @PermissionName(CORE_READ_PERM) + public IndexVersionResponse fetchIndexVersion() throws IOException { + return doFetchIndexVersion(); + } + + @Override + @PermissionName(CORE_READ_PERM) + public FileListResponse fetchFileList(long gen) { + return doFetchFileList(gen); + } + + @Override + @PermissionName(CORE_READ_PERM) + public StreamingOutput fetchFile( + String filePath, + String dirType, + String offset, + String len, + Boolean compression, + Boolean checksum, + double maxWriteMBPerSec, + Long gen) { + if (dirType == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Must provide a dirType "); + } + return doFetchFile( + filePath, dirType, offset, len, compression, checksum, maxWriteMBPerSec, gen); + } +} diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CoreReplicationAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CoreReplicationAPI.java deleted file mode 100644 index f6bf4f4c05d..00000000000 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CoreReplicationAPI.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.handler.admin.api; - -import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE_V2; -import static org.apache.solr.security.PermissionNameProvider.Name.CORE_READ_PERM; - -import com.fasterxml.jackson.annotation.JsonProperty; -import io.swagger.v3.oas.annotations.Parameter; -import jakarta.inject.Inject; -import jakarta.ws.rs.GET; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.Produces; -import jakarta.ws.rs.QueryParam; -import jakarta.ws.rs.core.MediaType; -import java.io.IOException; -import java.util.List; -import org.apache.solr.client.api.model.SolrJerseyResponse; -import org.apache.solr.core.SolrCore; -import org.apache.solr.jersey.JacksonReflectMapWriter; -import org.apache.solr.jersey.PermissionName; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; - -/** - * V2 APIs for inspecting and replicating indices - * - *

These APIs are analogous to the v1 /coreName/replication APIs. - */ -@Path("/cores/{coreName}/replication") -public class CoreReplicationAPI extends ReplicationAPIBase { - - @Inject - public CoreReplicationAPI(SolrCore solrCore, SolrQueryRequest req, SolrQueryResponse rsp) { - super(solrCore, req, rsp); - } - - @GET - @Path("/indexversion") - @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, BINARY_CONTENT_TYPE_V2}) - @PermissionName(CORE_READ_PERM) - public IndexVersionResponse fetchIndexVersion() throws IOException { - return doFetchIndexVersion(); - } - - @GET - @Path("/files") - @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, BINARY_CONTENT_TYPE_V2}) - @PermissionName(CORE_READ_PERM) - public FileListResponse fetchFileList( - @Parameter(description = "The generation number of the index", required = true) - @QueryParam("generation") - long gen) { - return doFetchFileList(gen); - } - - /** Response for {@link CoreReplicationAPI#fetchIndexVersion()}. */ - public static class IndexVersionResponse extends SolrJerseyResponse { - - @JsonProperty("indexversion") - public Long indexVersion; - - @JsonProperty("generation") - public Long generation; - - @JsonProperty("status") - public String status; - - public IndexVersionResponse() {} - - public IndexVersionResponse(Long indexVersion, Long generation, String status) { - this.indexVersion = indexVersion; - this.generation = generation; - this.status = status; - } - } - - /** Response for {@link CoreReplicationAPI#fetchFileList(long)}. */ - public static class FileListResponse extends SolrJerseyResponse { - @JsonProperty("filelist") - public List fileList; - - @JsonProperty("confFiles") - public List confFiles; - - @JsonProperty("status") - public String status; - - @JsonProperty("message") - public String message; - - @JsonProperty("exception") - public Exception exception; - - public FileListResponse() {} - } - - /** - * Contained in {@link CoreReplicationAPI.FileListResponse}, this holds metadata from a file for - * an index - */ - public static class FileMetaData implements JacksonReflectMapWriter { - - @JsonProperty("size") - public long size; - - @JsonProperty("name") - public String name; - - @JsonProperty("checksum") - public long checksum; - - @JsonProperty("alias") - public String alias; - - public FileMetaData() {} - - public FileMetaData(long size, String name, long checksum) { - this.size = size; - this.name = name; - this.checksum = checksum; - } - } -} diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateAliasAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateAlias.java similarity index 69% rename from solr/core/src/java/org/apache/solr/handler/admin/api/CreateAliasAPI.java rename to solr/core/src/java/org/apache/solr/handler/admin/api/CreateAlias.java index 8e8ecaf405f..c0497891b78 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateAliasAPI.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateAlias.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE_V2; import static org.apache.solr.client.solrj.request.beans.V2ApiConstants.COLLECTIONS; import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.cloud.api.collections.RoutedAlias.CATEGORY; @@ -26,6 +25,7 @@ import static org.apache.solr.cloud.api.collections.RoutedAlias.TIME; import static org.apache.solr.cloud.api.collections.TimeRoutedAlias.ROUTER_MAX_FUTURE; import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST; +import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR; import static org.apache.solr.common.params.CollectionAdminParams.ROUTER_PREFIX; import static org.apache.solr.common.params.CommonAdminParams.ASYNC; import static org.apache.solr.common.params.CommonParams.NAME; @@ -33,23 +33,19 @@ import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; import jakarta.inject.Inject; -import jakarta.ws.rs.POST; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.Produces; -import jakarta.ws.rs.core.MediaType; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; -import java.util.List; import java.util.Locale; import java.util.Map; -import org.apache.solr.client.api.model.CreateCollectionRequestBody; +import org.apache.solr.client.api.endpoint.CreateAliasApi; +import org.apache.solr.client.api.model.CategoryRoutedAliasProperties; +import org.apache.solr.client.api.model.CreateAliasRequestBody; +import org.apache.solr.client.api.model.RoutedAliasProperties; import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; +import org.apache.solr.client.api.model.TimeRoutedAliasProperties; import org.apache.solr.client.solrj.RoutedAliasTypes; import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.client.solrj.util.SolrIdentifierValidator; @@ -66,7 +62,6 @@ import org.apache.solr.common.util.StrUtils; import org.apache.solr.core.CoreContainer; import org.apache.solr.handler.admin.CollectionsHandler; -import org.apache.solr.jersey.JacksonReflectMapWriter; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -77,18 +72,16 @@ * *

This API is analogous to the v1 /admin/collections?action=CREATEALIAS command. */ -@Path("/aliases") -public class CreateAliasAPI extends AdminAPIBase { +public class CreateAlias extends AdminAPIBase implements CreateAliasApi { @Inject - public CreateAliasAPI( + public CreateAlias( CoreContainer coreContainer, SolrQueryRequest solrQueryRequest, SolrQueryResponse solrQueryResponse) { super(coreContainer, solrQueryRequest, solrQueryResponse); } - @POST - @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, BINARY_CONTENT_TYPE_V2}) + @Override @PermissionName(COLL_EDIT_PERM) public SolrJerseyResponse createAlias(CreateAliasRequestBody requestBody) throws Exception { final SubResponseAccumulatingJerseyResponse response = @@ -98,7 +91,7 @@ public SolrJerseyResponse createAlias(CreateAliasRequestBody requestBody) throws if (requestBody == null) { throw new SolrException(BAD_REQUEST, "Request body is required but missing"); } - requestBody.validate(); + validateRequestBody(requestBody); ZkNodeProps remoteMessage; // Validation ensures that the request has either collections or a router but not both. @@ -153,10 +146,12 @@ public static ZkNodeProps createRemoteMessageForRoutedAlias(CreateAliasRequestBo if (requestBody.routers.size() > 1) { // Multi-dimensional alias for (int i = 0; i < requestBody.routers.size(); i++) { - requestBody.routers.get(i).addRemoteMessageProperties(remoteMessage, "router." + i + "."); + createValidationHelper(requestBody.routers.get(i)) + .addRemoteMessageProperties(remoteMessage, "router." + i + "."); } } else if (requestBody.routers.size() == 1) { // Single dimensional alias - requestBody.routers.get(0).addRemoteMessageProperties(remoteMessage, "router."); + createValidationHelper(requestBody.routers.get(0)) + .addRemoteMessageProperties(remoteMessage, "router."); } if (requestBody.collCreationParameters != null) { @@ -213,9 +208,9 @@ public static RoutedAliasProperties createFromSolrParams( String type, SolrParams params, String propertyPrefix) { final String typeLower = type.toLowerCase(Locale.ROOT); if (typeLower.startsWith(TIME)) { - return TimeRoutedAliasProperties.createFromSolrParams(params, propertyPrefix); + return TimeRoutedAliasValidationHelper.createFromSolrParams(params, propertyPrefix); } else if (typeLower.startsWith(CATEGORY)) { - return CategoryRoutedAliasProperties.createFromSolrParams(params, propertyPrefix); + return CategoryRoutedAliasValidationHelper.createFromSolrParams(params, propertyPrefix); } else { throw new SolrException( BAD_REQUEST, @@ -226,71 +221,46 @@ public static RoutedAliasProperties createFromSolrParams( } } - public static class CreateAliasRequestBody implements JacksonReflectMapWriter { - @JsonProperty(required = true) - public String name; + public static void validateRequestBody(CreateAliasRequestBody requestBody) { + SolrIdentifierValidator.validateAliasName(requestBody.name); - @JsonProperty("collections") - public List collections; - - @JsonProperty(ASYNC) - public String async; - - @JsonProperty("routers") - public List routers; - - @JsonProperty("create-collection") - public CreateCollectionRequestBody collCreationParameters; - - public void validate() { - SolrIdentifierValidator.validateAliasName(name); + if (CollectionUtil.isEmpty(requestBody.collections) + && CollectionUtil.isEmpty(requestBody.routers)) { + throw new SolrException( + BAD_REQUEST, + "Alias creation requires either a list of either collections (for creating a traditional alias) or routers (for creating a routed alias)"); + } - if (CollectionUtil.isEmpty(collections) && CollectionUtil.isEmpty(routers)) { + if (CollectionUtil.isNotEmpty(requestBody.routers)) { + requestBody.routers.forEach(r -> createValidationHelper(r).validate()); + if (CollectionUtil.isNotEmpty(requestBody.collections)) { throw new SolrException( - BAD_REQUEST, - "Alias creation requires either a list of either collections (for creating a traditional alias) or routers (for creating a routed alias)"); + BAD_REQUEST, "Collections cannot be specified when creating a routed alias."); } - if (CollectionUtil.isNotEmpty(routers)) { - routers.forEach(r -> r.validate()); - if (CollectionUtil.isNotEmpty(collections)) { + final var createCollReqBody = requestBody.collCreationParameters; + if (createCollReqBody != null) { + if (createCollReqBody.name != null) { throw new SolrException( - BAD_REQUEST, "Collections cannot be specified when creating a routed alias."); + BAD_REQUEST, + "routed aliases calculate names for their " + + "dependent collections, you cannot specify the name."); } - - final var createCollReqBody = collCreationParameters; - if (createCollReqBody != null) { - if (createCollReqBody.name != null) { - throw new SolrException( - BAD_REQUEST, - "routed aliases calculate names for their " - + "dependent collections, you cannot specify the name."); - } - if (createCollReqBody.config == null) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - "Routed alias creation requires a configset name to use for any collections created by the alias."); - } + if (createCollReqBody.config == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Routed alias creation requires a configset name to use for any collections created by the alias."); } } } } - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") - @JsonSubTypes({ - @JsonSubTypes.Type(value = TimeRoutedAliasProperties.class, name = "time"), - @JsonSubTypes.Type(value = CategoryRoutedAliasProperties.class, name = "category") - }) - public abstract static class RoutedAliasProperties implements JacksonReflectMapWriter { - @JsonProperty(required = true) - public String field; + private interface RoutedAliasValidationHelper { + void validate(); - public abstract void validate(); + void addRemoteMessageProperties(Map remoteMessage, String prefix); - public abstract void addRemoteMessageProperties( - Map remoteMessage, String prefix); - - protected void ensureRequiredFieldPresent(Object val, String name) { + default void ensureRequiredFieldPresent(Object val, String name) { if (val == null) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Missing required parameter: " + name); @@ -298,38 +268,38 @@ protected void ensureRequiredFieldPresent(Object val, String name) { } } - public static class TimeRoutedAliasProperties extends RoutedAliasProperties { - // Expected to be a date/time in ISO format, or 'NOW' - @JsonProperty(required = true) - public String start; - - // TODO Change this to 'timezone' or something less abbreviated - @JsonProperty("tz") - public String tz; - - @JsonProperty(required = true) - public String interval; - - @JsonProperty("maxFutureMs") - public Long maxFutureMs; + private static RoutedAliasValidationHelper createValidationHelper( + RoutedAliasProperties routedAliasProperties) { + if (routedAliasProperties instanceof TimeRoutedAliasProperties) { + return new TimeRoutedAliasValidationHelper((TimeRoutedAliasProperties) routedAliasProperties); + } else if (routedAliasProperties instanceof CategoryRoutedAliasProperties) { + return new CategoryRoutedAliasValidationHelper( + (CategoryRoutedAliasProperties) routedAliasProperties); + } else { + throw new SolrException( + SERVER_ERROR, "Unrecognized routed-alias type provided: " + routedAliasProperties); + } + } - @JsonProperty("preemptiveCreateMath") - public String preemptiveCreateMath; + public static class TimeRoutedAliasValidationHelper implements RoutedAliasValidationHelper { + private final TimeRoutedAliasProperties aliasProperties; - @JsonProperty("autoDeleteAge") - public String autoDeleteAge; + public TimeRoutedAliasValidationHelper(TimeRoutedAliasProperties aliasProperties) { + this.aliasProperties = aliasProperties; + } @Override public void validate() { - ensureRequiredFieldPresent(field, "'field' on time routed alias"); - ensureRequiredFieldPresent(start, "'start' on time routed alias"); - ensureRequiredFieldPresent(interval, "'interval' on time routed alias"); + ensureRequiredFieldPresent(aliasProperties.field, "'field' on time routed alias"); + ensureRequiredFieldPresent(aliasProperties.start, "'start' on time routed alias"); + ensureRequiredFieldPresent(aliasProperties.interval, "'interval' on time routed alias"); // Ensures that provided 'start' and optional 'tz' are of the right format. - TimeRoutedAlias.parseStringAsInstant(start, TimeZoneUtils.parseTimezone(tz)); + TimeRoutedAlias.parseStringAsInstant( + aliasProperties.start, TimeZoneUtils.parseTimezone(aliasProperties.tz)); // maxFutureMs must be > 0 if provided - if (maxFutureMs != null && maxFutureMs < 0) { + if (aliasProperties.maxFutureMs != null && aliasProperties.maxFutureMs < 0) { throw new SolrException(BAD_REQUEST, ROUTER_MAX_FUTURE + " must be >= 0"); } } @@ -337,15 +307,17 @@ public void validate() { @Override public void addRemoteMessageProperties(Map remoteMessage, String prefix) { remoteMessage.put(prefix + CoreAdminParams.NAME, TIME); - remoteMessage.put(prefix + "field", field); - remoteMessage.put(prefix + "start", start); - remoteMessage.put(prefix + "interval", interval); - - if (tz != null) remoteMessage.put(prefix + "tz", tz); - if (maxFutureMs != null) remoteMessage.put(prefix + "maxFutureMs", maxFutureMs); - if (preemptiveCreateMath != null) - remoteMessage.put(prefix + "preemptiveCreateMath", preemptiveCreateMath); - if (autoDeleteAge != null) remoteMessage.put(prefix + "autoDeleteAge", autoDeleteAge); + remoteMessage.put(prefix + "field", aliasProperties.field); + remoteMessage.put(prefix + "start", aliasProperties.start); + remoteMessage.put(prefix + "interval", aliasProperties.interval); + + if (aliasProperties.tz != null) remoteMessage.put(prefix + "tz", aliasProperties.tz); + if (aliasProperties.maxFutureMs != null) + remoteMessage.put(prefix + "maxFutureMs", aliasProperties.maxFutureMs); + if (aliasProperties.preemptiveCreateMath != null) + remoteMessage.put(prefix + "preemptiveCreateMath", aliasProperties.preemptiveCreateMath); + if (aliasProperties.autoDeleteAge != null) + remoteMessage.put(prefix + "autoDeleteAge", aliasProperties.autoDeleteAge); } public static TimeRoutedAliasProperties createFromSolrParams( @@ -365,25 +337,27 @@ public static TimeRoutedAliasProperties createFromSolrParams( } } - public static class CategoryRoutedAliasProperties extends RoutedAliasProperties { - @JsonProperty("maxCardinality") - public Long maxCardinality; + public static class CategoryRoutedAliasValidationHelper implements RoutedAliasValidationHelper { + private final CategoryRoutedAliasProperties aliasProperties; - @JsonProperty("mustMatch") - public String mustMatch; + public CategoryRoutedAliasValidationHelper(CategoryRoutedAliasProperties aliasProperties) { + this.aliasProperties = aliasProperties; + } @Override public void validate() { - ensureRequiredFieldPresent(field, "'field' on category routed alias"); + ensureRequiredFieldPresent(aliasProperties.field, "'field' on category routed alias"); } @Override public void addRemoteMessageProperties(Map remoteMessage, String prefix) { remoteMessage.put(prefix + CoreAdminParams.NAME, CATEGORY); - remoteMessage.put(prefix + "field", field); + remoteMessage.put(prefix + "field", aliasProperties.field); - if (maxCardinality != null) remoteMessage.put(prefix + "maxCardinality", maxCardinality); - if (StrUtils.isNotBlank(mustMatch)) remoteMessage.put(prefix + "mustMatch", mustMatch); + if (aliasProperties.maxCardinality != null) + remoteMessage.put(prefix + "maxCardinality", aliasProperties.maxCardinality); + if (StrUtils.isNotBlank(aliasProperties.mustMatch)) + remoteMessage.put(prefix + "mustMatch", aliasProperties.mustMatch); } public static CategoryRoutedAliasProperties createFromSolrParams( diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionBackup.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionBackup.java index 09729da6ea2..94c1c7768fa 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionBackup.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionBackup.java @@ -38,7 +38,7 @@ import jakarta.inject.Inject; import java.util.HashMap; import java.util.Map; -import org.apache.solr.client.api.endpoint.CreateCollectionBackupApi; +import org.apache.solr.client.api.endpoint.CollectionBackupApi; import org.apache.solr.client.api.model.CreateCollectionBackupRequestBody; import org.apache.solr.client.api.model.CreateCollectionBackupResponseBody; import org.apache.solr.client.api.model.SolrJerseyResponse; @@ -58,7 +58,7 @@ import org.apache.zookeeper.common.StringUtils; /** V2 API for creating a new "backup" of a specified collection */ -public class CreateCollectionBackup extends BackupAPIBase implements CreateCollectionBackupApi { +public class CreateCollectionBackup extends BackupAPIBase implements CollectionBackupApi.Create { private final ObjectMapper objectMapper; @Inject diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionSnapshot.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionSnapshot.java index dd4a7a4033b..9d0a6d295af 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionSnapshot.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionSnapshot.java @@ -26,7 +26,7 @@ import jakarta.inject.Inject; import java.util.HashMap; import java.util.Map; -import org.apache.solr.client.api.endpoint.CreateCollectionSnapshotApi; +import org.apache.solr.client.api.endpoint.CollectionSnapshotApis; import org.apache.solr.client.api.model.CreateCollectionSnapshotRequestBody; import org.apache.solr.client.api.model.CreateCollectionSnapshotResponse; import org.apache.solr.client.solrj.SolrResponse; @@ -43,7 +43,8 @@ import org.apache.solr.response.SolrQueryResponse; /** V2 API implementation for creating a collection-level snapshot. */ -public class CreateCollectionSnapshot extends AdminAPIBase implements CreateCollectionSnapshotApi { +public class CreateCollectionSnapshot extends AdminAPIBase + implements CollectionSnapshotApis.Create { @Inject public CreateCollectionSnapshot( diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionSnapshot.java b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionSnapshot.java index 5ead94e0432..abe76571dde 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionSnapshot.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/DeleteCollectionSnapshot.java @@ -26,7 +26,7 @@ import jakarta.inject.Inject; import java.util.HashMap; import java.util.Map; -import org.apache.solr.client.api.endpoint.DeleteCollectionSnapshotApi; +import org.apache.solr.client.api.endpoint.CollectionSnapshotApis; import org.apache.solr.client.api.model.DeleteCollectionSnapshotResponse; import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.common.cloud.ZkNodeProps; @@ -39,7 +39,8 @@ import org.apache.solr.response.SolrQueryResponse; /** V2 API impl for Deleting Collection Snapshots. */ -public class DeleteCollectionSnapshot extends AdminAPIBase implements DeleteCollectionSnapshotApi { +public class DeleteCollectionSnapshot extends AdminAPIBase + implements CollectionSnapshotApis.Delete { @Inject public DeleteCollectionSnapshot( diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/GetSchema.java b/solr/core/src/java/org/apache/solr/handler/admin/api/GetSchema.java index 909d0cfe997..7d914533b35 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/GetSchema.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/GetSchema.java @@ -110,8 +110,7 @@ public SchemaZkVersionResponse getSchemaZkVersion(Integer refreshIfBelowVersion) final SchemaZkVersionResponse response = instantiateJerseyResponse(SchemaZkVersionResponse.class); int zkVersion = -1; - if (solrCore.getLatestSchema() instanceof ManagedIndexSchema) { - ManagedIndexSchema managed = (ManagedIndexSchema) solrCore.getLatestSchema(); + if (solrCore.getLatestSchema() instanceof ManagedIndexSchema managed) { zkVersion = managed.getSchemaZkVersion(); if (refreshIfBelowVersion != -1 && zkVersion < refreshIfBelowVersion) { log.info( diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/GetSchemaFieldAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/GetSchemaField.java similarity index 72% rename from solr/core/src/java/org/apache/solr/handler/admin/api/GetSchemaFieldAPI.java rename to solr/core/src/java/org/apache/solr/handler/admin/api/GetSchemaField.java index 5050787429c..70fc868bee5 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/GetSchemaFieldAPI.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/GetSchemaField.java @@ -17,19 +17,18 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE_V2; - -import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.inject.Inject; -import jakarta.ws.rs.GET; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.PathParam; -import jakarta.ws.rs.Produces; -import jakarta.ws.rs.core.MediaType; import java.util.List; import java.util.Map; import org.apache.solr.api.JerseyResource; -import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.endpoint.GetSchemaApi; +import org.apache.solr.client.api.model.SchemaGetDynamicFieldInfoResponse; +import org.apache.solr.client.api.model.SchemaGetFieldInfoResponse; +import org.apache.solr.client.api.model.SchemaGetFieldTypeInfoResponse; +import org.apache.solr.client.api.model.SchemaListCopyFieldsResponse; +import org.apache.solr.client.api.model.SchemaListDynamicFieldsResponse; +import org.apache.solr.client.api.model.SchemaListFieldTypesResponse; +import org.apache.solr.client.api.model.SchemaListFieldsResponse; import org.apache.solr.common.MapWriter; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.SolrClassLoader; @@ -55,8 +54,7 @@ *

  • /fieldtypes/{fieldTypeName} * */ -@Path("/{a:cores|collections}/{collectionName}/schema") -public class GetSchemaFieldAPI /*extends GetSchemaAPI*/ extends JerseyResource { +public class GetSchemaField extends JerseyResource implements GetSchemaApi.Fields { private final IndexSchema indexSchema; private final SolrParams params; @@ -64,14 +62,12 @@ public class GetSchemaFieldAPI /*extends GetSchemaAPI*/ extends JerseyResource { // TODO Stop using SolrParams here and instead give API methods parameters representing only those // query-params that they support @Inject - public GetSchemaFieldAPI(IndexSchema indexSchema, SolrParams params) { + public GetSchemaField(IndexSchema indexSchema, SolrParams params) { this.indexSchema = indexSchema; this.params = params; } - @GET - @Path("/fields") - @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, BINARY_CONTENT_TYPE_V2}) + @Override @PermissionName(PermissionNameProvider.Name.SCHEMA_READ_PERM) public SchemaListFieldsResponse listSchemaFields() { SchemaListFieldsResponse response = instantiateJerseyResponse(SchemaListFieldsResponse.class); @@ -82,16 +78,9 @@ public SchemaListFieldsResponse listSchemaFields() { return response; } - public static class SchemaListFieldsResponse extends SolrJerseyResponse { - @JsonProperty("fields") - public List fields; - } - - @GET - @Path("/fields/{fieldName}") - @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_ATOM_XML, BINARY_CONTENT_TYPE_V2}) + @Override @PermissionName(PermissionNameProvider.Name.SCHEMA_READ_PERM) - public SchemaGetFieldInfoResponse getFieldInfo(@PathParam("fieldName") String fieldName) { + public SchemaGetFieldInfoResponse getFieldInfo(String fieldName) { if (fieldName == null) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Field name must not be null"); } @@ -107,14 +96,7 @@ public SchemaGetFieldInfoResponse getFieldInfo(@PathParam("fieldName") String fi throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "No such field [" + fieldName + "]"); } - public static class SchemaGetFieldInfoResponse extends SolrJerseyResponse { - @JsonProperty("field") - public SimpleOrderedMap fieldInfo; - } - - @GET - @Path("/copyfields") - @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, BINARY_CONTENT_TYPE_V2}) + @Override @PermissionName(PermissionNameProvider.Name.SCHEMA_READ_PERM) public SchemaListCopyFieldsResponse listCopyFields() { SchemaListCopyFieldsResponse response = @@ -126,14 +108,7 @@ public SchemaListCopyFieldsResponse listCopyFields() { return response; } - public static class SchemaListCopyFieldsResponse extends SolrJerseyResponse { - @JsonProperty("copyFields") - public List copyFields; - } - - @GET - @Path("/dynamicfields") - @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, BINARY_CONTENT_TYPE_V2}) + @Override @PermissionName(PermissionNameProvider.Name.SCHEMA_READ_PERM) public SchemaListDynamicFieldsResponse listDynamicFields() { SchemaListDynamicFieldsResponse response = @@ -145,17 +120,9 @@ public SchemaListDynamicFieldsResponse listDynamicFields() { return response; } - public static class SchemaListDynamicFieldsResponse extends SolrJerseyResponse { - @JsonProperty("dynamicFields") - public List dynamicFields; - } - - @GET - @Path("/dynamicfields/{fieldName}") - @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_ATOM_XML, BINARY_CONTENT_TYPE_V2}) + @Override @PermissionName(PermissionNameProvider.Name.SCHEMA_READ_PERM) - public SchemaGetDynamicFieldInfoResponse getDynamicFieldInfo( - @PathParam("fieldName") String fieldName) { + public SchemaGetDynamicFieldInfoResponse getDynamicFieldInfo(String fieldName) { if (fieldName == null) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Dynamic field name must not be null"); @@ -174,14 +141,7 @@ public SchemaGetDynamicFieldInfoResponse getDynamicFieldInfo( SolrException.ErrorCode.NOT_FOUND, "No such dynamic field [" + fieldName + "]"); } - public static class SchemaGetDynamicFieldInfoResponse extends SolrJerseyResponse { - @JsonProperty("dynamicField") - public SimpleOrderedMap dynamicFieldInfo; - } - - @GET - @Path("/fieldtypes") - @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, BINARY_CONTENT_TYPE_V2}) + @Override @PermissionName(PermissionNameProvider.Name.SCHEMA_READ_PERM) public SchemaListFieldTypesResponse listSchemaFieldTypes() { SchemaListFieldTypesResponse response = @@ -193,17 +153,9 @@ public SchemaListFieldTypesResponse listSchemaFieldTypes() { return response; } - public static class SchemaListFieldTypesResponse extends SolrJerseyResponse { - @JsonProperty("fieldTypes") - public List fieldTypes; - } - - @GET - @Path("/fieldtypes/{fieldTypeName}") - @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_ATOM_XML, BINARY_CONTENT_TYPE_V2}) + @Override @PermissionName(PermissionNameProvider.Name.SCHEMA_READ_PERM) - public SchemaGetFieldTypeInfoResponse getFieldTypeInfo( - @PathParam("fieldTypeName") String fieldTypeName) { + public SchemaGetFieldTypeInfoResponse getFieldTypeInfo(String fieldTypeName) { if (fieldTypeName == null) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Field type name must not be null"); @@ -223,11 +175,6 @@ public SchemaGetFieldTypeInfoResponse getFieldTypeInfo( SolrException.ErrorCode.NOT_FOUND, "No such field type [" + fieldTypeName + "]"); } - public static class SchemaGetFieldTypeInfoResponse extends SolrJerseyResponse { - @JsonProperty("fieldType") - public SimpleOrderedMap fieldTypeInfo; - } - private List listAllFieldsOfType(String realName, SolrParams params) { String camelCaseRealName = IndexSchema.nameMapping.get(realName); Map propertyValues = indexSchema.getNamedPropertyValues(realName, params); @@ -246,8 +193,7 @@ private SimpleOrderedMap retrieveFieldInfoOfType( String camelCaseRealName = IndexSchema.nameMapping.get(realName); Map propertyValues = indexSchema.getNamedPropertyValues(realName, params); Object o = propertyValues.get(camelCaseRealName); - if (o instanceof List) { - List list = (List) o; + if (o instanceof List list) { for (Object obj : list) { if (obj instanceof SimpleOrderedMap) { SimpleOrderedMap fieldInfo = (SimpleOrderedMap) obj; @@ -269,8 +215,7 @@ private SimpleOrderedMap retrieveFieldInfoOfType( * the response */ private void insertPackageInfo(Object o) { - if (o instanceof List) { - List l = (List) o; + if (o instanceof List l) { for (Object o1 : l) { if (o1 instanceof NamedList || o1 instanceof List) insertPackageInfo(o1); } @@ -283,8 +228,7 @@ private void insertPackageInfo(Object o) { if (v instanceof NamedList || v instanceof List) insertPackageInfo(v); }); Object v = nl.get("class"); - if (v instanceof String) { - String klas = (String) v; + if (v instanceof String klas) { PluginInfo.ClassName parsedClassName = new PluginInfo.ClassName(klas); if (parsedClassName.pkg != null) { SolrClassLoader solrClassLoader = indexSchema.getSolrClassLoader(); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/ListCollectionSnapshotsAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/ListCollectionSnapshots.java similarity index 61% rename from solr/core/src/java/org/apache/solr/handler/admin/api/ListCollectionSnapshotsAPI.java rename to solr/core/src/java/org/apache/solr/handler/admin/api/ListCollectionSnapshots.java index dec0f5998f6..9bbc94f202b 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/ListCollectionSnapshotsAPI.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/ListCollectionSnapshots.java @@ -16,20 +16,13 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE_V2; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_READ_PERM; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.swagger.v3.oas.annotations.Parameter; -import io.swagger.v3.oas.annotations.media.Schema; import jakarta.inject.Inject; -import jakarta.ws.rs.GET; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.PathParam; -import jakarta.ws.rs.Produces; import java.util.Collection; import java.util.Map; -import org.apache.solr.client.api.model.AsyncJerseyResponse; +import org.apache.solr.client.api.endpoint.CollectionSnapshotApis; +import org.apache.solr.client.api.model.ListCollectionSnapshotsResponse; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.util.CollectionUtil; import org.apache.solr.core.CoreContainer; @@ -39,12 +32,11 @@ import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; -/** V2 API for Listing Collection Snapshots. */ -@Path("/collections/{collName}/snapshots") -public class ListCollectionSnapshotsAPI extends AdminAPIBase { +/** V2 API implementation for Listing Collection Snapshots. */ +public class ListCollectionSnapshots extends AdminAPIBase implements CollectionSnapshotApis.List { @Inject - public ListCollectionSnapshotsAPI( + public ListCollectionSnapshots( CoreContainer coreContainer, SolrQueryRequest solrQueryRequest, SolrQueryResponse solrQueryResponse) { @@ -52,16 +44,12 @@ public ListCollectionSnapshotsAPI( } /** This API is analogous to V1's (POST /solr/admin/collections?action=LISTSNAPSHOTS) */ - @GET - @Produces({"application/json", "application/xml", BINARY_CONTENT_TYPE_V2}) + @Override @PermissionName(COLL_READ_PERM) - public ListSnapshotsResponse listSnapshots( - @Parameter(description = "The name of the collection.", required = true) - @PathParam("collName") - String collName) - throws Exception { + public ListCollectionSnapshotsResponse listSnapshots(String collName) throws Exception { - final ListSnapshotsResponse response = instantiateJerseyResponse(ListSnapshotsResponse.class); + final ListCollectionSnapshotsResponse response = + instantiateJerseyResponse(ListCollectionSnapshotsResponse.class); final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer(); recordCollectionForLogAndTracing(collName, solrQueryRequest); @@ -71,20 +59,12 @@ public ListSnapshotsResponse listSnapshots( Collection m = SolrSnapshotManager.listSnapshots(client, collectionName); - Map snapshots = CollectionUtil.newHashMap(m.size()); + final Map snapshots = CollectionUtil.newHashMap(m.size()); for (CollectionSnapshotMetaData metaData : m) { snapshots.put(metaData.getName(), metaData); } - response.snapshots = snapshots; return response; } - - /** The Response for {@link ListCollectionSnapshotsAPI}'s {@link #listSnapshots(String)} */ - public static class ListSnapshotsResponse extends AsyncJerseyResponse { - @Schema(description = "The snapshots for the collection.") - @JsonProperty(SolrSnapshotManager.SNAPSHOTS_INFO) - public Map snapshots; - } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/ListCollections.java b/solr/core/src/java/org/apache/solr/handler/admin/api/ListCollections.java index 4b54b86ae81..81c5fac32c5 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/ListCollections.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/ListCollections.java @@ -20,10 +20,7 @@ import static org.apache.solr.security.PermissionNameProvider.Name.COLL_READ_PERM; import jakarta.inject.Inject; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; -import java.util.Map; import org.apache.solr.client.api.endpoint.ListCollectionsApi; import org.apache.solr.client.api.model.ListCollectionsResponse; import org.apache.solr.common.cloud.DocCollection; @@ -51,10 +48,16 @@ public ListCollectionsResponse listCollections() { instantiateJerseyResponse(ListCollectionsResponse.class); validateZooKeeperAwareCoreContainer(coreContainer); - Map collections = - coreContainer.getZkController().getZkStateReader().getClusterState().getCollectionsMap(); - List collectionList = new ArrayList<>(collections.keySet()); - Collections.sort(collectionList); + // resolve each name to ensure it exists. + // TODO https://issues.apache.org/jira/browse/SOLR-16909 to go direct to ZK? + List collectionList = + coreContainer + .getZkController() + .getClusterState() + .collectionStream() + .map(DocCollection::getName) + .sorted() + .toList(); // XXX should we add aliases here? response.collections = collectionList; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/NodeLoggingAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/NodeLogging.java similarity index 57% rename from solr/core/src/java/org/apache/solr/handler/admin/api/NodeLoggingAPI.java rename to solr/core/src/java/org/apache/solr/handler/admin/api/NodeLogging.java index bbcd4cc8de4..5594d5f084e 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/NodeLoggingAPI.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/NodeLogging.java @@ -17,29 +17,28 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE_V2; import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST; import static org.apache.solr.security.PermissionNameProvider.Name.CONFIG_EDIT_PERM; import static org.apache.solr.security.PermissionNameProvider.Name.CONFIG_READ_PERM; -import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.inject.Inject; -import jakarta.ws.rs.GET; -import jakarta.ws.rs.PUT; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.Produces; -import jakarta.ws.rs.QueryParam; import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import org.apache.solr.api.JerseyResource; -import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.endpoint.NodeLoggingApis; +import org.apache.solr.client.api.model.ListLevelsResponse; +import org.apache.solr.client.api.model.LogLevelChange; +import org.apache.solr.client.api.model.LogLevelInfo; +import org.apache.solr.client.api.model.LogMessageInfo; +import org.apache.solr.client.api.model.LogMessagesResponse; +import org.apache.solr.client.api.model.LoggingResponse; +import org.apache.solr.client.api.model.SetThresholdRequestBody; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; import org.apache.solr.core.CoreContainer; -import org.apache.solr.jersey.JacksonReflectMapWriter; import org.apache.solr.jersey.PermissionName; import org.apache.solr.logging.LogWatcher; import org.slf4j.Logger; @@ -51,8 +50,7 @@ * *

    These APIs ('/api/node/logging' and descendants) are analogous to the v1 /admin/info/logging. */ -@Path("/node/logging") -public class NodeLoggingAPI extends JerseyResource { +public class NodeLogging extends JerseyResource implements NodeLoggingApis { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -60,15 +58,13 @@ public class NodeLoggingAPI extends JerseyResource { private final LogWatcher watcher; @Inject - public NodeLoggingAPI(CoreContainer coreContainer) { + public NodeLogging(CoreContainer coreContainer) { this.coreContainer = coreContainer; this.watcher = coreContainer.getLogging(); } - @Path("/levels") - @GET + @Override @PermissionName(CONFIG_READ_PERM) - @Produces({"application/json", "application/xml", BINARY_CONTENT_TYPE_V2}) public ListLevelsResponse listAllLoggersAndLevels() { ensureLogWatcherEnabled(); final ListLevelsResponse response = instantiateLoggingResponse(ListLevelsResponse.class); @@ -86,10 +82,8 @@ public ListLevelsResponse listAllLoggersAndLevels() { return response; } - @Path("/levels") - @PUT + @Override @PermissionName(CONFIG_EDIT_PERM) - @Produces({"application/json", "application/xml", BINARY_CONTENT_TYPE_V2}) public LoggingResponse modifyLocalLogLevel(List requestBody) { ensureLogWatcherEnabled(); final LoggingResponse response = instantiateLoggingResponse(LoggingResponse.class); @@ -104,11 +98,9 @@ public LoggingResponse modifyLocalLogLevel(List requestBody) { return response; } - @Path("/messages") - @GET + @Override @PermissionName(CONFIG_READ_PERM) - @Produces({"application/json", "application/xml", BINARY_CONTENT_TYPE_V2}) - public LogMessagesResponse fetchLocalLogMessages(@QueryParam("since") Long boundingTimeMillis) { + public LogMessagesResponse fetchLocalLogMessages(Long boundingTimeMillis) { ensureLogWatcherEnabled(); final LogMessagesResponse response = instantiateLoggingResponse(LogMessagesResponse.class); if (boundingTimeMillis == null) { @@ -137,10 +129,8 @@ public LogMessagesResponse fetchLocalLogMessages(@QueryParam("since") Long bound return response; } - @Path("/messages/threshold") - @PUT + @Override @PermissionName(CONFIG_EDIT_PERM) - @Produces({"application/json", "application/xml", BINARY_CONTENT_TYPE_V2}) public LoggingResponse setMessageThreshold(SetThresholdRequestBody requestBody) { ensureLogWatcherEnabled(); final LoggingResponse response = instantiateLoggingResponse(LoggingResponse.class); @@ -175,99 +165,19 @@ private T instantiateLoggingResponse(Class clazz) return response; } - /** Generic logging response that includes the name of the log watcher (e.g. "Log4j2") */ - public static class LoggingResponse extends SolrJerseyResponse { - @JsonProperty("watcher") - public String watcherName; - } - - /** A user-requested modification in the level that a specified logger reports at. */ - public static class LogLevelChange implements JacksonReflectMapWriter { - public LogLevelChange() {} - - public LogLevelChange(String logger, String level) { - this.logger = logger; - this.level = level; - } + public static List parseLogLevelChanges(String[] rawChangeValues) { + final List changes = new ArrayList<>(); - @JsonProperty public String logger; - @JsonProperty public String level; - - public static List createRequestBodyFromV1Params(String[] rawChangeValues) { - final List changes = new ArrayList<>(); - - for (String rawChange : rawChangeValues) { - String[] split = rawChange.split(":"); - if (split.length != 2) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Invalid format, expected level:value, got " + rawChange); - } - changes.add(new LogLevelChange(split[0], split[1])); + for (String rawChange : rawChangeValues) { + String[] split = rawChange.split(":"); + if (split.length != 2) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Invalid format, expected level:value, got " + rawChange); } - - return changes; - } - } - - /** The request body for the 'PUT /api/node/logging/messages/threshold' API. */ - public static class SetThresholdRequestBody implements JacksonReflectMapWriter { - public SetThresholdRequestBody() {} - - public SetThresholdRequestBody(String level) { - this.level = level; - } - - @JsonProperty(required = true) - public String level; - } - - /** Response format for the 'GET /api/node/logging/messages' API. */ - public static class LogMessagesResponse extends LoggingResponse { - @JsonProperty public LogMessageInfo info; - - @JsonProperty("history") - public SolrDocumentList docs; - } - - /** Metadata about the log messages returned by the 'GET /api/node/logging/messages' API */ - public static class LogMessageInfo implements JacksonReflectMapWriter { - @JsonProperty("since") - public Long boundingTimeMillis; - - @JsonProperty public Boolean found; - @JsonProperty public List levels; - - @JsonProperty("last") - public long lastRecordTimestampMillis; - - @JsonProperty public int buffer; - @JsonProperty public String threshold; - } - - /** Response format for the 'GET /api/node/logging/levels' API. */ - public static class ListLevelsResponse extends LoggingResponse { - @JsonProperty public List levels; - @JsonProperty public List loggers; - } - - /** Representation of a single logger and its current state. */ - public static class LogLevelInfo implements JacksonReflectMapWriter { - public LogLevelInfo() {} - - public LogLevelInfo(String name, String level, boolean set) { - this.name = name; - this.level = level; - this.set = set; + changes.add(new LogLevelChange(split[0], split[1])); } - @JsonProperty("name") - public String name; - - @JsonProperty("level") - public String level; - - @JsonProperty("set") - public boolean set; + return changes; } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/ReplicationAPIBase.java b/solr/core/src/java/org/apache/solr/handler/admin/api/ReplicationAPIBase.java index f438ecc2ac7..638312e592d 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/ReplicationAPIBase.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/ReplicationAPIBase.java @@ -19,11 +19,24 @@ import static org.apache.solr.handler.ReplicationHandler.ERR_STATUS; import static org.apache.solr.handler.ReplicationHandler.OK_STATUS; +import jakarta.ws.rs.core.StreamingOutput; import java.io.IOException; +import java.io.OutputStream; import java.lang.invoke.MethodHandles; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Objects; +import java.util.regex.Pattern; +import java.util.zip.Adler32; +import java.util.zip.Checksum; +import java.util.zip.DeflaterOutputStream; +import org.apache.commons.io.output.CloseShieldOutputStream; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.SegmentCommitInfo; @@ -31,7 +44,13 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.RateLimiter; import org.apache.solr.api.JerseyResource; +import org.apache.solr.client.api.model.FileListResponse; +import org.apache.solr.client.api.model.FileMetaData; +import org.apache.solr.client.api.model.IndexVersionResponse; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.FastOutputStream; import org.apache.solr.core.DirectoryFactory; import org.apache.solr.core.IndexDeletionPolicyWrapper; import org.apache.solr.core.SolrCore; @@ -44,7 +63,24 @@ /** A common parent for "replication" (i.e. replication-level) APIs. */ public abstract class ReplicationAPIBase extends JerseyResource { + public static final String CONF_FILE_SHORT = "cf"; + public static final String TLOG_FILE = "tlogFile"; + public static final String FILE_STREAM = "filestream"; + public static final String STATUS = "status"; + public static final int PACKET_SZ = 1024 * 1024; // 1MB + public static final String GENERATION = "generation"; + public static final String OFFSET = "offset"; + public static final String LEN = "len"; + public static final String FILE = "file"; + public static final String MAX_WRITE_PER_SECOND = "maxWriteMBPerSec"; + public static final String CHECKSUM = "checksum"; + public static final String COMPRESSION = "compression"; + public static final String POLL_INTERVAL = "pollInterval"; + public static final String INTERVAL_ERR_MSG = + "The " + POLL_INTERVAL + " must be in this format 'HH:mm:ss'"; + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final Pattern INTERVAL_PATTERN = Pattern.compile("(\\d*?):(\\d*?):(\\d*)"); protected final SolrCore solrCore; protected final SolrQueryRequest solrQueryRequest; protected final SolrQueryResponse solrQueryResponse; @@ -56,23 +92,48 @@ public ReplicationAPIBase( this.solrQueryResponse = solrQueryResponse; } - protected CoreReplicationAPI.IndexVersionResponse doFetchIndexVersion() throws IOException { + protected IndexVersionResponse doFetchIndexVersion() throws IOException { ReplicationHandler replicationHandler = (ReplicationHandler) solrCore.getRequestHandler(ReplicationHandler.PATH); return replicationHandler.getIndexVersionResponse(); } - protected CoreReplicationAPI.FileListResponse doFetchFileList(long generation) { + protected FileListResponse doFetchFileList(long generation) { ReplicationHandler replicationHandler = (ReplicationHandler) solrCore.getRequestHandler(ReplicationHandler.PATH); return getFileList(generation, replicationHandler); } - protected CoreReplicationAPI.FileListResponse getFileList( - long generation, ReplicationHandler replicationHandler) { + protected DirectoryFileStream doFetchFile( + String filePath, + String dirType, + String offset, + String len, + boolean compression, + boolean checksum, + double maxWriteMBPerSec, + Long gen) { + DirectoryFileStream dfs; + if (Objects.equals(dirType, CONF_FILE_SHORT)) { + dfs = + new LocalFsConfFileStream( + filePath, dirType, offset, len, compression, checksum, maxWriteMBPerSec, gen); + } else if (Objects.equals(dirType, TLOG_FILE)) { + dfs = + new LocalFsTlogFileStream( + filePath, dirType, offset, len, compression, checksum, maxWriteMBPerSec, gen); + } else { + dfs = + new DirectoryFileStream( + filePath, dirType, offset, len, compression, checksum, maxWriteMBPerSec, gen); + } + solrQueryResponse.add(FILE_STREAM, dfs); + return dfs; + } + + protected FileListResponse getFileList(long generation, ReplicationHandler replicationHandler) { final IndexDeletionPolicyWrapper delPol = solrCore.getDeletionPolicy(); - final CoreReplicationAPI.FileListResponse filesResponse = - new CoreReplicationAPI.FileListResponse(); + final FileListResponse filesResponse = new FileListResponse(); IndexCommit commit = null; try { @@ -96,7 +157,7 @@ protected CoreReplicationAPI.FileListResponse getFileList( } assert null != commit; - List result = new ArrayList<>(); + List result = new ArrayList<>(); Directory dir = null; try { dir = @@ -109,7 +170,7 @@ protected CoreReplicationAPI.FileListResponse getFileList( SegmentInfos infos = SegmentInfos.readCommit(dir, commit.getSegmentsFileName()); for (SegmentCommitInfo commitInfo : infos) { for (String file : commitInfo.files()) { - CoreReplicationAPI.FileMetaData metaData = new CoreReplicationAPI.FileMetaData(); + FileMetaData metaData = new FileMetaData(); metaData.name = file; metaData.size = dir.fileLength(file); @@ -127,7 +188,7 @@ protected CoreReplicationAPI.FileListResponse getFileList( } // add the segments_N file - CoreReplicationAPI.FileMetaData fileMetaData = new CoreReplicationAPI.FileMetaData(); + FileMetaData fileMetaData = new FileMetaData(); fileMetaData.name = infos.getSegmentsFileName(); fileMetaData.size = dir.fileLength(infos.getSegmentsFileName()); if (infos.getId() != null) { @@ -185,8 +246,318 @@ protected CoreReplicationAPI.FileListResponse getFileList( return filesResponse; } + /** This class is used to read and send files in the lucene index */ + protected class DirectoryFileStream implements SolrCore.RawWriter, StreamingOutput { + protected FastOutputStream fos; + + protected Long indexGen; + protected IndexDeletionPolicyWrapper delPolicy; + + protected String fileName; + protected String cfileName; + protected String tlogFileName; + protected String sOffset; + protected String sLen; + protected final boolean compress; + protected boolean useChecksum; + + protected long offset = -1; + protected int len = -1; + + protected Checksum checksum; + + private RateLimiter rateLimiter; + + byte[] buf; + + public DirectoryFileStream( + String file, + String dirType, + String offset, + String len, + boolean compression, + boolean useChecksum, + double maxWriteMBPerSec, + Long gen) { + delPolicy = solrCore.getDeletionPolicy(); + + fileName = validateFilenameOrError(file); + + switch (dirType) { + case CONF_FILE_SHORT: + cfileName = file; + break; + case TLOG_FILE: + tlogFileName = file; + break; + default: + fileName = file; + break; + } + + this.sOffset = offset; + this.sLen = len; + this.compress = compression; + this.useChecksum = useChecksum; + this.indexGen = gen; + if (useChecksum) { + checksum = new Adler32(); + } + // No throttle if MAX_WRITE_PER_SECOND is not specified + if (maxWriteMBPerSec == 0) { + this.rateLimiter = new RateLimiter.SimpleRateLimiter(Double.MAX_VALUE); + } else { + this.rateLimiter = new RateLimiter.SimpleRateLimiter(maxWriteMBPerSec); + } + } + + // Throw exception on directory traversal attempts + protected String validateFilenameOrError(String fileName) { + if (fileName != null) { + Path filePath = Paths.get(fileName); + filePath.forEach( + subpath -> { + if ("..".equals(subpath.toString())) { + throw new SolrException( + SolrException.ErrorCode.FORBIDDEN, "File name cannot contain .."); + } + }); + if (filePath.isAbsolute()) { + throw new SolrException(SolrException.ErrorCode.FORBIDDEN, "File name must be relative"); + } + return fileName; + } else return null; + } + + protected void initWrite() throws IOException { + this.offset = (sOffset != null) ? Long.parseLong(sOffset) : -1; + this.len = (sLen != null) ? Integer.parseInt(sLen) : -1; + if (fileName == null && cfileName == null && tlogFileName == null) { + // no filename do nothing + writeNothingAndFlush(); + } + buf = new byte[(len == -1 || len > PACKET_SZ) ? PACKET_SZ : len]; + + // reserve commit point till write is complete + if (indexGen != null) { + delPolicy.saveCommitPoint(indexGen); + } + } + + protected void createOutputStream(OutputStream out) { + // DeflaterOutputStream requires a close call, but don't close the request outputstream + out = new CloseShieldOutputStream(out); + if (compress) { + fos = new FastOutputStream(new DeflaterOutputStream(out)); + } else { + fos = new FastOutputStream(out); + } + } + + protected void extendReserveAndReleaseCommitPoint() { + ReplicationHandler replicationHandler = + (ReplicationHandler) solrCore.getRequestHandler(ReplicationHandler.PATH); + + if (indexGen != null) { + // Reserve the commit point for another 10s for the next file to be to fetched. + // We need to keep extending the commit reservation between requests so that the replica can + // fetch all the files correctly. + delPolicy.setReserveDuration(indexGen, replicationHandler.getReserveCommitDuration()); + + // release the commit point as the write is complete + delPolicy.releaseCommitPoint(indexGen); + } + } + + @Override + public void write(OutputStream out) throws IOException { + createOutputStream(out); + + IndexInput in = null; + try { + initWrite(); + + Directory dir = solrCore.withSearcher(searcher -> searcher.getIndexReader().directory()); + in = dir.openInput(fileName, IOContext.READONCE); + // if offset is mentioned move the pointer to that point + if (offset != -1) in.seek(offset); + + long filelen = dir.fileLength(fileName); + long maxBytesBeforePause = 0; + + while (true) { + offset = offset == -1 ? 0 : offset; + int read = (int) Math.min(buf.length, filelen - offset); + in.readBytes(buf, 0, read); + + fos.writeInt(read); + if (useChecksum) { + checksum.reset(); + checksum.update(buf, 0, read); + fos.writeLong(checksum.getValue()); + } + fos.write(buf, 0, read); + fos.flush(); + log.debug("Wrote {} bytes for file {}", offset + read, fileName); // nowarn + + // Pause if necessary + maxBytesBeforePause += read; + if (maxBytesBeforePause >= rateLimiter.getMinPauseCheckBytes()) { + rateLimiter.pause(maxBytesBeforePause); + maxBytesBeforePause = 0; + } + if (read != buf.length) { + writeNothingAndFlush(); + // we close because DeflaterOutputStream requires a close call, but the request + // outputstream is protected + fos.close(); + break; + } + offset += read; + in.seek(offset); + } + } catch (IOException e) { + log.warn( + "Exception while writing response for params fileName={} cfileName={} tlogFileName={} offset={} len={} compression={} generation={} checksum={}", + fileName, + cfileName, + tlogFileName, + sOffset, + sLen, + compress, + indexGen, + useChecksum); + } finally { + if (in != null) { + in.close(); + } + extendReserveAndReleaseCommitPoint(); + } + } + + /** Used to write a marker for EOF */ + protected void writeNothingAndFlush() throws IOException { + fos.writeInt(0); + fos.flush(); + } + } + + /** This is used to write files in the conf directory. */ + protected abstract class LocalFsFileStream extends DirectoryFileStream { + + private Path file; + + public LocalFsFileStream( + String file, + String dirType, + String offset, + String len, + boolean compression, + boolean useChecksum, + double maxWriteMBPerSec, + Long gen) { + super(file, dirType, offset, len, compression, useChecksum, maxWriteMBPerSec, gen); + this.file = this.initFile(); + } + + protected abstract Path initFile(); + + @Override + public void write(OutputStream out) throws IOException { + createOutputStream(out); + try { + initWrite(); + + if (Files.isReadable(file)) { + try (SeekableByteChannel channel = Files.newByteChannel(file)) { + // if offset is mentioned move the pointer to that point + if (offset != -1) channel.position(offset); + ByteBuffer bb = ByteBuffer.wrap(buf); + + while (true) { + bb.clear(); + long bytesRead = channel.read(bb); + if (bytesRead <= 0) { + writeNothingAndFlush(); + // we close because DeflaterOutputStream requires a close call, but the request + // outputstream is protected + fos.close(); + break; + } + fos.writeInt((int) bytesRead); + if (useChecksum) { + checksum.reset(); + checksum.update(buf, 0, (int) bytesRead); + fos.writeLong(checksum.getValue()); + } + fos.write(buf, 0, (int) bytesRead); + fos.flush(); + } + } + } else { + writeNothingAndFlush(); + } + } catch (IOException e) { + log.warn( + "Exception while writing response for params fileName={} cfileName={} tlogFileName={} offset={} len={} compression={} generation={} checksum={}", + fileName, + cfileName, + tlogFileName, + sOffset, + sLen, + compress, + indexGen, + useChecksum); + } finally { + extendReserveAndReleaseCommitPoint(); + } + } + } + + protected class LocalFsTlogFileStream extends LocalFsFileStream { + + public LocalFsTlogFileStream( + String file, + String dirType, + String offset, + String len, + boolean compression, + boolean useChecksum, + double maxWriteMBPerSec, + Long gen) { + super(file, dirType, offset, len, compression, useChecksum, maxWriteMBPerSec, gen); + } + + @Override + protected Path initFile() { + // if it is a tlog file read from tlog directory + return Path.of(solrCore.getUpdateHandler().getUpdateLog().getTlogDir(), tlogFileName); + } + } + + protected class LocalFsConfFileStream extends LocalFsFileStream { + + public LocalFsConfFileStream( + String file, + String dirType, + String offset, + String len, + boolean compression, + boolean useChecksum, + double maxWriteMBPerSec, + Long gen) { + super(file, dirType, offset, len, compression, useChecksum, maxWriteMBPerSec, gen); + } + + @Override + protected Path initFile() { + // if it is a conf file read from config directory + return solrCore.getResourceLoader().getConfigPath().resolve(cfileName); + } + } + private void reportErrorOnResponse( - CoreReplicationAPI.FileListResponse fileListResponse, String message, Exception e) { + FileListResponse fileListResponse, String message, Exception e) { fileListResponse.status = ERR_STATUS; fileListResponse.message = message; if (e != null) { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollectionAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollection.java similarity index 76% rename from solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollectionAPI.java rename to solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollection.java index ff8d9ac9690..6d2a9eb25a3 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollectionAPI.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollection.java @@ -17,8 +17,6 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE_V2; -import static org.apache.solr.client.solrj.request.beans.V2ApiConstants.CREATE_COLLECTION_KEY; import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION; import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP; import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF; @@ -35,18 +33,13 @@ import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT; import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM; -import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.inject.Inject; -import jakarta.ws.rs.POST; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.PathParam; -import jakarta.ws.rs.Produces; -import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.solr.client.api.model.CreateCollectionRequestBody; +import org.apache.solr.client.api.endpoint.CollectionBackupApi; +import org.apache.solr.client.api.model.RestoreCollectionRequestBody; import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.api.model.SubResponseAccumulatingJerseyResponse; import org.apache.solr.client.solrj.SolrResponse; @@ -56,10 +49,10 @@ import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.params.CollectionParams; import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.handler.configsets.ConfigSetAPIBase; -import org.apache.solr.jersey.JacksonReflectMapWriter; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -69,8 +62,7 @@ * *

    This API is analogous to the v1 /admin/collections?action=RESTORE command. */ -@Path("/backups/{backupName}/restore") -public class RestoreCollectionAPI extends BackupAPIBase { +public class RestoreCollection extends BackupAPIBase implements CollectionBackupApi.Restore { private static final Set CREATE_PARAM_ALLOWLIST = Stream.concat( @@ -83,19 +75,17 @@ public class RestoreCollectionAPI extends BackupAPIBase { .collect(Collectors.toUnmodifiableSet()); @Inject - public RestoreCollectionAPI( + public RestoreCollection( CoreContainer coreContainer, SolrQueryRequest solrQueryRequest, SolrQueryResponse solrQueryResponse) { super(coreContainer, solrQueryRequest, solrQueryResponse); } - @POST - @Produces({"application/json", "application/xml", BINARY_CONTENT_TYPE_V2}) + @Override @PermissionName(COLL_EDIT_PERM) public SubResponseAccumulatingJerseyResponse restoreCollection( - @PathParam("backupName") String backupName, RestoreCollectionRequestBody requestBody) - throws Exception { + String backupName, RestoreCollectionRequestBody requestBody) throws Exception { final var response = instantiateJerseyResponse(SubResponseAccumulatingJerseyResponse.class); if (requestBody == null) { @@ -159,7 +149,7 @@ public SubResponseAccumulatingJerseyResponse restoreCollection( public ZkNodeProps createRemoteMessage( String backupName, RestoreCollectionRequestBody requestBody) { - final Map remoteMessage = requestBody.toMap(new HashMap<>()); + final Map remoteMessage = Utils.reflectToMap(requestBody); // If the RESTORE is setup to create a new collection, copy those parameters first final var createReqBody = requestBody.createCollectionParams; @@ -177,12 +167,7 @@ public ZkNodeProps createRemoteMessage( // Copy restore-specific parameters remoteMessage.put(QUEUE_OPERATION, CollectionParams.CollectionAction.RESTORE.toLower()); - remoteMessage.put(COLLECTION_PROP, requestBody.collection); remoteMessage.put(NAME, backupName); - remoteMessage.put(BACKUP_LOCATION, requestBody.location); - if (requestBody.backupId != null) remoteMessage.put(BACKUP_ID, requestBody.backupId); - if (requestBody.repository != null) - remoteMessage.put(BACKUP_REPOSITORY, requestBody.repository); remoteMessage.put( TRUSTED, ConfigSetAPIBase.isTrusted( @@ -198,39 +183,24 @@ public static SolrJerseyResponse invokeFromV1Params( final var params = solrQueryRequest.getParams(); params.required().check(NAME, COLLECTION_PROP); final String backupName = params.get(NAME); - final var requestBody = RestoreCollectionRequestBody.fromV1Params(params); + final var requestBody = createRequestBodyFromV1Params(params); final var restoreApi = - new RestoreCollectionAPI(coreContainer, solrQueryRequest, solrQueryResponse); + new RestoreCollection(coreContainer, solrQueryRequest, solrQueryResponse); return restoreApi.restoreCollection(backupName, requestBody); } - /** Request body for the v2 "restore collection" API. */ - public static class RestoreCollectionRequestBody implements JacksonReflectMapWriter { - @JsonProperty(required = true) - public String collection; - - @JsonProperty public String location; - @JsonProperty public String repository; - @JsonProperty public Integer backupId; - - @JsonProperty(CREATE_COLLECTION_KEY) - public CreateCollectionRequestBody createCollectionParams; + public static RestoreCollectionRequestBody createRequestBodyFromV1Params(SolrParams solrParams) { + final var restoreBody = new RestoreCollectionRequestBody(); + restoreBody.collection = solrParams.get(COLLECTION_PROP); + restoreBody.location = solrParams.get(BACKUP_LOCATION); + restoreBody.repository = solrParams.get(BACKUP_REPOSITORY); + restoreBody.backupId = solrParams.getInt(BACKUP_ID); + restoreBody.async = solrParams.get(ASYNC); - @JsonProperty public String async; + restoreBody.createCollectionParams = + CreateCollection.createRequestBodyFromV1Params(solrParams, false); - public static RestoreCollectionRequestBody fromV1Params(SolrParams solrParams) { - final var restoreBody = new RestoreCollectionRequestBody(); - restoreBody.collection = solrParams.get(COLLECTION_PROP); - restoreBody.location = solrParams.get(BACKUP_LOCATION); - restoreBody.repository = solrParams.get(BACKUP_REPOSITORY); - restoreBody.backupId = solrParams.getInt(BACKUP_ID); - restoreBody.async = solrParams.get(ASYNC); - - restoreBody.createCollectionParams = - CreateCollection.createRequestBodyFromV1Params(solrParams, false); - - return restoreBody; - } + return restoreBody; } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/SnapshotBackupAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/SnapshotBackupAPI.java index e4024b119c5..d2fbabc04c0 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/SnapshotBackupAPI.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/SnapshotBackupAPI.java @@ -16,37 +16,33 @@ */ package org.apache.solr.handler.admin.api; -import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE_V2; import static org.apache.solr.handler.ReplicationHandler.ERR_STATUS; import static org.apache.solr.security.PermissionNameProvider.Name.CORE_EDIT_PERM; -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.parameters.RequestBody; import jakarta.inject.Inject; -import jakarta.ws.rs.POST; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.Produces; -import jakarta.ws.rs.core.MediaType; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.function.Consumer; import org.apache.solr.api.JerseyResource; -import org.apache.solr.client.api.model.SolrJerseyResponse; +import org.apache.solr.client.api.endpoint.ReplicationBackupApis; +import org.apache.solr.client.api.model.ReplicationBackupRequestBody; +import org.apache.solr.client.api.model.ReplicationBackupResponse; import org.apache.solr.common.SolrException; -import org.apache.solr.common.annotation.JsonProperty; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; import org.apache.solr.handler.ReplicationHandler; import org.apache.solr.handler.ReplicationHandler.ReplicationHandlerConfig; -import org.apache.solr.jersey.JacksonReflectMapWriter; import org.apache.solr.jersey.PermissionName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** V2 endpoint for Backup API used for User-Managed clusters and Single-Node Installation. */ -@Path("/cores/{coreName}/replication/backups") -public class SnapshotBackupAPI extends JerseyResource { +/** + * v2 API implementation for replication-handler based backup creation. + * + *

    This is the main backup functionality available to 'standalone' users. + */ +public class SnapshotBackupAPI extends JerseyResource implements ReplicationBackupApis { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private final SolrCore solrCore; @@ -62,22 +58,17 @@ public SnapshotBackupAPI(SolrCore solrCore, ReplicationHandlerConfig replication * This API (POST /api/cores/coreName/replication/backups {...}) is analogous to the v1 * /solr/coreName/replication?command=backup */ - @POST - @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, BINARY_CONTENT_TYPE_V2}) - @Operation(summary = "Backup command using ReplicationHandler") + @Override @PermissionName(CORE_EDIT_PERM) - public BackupReplicationResponse createBackup( - @RequestBody BackupReplicationRequestBody backupReplicationPayload) throws Exception { + public ReplicationBackupResponse createBackup( + @RequestBody ReplicationBackupRequestBody backupReplicationPayload) { ensureRequiredRequestBodyProvided(backupReplicationPayload); - ReplicationHandler replicationHandler = - (ReplicationHandler) solrCore.getRequestHandler(ReplicationHandler.PATH); - return doBackup(replicationHandler, backupReplicationPayload); + return doBackup(backupReplicationPayload); } - private BackupReplicationResponse doBackup( - ReplicationHandler replicationHandler, - BackupReplicationRequestBody backupReplicationPayload) { - BackupReplicationResponse response = instantiateJerseyResponse(BackupReplicationResponse.class); + private ReplicationBackupResponse doBackup( + ReplicationBackupRequestBody backupReplicationPayload) { + ReplicationBackupResponse response = instantiateJerseyResponse(ReplicationBackupResponse.class); int numberToKeep = backupReplicationPayload.numberToKeep; int numberBackupsToKeep = replicationHandlerConfig.getNumberBackupsToKeep(); String location = backupReplicationPayload.location; @@ -128,63 +119,8 @@ protected void doSnapShoot( resultConsumer); } - /* POJO for v2 endpoints request body. */ - public static class BackupReplicationRequestBody implements JacksonReflectMapWriter { - - public BackupReplicationRequestBody() {} - - public BackupReplicationRequestBody( - String location, String name, int numberToKeep, String repository, String commitName) { - this.location = location; - this.name = name; - this.numberToKeep = numberToKeep; - this.repository = repository; - this.commitName = commitName; - } - - @Schema(description = "The path where the backup will be created") - @JsonProperty - public String location; - - @Schema(description = "The backup will be created in a directory called snapshot.") - @JsonProperty - public String name; - - @Schema(description = "The number of backups to keep.") - @JsonProperty - public int numberToKeep; - - @Schema(description = "The name of the repository to be used for e backup.") - @JsonProperty - public String repository; - - @Schema( - description = - "The name of the commit which was used while taking a snapshot using the CREATESNAPSHOT command.") - @JsonProperty - public String commitName; - } - - /** Response for {@link SnapshotBackupAPI#createBackup(BackupReplicationRequestBody)}. */ - public static class BackupReplicationResponse extends SolrJerseyResponse { - - @JsonProperty("result") - public NamedList result; - - @JsonProperty("status") - public String status; - - @JsonProperty("message") - public String message; - - @JsonProperty("exception") - public Exception exception; - - public BackupReplicationResponse() {} - } - private static void reportErrorOnResponse( - BackupReplicationResponse response, String message, Exception e) { + ReplicationBackupResponse response, String message, Exception e) { response.status = ERR_STATUS; response.message = message; if (e != null) { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/V2NodeLoggingAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/V2NodeLoggingAPI.java deleted file mode 100644 index a2b97609d0a..00000000000 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/V2NodeLoggingAPI.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.handler.admin.api; - -import static org.apache.solr.security.PermissionNameProvider.Name.CONFIG_EDIT_PERM; - -import jakarta.inject.Inject; -import jakarta.ws.rs.GET; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.Produces; -import jakarta.ws.rs.core.MediaType; -import java.util.List; -import org.apache.solr.jersey.PermissionName; -import org.apache.solr.logging.LogWatcher; - -/** - * V2 API for getting log levels on an individual node. - * - *

    The endpoint is defined as api/node/logging/levels - */ -@Path("/api/node/logging") -public class V2NodeLoggingAPI { - private final LogWatcher watcher; - - @Inject - public V2NodeLoggingAPI(LogWatcher watcher) { - this.watcher = watcher; - } - - @GET - @Path("/levels") - @PermissionName(CONFIG_EDIT_PERM) - @Produces(MediaType.APPLICATION_JSON) - public List getAllLevels() { - return this.watcher.getAllLevels(); - } -} diff --git a/solr/core/src/java/org/apache/solr/handler/api/V2ApiUtils.java b/solr/core/src/java/org/apache/solr/handler/api/V2ApiUtils.java index e2a907d07d4..9a96b34afc0 100644 --- a/solr/core/src/java/org/apache/solr/handler/api/V2ApiUtils.java +++ b/solr/core/src/java/org/apache/solr/handler/api/V2ApiUtils.java @@ -19,7 +19,7 @@ import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE_V2; import static org.apache.solr.common.params.CommonParams.WT; -import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM; +import static org.apache.solr.handler.admin.api.ReplicationAPIBase.FILE_STREAM; import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java b/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java index 001b6c3338a..0adcdb93b1f 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java @@ -338,9 +338,8 @@ protected Object merge(Object source, Object dest, Set exclude) { } } - if (source instanceof NamedList && dest instanceof NamedList) { + if (source instanceof NamedList sl && dest instanceof NamedList) { NamedList tmp = new NamedList<>(); - NamedList sl = (NamedList) source; @SuppressWarnings("unchecked") NamedList dl = (NamedList) dest; for (int i = 0; i < sl.size(); i++) { diff --git a/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java b/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java index 45a498bdffa..e998411ed4a 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java @@ -144,9 +144,7 @@ public void process(ResponseBuilder rb) throws IOException { if (filters != null) { int cost = Integer.MAX_VALUE; for (Query q : filters) { - if (q instanceof CollapsingQParserPlugin.CollapsingPostFilter) { - CollapsingQParserPlugin.CollapsingPostFilter cp = - (CollapsingQParserPlugin.CollapsingPostFilter) q; + if (q instanceof CollapsingQParserPlugin.CollapsingPostFilter cp) { // if there are multiple collapse pick the low cost one // if cost are equal then first one is picked if (cp.getCost() < cost) { diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java index 219197d8cd8..320a5fe70d7 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java @@ -31,6 +31,7 @@ import net.jcip.annotations.NotThreadSafe; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrResponse; +import org.apache.solr.client.solrj.impl.Http2SolrClient; import org.apache.solr.client.solrj.impl.LBHttp2SolrClient; import org.apache.solr.client.solrj.impl.LBSolrClient; import org.apache.solr.client.solrj.request.QueryRequest; @@ -113,7 +114,7 @@ public class HttpShardHandler extends ShardHandler { protected AtomicInteger pending; private final Map> shardToURLs; - protected LBHttp2SolrClient lbClient; + protected LBHttp2SolrClient lbClient; public HttpShardHandler(HttpShardHandlerFactory httpShardHandlerFactory) { this.httpShardHandlerFactory = httpShardHandlerFactory; diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java index 2bfc4cb236a..1437dee63ea 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java +++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandlerFactory.java @@ -83,8 +83,9 @@ public class HttpShardHandlerFactory extends ShardHandlerFactory protected ExecutorService commExecutor; protected volatile Http2SolrClient defaultClient; + protected Http2SolrClient.Builder httpSolrClientBuilder; protected InstrumentedHttpListenerFactory httpListenerFactory; - protected LBHttp2SolrClient loadbalancer; + protected LBHttp2SolrClient loadbalancer; int corePoolSize = 0; int maximumPoolSize = Integer.MAX_VALUE; @@ -305,16 +306,16 @@ public void init(PluginInfo info) { sb); int soTimeout = getParameter(args, HttpClientUtil.PROP_SO_TIMEOUT, HttpClientUtil.DEFAULT_SO_TIMEOUT, sb); - - this.defaultClient = + this.httpSolrClientBuilder = new Http2SolrClient.Builder() .withConnectionTimeout(connectionTimeout, TimeUnit.MILLISECONDS) .withIdleTimeout(soTimeout, TimeUnit.MILLISECONDS) .withExecutor(commExecutor) .withMaxConnectionsPerHost(maxConnectionsPerHost) - .build(); - this.defaultClient.addListenerFactory(this.httpListenerFactory); - this.loadbalancer = new LBHttp2SolrClient.Builder(defaultClient).build(); + .addListenerFactory(this.httpListenerFactory); + this.defaultClient = httpSolrClientBuilder.build(); + + this.loadbalancer = new LBHttp2SolrClient.Builder<>(httpSolrClientBuilder).build(); initReplicaListTransformers(getParameter(args, "replicaRouting", null, sb)); @@ -324,7 +325,7 @@ public void init(PluginInfo info) { @Override public void setSecurityBuilder(HttpClientBuilderPlugin clientBuilderPlugin) { if (clientBuilderPlugin != null) { - clientBuilderPlugin.setup(defaultClient); + clientBuilderPlugin.setup(httpSolrClientBuilder, defaultClient); } } diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java index 3acb78ed78f..0552c6a0c4e 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java @@ -195,8 +195,7 @@ public void prepare(ResponseBuilder rb) throws IOException { if (rankQueryString != null) { QParser rqparser = QParser.getParser(rankQueryString, req); Query rq = rqparser.getQuery(); - if (rq instanceof RankQuery) { - RankQuery rankQuery = (RankQuery) rq; + if (rq instanceof RankQuery rankQuery) { rb.setRankQuery(rankQuery); MergeStrategy mergeStrategy = rankQuery.getMergeStrategy(); if (mergeStrategy != null) { @@ -615,17 +614,23 @@ protected int groupedDistributedProcess(ResponseBuilder rb) { } protected int regularDistributedProcess(ResponseBuilder rb) { - if (rb.stage < ResponseBuilder.STAGE_PARSE_QUERY) return ResponseBuilder.STAGE_PARSE_QUERY; + if (rb.stage < ResponseBuilder.STAGE_PARSE_QUERY) { + return ResponseBuilder.STAGE_PARSE_QUERY; + } if (rb.stage == ResponseBuilder.STAGE_PARSE_QUERY) { createDistributedStats(rb); return ResponseBuilder.STAGE_EXECUTE_QUERY; } - if (rb.stage < ResponseBuilder.STAGE_EXECUTE_QUERY) return ResponseBuilder.STAGE_EXECUTE_QUERY; + if (rb.stage < ResponseBuilder.STAGE_EXECUTE_QUERY) { + return ResponseBuilder.STAGE_EXECUTE_QUERY; + } if (rb.stage == ResponseBuilder.STAGE_EXECUTE_QUERY) { createMainQuery(rb); return ResponseBuilder.STAGE_GET_FIELDS; } - if (rb.stage < ResponseBuilder.STAGE_GET_FIELDS) return ResponseBuilder.STAGE_GET_FIELDS; + if (rb.stage < ResponseBuilder.STAGE_GET_FIELDS) { + return ResponseBuilder.STAGE_GET_FIELDS; + } if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS && !rb.onePassDistributedQuery) { createRetrieveDocs(rb); return ResponseBuilder.STAGE_DONE; diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java index c7bef72294c..155efd7674a 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java @@ -1117,10 +1117,9 @@ protected ElevatingQuery(String queryString, boolean subsetMatch) { @Override public boolean equals(Object o) { - if (!(o instanceof ElevatingQuery)) { + if (!(o instanceof ElevatingQuery eq)) { return false; } - ElevatingQuery eq = (ElevatingQuery) o; return queryString.equals(eq.queryString) && subsetMatch == eq.subsetMatch; } diff --git a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java index f1fc77d9ceb..6118b21a086 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java @@ -879,8 +879,7 @@ private static SolrInputDocument toSolrInputDocument(SolrDocument doc, IndexSche if ((!sf.hasDocValues() && !sf.stored()) || schema.isCopyFieldTarget(sf)) continue; } for (Object val : doc.getFieldValues(fname)) { - if (val instanceof IndexableField) { - IndexableField f = (IndexableField) val; + if (val instanceof IndexableField f) { // materialize: if (sf != null) { val = sf.getType().toObject(f); // object or external string? diff --git a/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java b/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java index 55addf143f8..cf513bb63c7 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java @@ -122,13 +122,13 @@ public ResponseBuilder( * public static final String LOCAL_SHARD = "local"; public static final String DOC_QUERY = "dq"; * * */ - public static int STAGE_START = 0; + public static final int STAGE_START = 0; - public static int STAGE_PARSE_QUERY = 1000; - public static int STAGE_TOP_GROUPS = 1500; - public static int STAGE_EXECUTE_QUERY = 2000; - public static int STAGE_GET_FIELDS = 3000; - public static int STAGE_DONE = Integer.MAX_VALUE; + public static final int STAGE_PARSE_QUERY = 1000; + public static final int STAGE_TOP_GROUPS = 1500; + public static final int STAGE_EXECUTE_QUERY = 2000; + public static final int STAGE_GET_FIELDS = 3000; + public static final int STAGE_DONE = Integer.MAX_VALUE; public int stage; // What stage is this current request at? diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java index 2c1503a907f..2d6b36ac3cb 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java @@ -18,8 +18,17 @@ import static org.apache.solr.common.params.CommonParams.DISTRIB; import static org.apache.solr.common.params.CommonParams.PATH; +import static org.apache.solr.handler.component.ResponseBuilder.STAGE_DONE; +import static org.apache.solr.handler.component.ResponseBuilder.STAGE_EXECUTE_QUERY; +import static org.apache.solr.handler.component.ResponseBuilder.STAGE_GET_FIELDS; +import static org.apache.solr.handler.component.ResponseBuilder.STAGE_PARSE_QUERY; +import static org.apache.solr.handler.component.ResponseBuilder.STAGE_START; +import static org.apache.solr.handler.component.ResponseBuilder.STAGE_TOP_GROUPS; +import static org.apache.solr.request.SolrRequestInfo.getQueryLimits; +import static org.apache.solr.response.SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_DETAILS_KEY; import com.codahale.metrics.Counter; +import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; import java.lang.invoke.MethodHandles; @@ -31,6 +40,8 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; import org.apache.lucene.index.ExitableDirectoryReader; import org.apache.lucene.search.TotalHits; import org.apache.solr.client.solrj.SolrRequest.SolrRequestType; @@ -411,24 +422,58 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw return; // Circuit breaker tripped, return immediately } - // creates a ShardHandler object only if it's needed - final ShardHandler shardHandler1 = getAndPrepShardHandler(req, rb); + processComponents(req, rsp, rb, timer, components); - if (timer == null) { - // non-debugging prepare phase - for (SearchComponent c : components) { - c.prepare(rb); - } - } else { - // debugging prepare phase - RTimerTree subt = timer.sub("prepare"); - for (SearchComponent c : components) { - rb.setTimer(subt.sub(c.getName())); - c.prepare(rb); - rb.getTimer().stop(); + // SOLR-5550: still provide shards.info if requested even for a short-circuited distrib request + if (!rb.isDistrib + && req.getParams().getBool(ShardParams.SHARDS_INFO, false) + && rb.shortCircuitedURL != null) { + NamedList shardInfo = new SimpleOrderedMap<>(); + SimpleOrderedMap nl = new SimpleOrderedMap<>(); + if (rsp.getException() != null) { + Throwable cause = rsp.getException(); + if (cause instanceof SolrServerException) { + cause = ((SolrServerException) cause).getRootCause(); + } else { + if (cause.getCause() != null) { + cause = cause.getCause(); + } + } + nl.add("error", cause.toString()); + if (!core.getCoreContainer().hideStackTrace()) { + StringWriter trace = new StringWriter(); + cause.printStackTrace(new PrintWriter(trace)); + nl.add("trace", trace.toString()); + } + } else if (rb.getResults() != null) { + nl.add("numFound", rb.getResults().docList.matches()); + nl.add( + "numFoundExact", + rb.getResults().docList.hitCountRelation() == TotalHits.Relation.EQUAL_TO); + nl.add("maxScore", rb.getResults().docList.maxScore()); } - subt.stop(); + nl.add("shardAddress", rb.shortCircuitedURL); + nl.add("time", req.getRequestTimer().getTime()); // elapsed time of this request so far + + int pos = rb.shortCircuitedURL.indexOf("://"); + String shardInfoName = + pos != -1 ? rb.shortCircuitedURL.substring(pos + 3) : rb.shortCircuitedURL; + shardInfo.add(shardInfoName, nl); + rsp.getValues().add(ShardParams.SHARDS_INFO, shardInfo); } + } + + private void processComponents( + SolrQueryRequest req, + SolrQueryResponse rsp, + ResponseBuilder rb, + RTimerTree timer, + List components) + throws IOException { + // creates a ShardHandler object only if it's needed + final ShardHandler shardHandler1 = getAndPrepShardHandler(req, rb); + + if (!prepareComponents(req, rb, timer, components)) return; { // Once all of our components have been prepared, check if this request involves a SortSpec. // If it does, and if our request includes a cursorMark param, then parse & init the @@ -445,19 +490,26 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw if (!rb.isDistrib) { // a normal non-distributed request - try { // The semantics of debugging vs not debugging are different enough that // it makes sense to have two control loops if (!rb.isDebug()) { // Process for (SearchComponent c : components) { + if (checkLimitsBefore(c, "process", rb.req, rb.rsp, components)) { + shortCircuitedResults(req, rb); + return; + } c.process(rb); } } else { // Process RTimerTree subt = timer.sub("process"); for (SearchComponent c : components) { + if (checkLimitsBefore(c, "process debug", rb.req, rb.rsp, components)) { + shortCircuitedResults(req, rb); + return; + } rb.setTimer(subt.sub(c.getName())); c.process(rb); rb.getTimer().stop(); @@ -471,22 +523,7 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw } } catch (ExitableDirectoryReader.ExitingReaderException ex) { log.warn("Query: {}; ", req.getParamString(), ex); - if (rb.rsp.getResponse() == null) { - rb.rsp.addResponse(new SolrDocumentList()); - - // If a cursorMark was passed, and we didn't progress, set - // the nextCursorMark to the same position - String cursorStr = rb.req.getParams().get(CursorMarkParams.CURSOR_MARK_PARAM); - if (null != cursorStr) { - rb.rsp.add(CursorMarkParams.CURSOR_MARK_NEXT, cursorStr); - } - } - if (rb.isDebug()) { - NamedList debug = new NamedList<>(); - debug.add("explain", new NamedList<>()); - rb.rsp.add("debug", debug); - } - rb.rsp.setPartialResults(rb.req); + shortCircuitedResults(req, rb); } } else { // a distributed request @@ -504,7 +541,10 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw // call all components for (SearchComponent c : components) { - // the next stage is the minimum of what all components report + if (checkLimitsBefore(c, "distrib", rb.req, rb.rsp, components)) { + shortCircuitedResults(req, rb); + return; + } // the next stage is the minimum of what all components report nextStage = Math.min(nextStage, c.distributedProcess(rb)); } @@ -558,6 +598,8 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw ? shardHandler1.takeCompletedIncludingErrors() : shardHandler1.takeCompletedOrError(); if (srsp == null) break; // no more requests to wait for + AtomicReference detailMesg = + new AtomicReference<>(); // or perhaps new Object[1] ? boolean anyResponsesPartial = srsp.getShardRequest().responses.stream() @@ -568,9 +610,20 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw return false; } Object recursive = resp.findRecursive("responseHeader", "partialResults"); + if (recursive != null) { + Object message = + "[Shard:" + + response.getShardAddress() + + "]" + + resp.findRecursive( + "responseHeader", + RESPONSE_HEADER_PARTIAL_RESULTS_DETAILS_KEY); + detailMesg.compareAndSet(null, message); // first one, ingore rest + } return recursive != null; }); if (anyResponsesPartial) { + rb.rsp.addPartialResponseDetail(detailMesg.get()); rsp.setPartialResults(rb.req); } // Was there an exception? @@ -594,6 +647,11 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw throwSolrException(srsp.getException()); } else { rsp.setPartialResults(rb.req); + if (publishCpuTime) { + totalShardCpuTime += computeShardCpuTime(srsp.getShardRequest().responses); + rsp.getResponseHeader().add(ThreadCpuTimer.CPU_TIME, totalShardCpuTime); + rsp.addToLog(ThreadCpuTimer.CPU_TIME, totalShardCpuTime); + } } } } @@ -602,6 +660,15 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw // let the components see the responses to the request for (SearchComponent c : components) { + if (checkLimitsBefore( + c, + "handleResponses next stage:" + stageInEnglish(nextStage), + rb.req, + rb.rsp, + components)) { + shortCircuitedResults(req, rb); + return; + } c.handleResponses(rb, srsp.getShardRequest()); } @@ -613,6 +680,10 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw } for (SearchComponent c : components) { + if (checkLimitsBefore( + c, "finishStage stage:" + stageInEnglish(nextStage), rb.req, rb.rsp, components)) { + return; + } c.finishStage(rb); } @@ -624,44 +695,97 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw rsp.addToLog(ThreadCpuTimer.CPU_TIME, totalShardCpuTime); } } + } - // SOLR-5550: still provide shards.info if requested even for a short circuited distrib request - if (!rb.isDistrib - && req.getParams().getBool(ShardParams.SHARDS_INFO, false) - && rb.shortCircuitedURL != null) { - NamedList shardInfo = new SimpleOrderedMap<>(); - SimpleOrderedMap nl = new SimpleOrderedMap<>(); - if (rsp.getException() != null) { - Throwable cause = rsp.getException(); - if (cause instanceof SolrServerException) { - cause = ((SolrServerException) cause).getRootCause(); - } else { - if (cause.getCause() != null) { - cause = cause.getCause(); - } + private static boolean prepareComponents( + SolrQueryRequest req, ResponseBuilder rb, RTimerTree timer, List components) + throws IOException { + if (timer == null) { + // non-debugging prepare phase + for (SearchComponent component : components) { + if (checkLimitsBefore(component, "prepare", rb.req, rb.rsp, components)) { + shortCircuitedResults(req, rb); + return false; } - nl.add("error", cause.toString()); - if (!core.getCoreContainer().hideStackTrace()) { - StringWriter trace = new StringWriter(); - cause.printStackTrace(new PrintWriter(trace)); - nl.add("trace", trace.toString()); + component.prepare(rb); + } + } else { + // debugging prepare phase + RTimerTree subt = timer.sub("prepare"); + for (SearchComponent c : components) { + if (checkLimitsBefore(c, "prepare debug", rb.req, rb.rsp, components)) { + shortCircuitedResults(req, rb); + return false; } - } else if (rb.getResults() != null) { - nl.add("numFound", rb.getResults().docList.matches()); - nl.add( - "numFoundExact", - rb.getResults().docList.hitCountRelation() == TotalHits.Relation.EQUAL_TO); - nl.add("maxScore", rb.getResults().docList.maxScore()); + rb.setTimer(subt.sub(c.getName())); + c.prepare(rb); + rb.getTimer().stop(); } - nl.add("shardAddress", rb.shortCircuitedURL); - nl.add("time", req.getRequestTimer().getTime()); // elapsed time of this request so far + subt.stop(); + } + return true; + } - int pos = rb.shortCircuitedURL.indexOf("://"); - String shardInfoName = - pos != -1 ? rb.shortCircuitedURL.substring(pos + 3) : rb.shortCircuitedURL; - shardInfo.add(shardInfoName, nl); - rsp.getValues().add(ShardParams.SHARDS_INFO, shardInfo); + private static String stageInEnglish(int nextStage) { + // This should probably be a enum, but that change should be its own ticket. + switch (nextStage) { + case STAGE_START: + return "START"; + case STAGE_PARSE_QUERY: + return "PARSE_QUERY"; + case STAGE_TOP_GROUPS: + return "TOP_GROUPS"; + case STAGE_EXECUTE_QUERY: + return "EXECUTE_QUERY"; + case STAGE_GET_FIELDS: + return "GET_FIELDS"; + // nobody wants to think it was DONE and canceled after it completed... + case STAGE_DONE: + return "FINISHING"; + default: + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, "Unrecognized stage:" + nextStage); + } + } + + private static void shortCircuitedResults(SolrQueryRequest req, ResponseBuilder rb) { + + if (rb.rsp.getResponse() == null) { + rb.rsp.addResponse(new SolrDocumentList()); + + // If a cursorMark was passed, and we didn't progress, set + // the nextCursorMark to the same position + String cursorStr = rb.req.getParams().get(CursorMarkParams.CURSOR_MARK_PARAM); + if (null != cursorStr) { + rb.rsp.add(CursorMarkParams.CURSOR_MARK_NEXT, cursorStr); + } } + if (rb.isDebug()) { + NamedList debug = new NamedList<>(); + debug.add("explain", new NamedList<>()); + rb.rsp.add("debug", debug); + } + rb.rsp.setPartialResults(rb.req); + } + + private static boolean checkLimitsBefore( + SearchComponent c, + String when, + SolrQueryRequest req, + SolrQueryResponse resp, + List components) { + + return getQueryLimits(req, resp) + .maybeExitWithPartialResults( + () -> + "[" + + when + + "] Limit(s) exceeded prior to " + + c.getName() + + " in " + + components.stream() + .map(SearchComponent::getName) + .collect(Collectors.toList())); } private long computeShardCpuTime(List responses) { diff --git a/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java b/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java index ad204a0904b..0b6414834f2 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java @@ -61,9 +61,8 @@ public ShardDoc() { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof ShardDoc)) return false; + if (!(o instanceof ShardDoc shardDoc)) return false; - ShardDoc shardDoc = (ShardDoc) o; return Objects.equals(id, shardDoc.id); } diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsField.java b/solr/core/src/java/org/apache/solr/handler/component/StatsField.java index e74e00f015b..6dbeea65dd7 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/StatsField.java +++ b/solr/core/src/java/org/apache/solr/handler/component/StatsField.java @@ -405,8 +405,7 @@ public DocSet computeBaseDocSet() throws IOException { // tagMap has entries of List>, but subject to change in the future if (!(olst instanceof Collection)) continue; for (Object o : (Collection) olst) { - if (!(o instanceof QParser)) continue; - QParser qp = (QParser) o; + if (!(o instanceof QParser qp)) continue; try { excludeSet.put(qp.getQuery(), Boolean.TRUE); } catch (SyntaxError e) { diff --git a/solr/core/src/java/org/apache/solr/handler/designer/DefaultSchemaSuggester.java b/solr/core/src/java/org/apache/solr/handler/designer/DefaultSchemaSuggester.java index dd98eb728e2..d3dc4acaf5b 100644 --- a/solr/core/src/java/org/apache/solr/handler/designer/DefaultSchemaSuggester.java +++ b/solr/core/src/java/org/apache/solr/handler/designer/DefaultSchemaSuggester.java @@ -275,11 +275,10 @@ protected boolean isText(List values) { int maxLength = -1; int maxTerms = -1; for (Object next : values) { - if (!(next instanceof String)) { + if (!(next instanceof String cs)) { return false; } - String cs = (String) next; int len = cs.length(); if (len > maxLength) { maxLength = len; diff --git a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerAPI.java b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerAPI.java index f34793aa744..1370c775540 100644 --- a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerAPI.java +++ b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerAPI.java @@ -254,7 +254,7 @@ public void updateFileContents(SolrQueryRequest req, SolrQueryResponse rsp) try { InMemoryResourceLoader loader = new InMemoryResourceLoader(coreContainer, mutableId, SOLR_CONFIG_XML, data); - SolrConfig.readFromResourceLoader(loader, SOLR_CONFIG_XML, requestIsTrusted, null); + SolrConfig.readFromResourceLoader(loader, SOLR_CONFIG_XML, null); } catch (Exception exc) { updateFileError = exc; } diff --git a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java index 90eb3eb0c1c..955aa1d98a9 100644 --- a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java +++ b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java @@ -78,7 +78,6 @@ import org.apache.solr.cloud.ZkSolrResourceLoader; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.SolrZkClient; @@ -168,24 +167,12 @@ Map analyzeField(String configSet, String fieldName, String fiel } List listCollectionsForConfig(String configSet) { - final List collections = new ArrayList<>(); - Map states = - zkStateReader().getClusterState().getCollectionStates(); - for (Map.Entry e : states.entrySet()) { - final String coll = e.getKey(); - if (coll.startsWith(DESIGNER_PREFIX)) { - continue; // ignore temp - } - - try { - if (configSet.equals(e.getValue().get().getConfigName()) && e.getValue().get() != null) { - collections.add(coll); - } - } catch (Exception exc) { - log.warn("Failed to get config name for {}", coll, exc); - } - } - return collections; + return zkStateReader() + .getClusterState() + .collectionStream() + .filter(c -> configSet.equals(c.getConfigName())) + .map(DocCollection::getName) + .toList(); } @SuppressWarnings("unchecked") @@ -337,8 +324,7 @@ protected boolean updateFieldType( // nice, the json for this field looks like // "synonymQueryStyle": // "org.apache.solr.parser.SolrQueryParserBase$SynonymQueryStyle:AS_SAME_TERM" - if (typeAttrs.get("synonymQueryStyle") instanceof String) { - String synonymQueryStyle = (String) typeAttrs.get("synonymQueryStyle"); + if (typeAttrs.get("synonymQueryStyle") instanceof String synonymQueryStyle) { if (synonymQueryStyle.lastIndexOf(':') != -1) { typeAttrs.put( "synonymQueryStyle", @@ -691,9 +677,7 @@ ManagedIndexSchema deleteNestedDocsFieldsIfNeeded(ManagedIndexSchema schema, boo SolrConfig loadSolrConfig(String configSet) { ZkSolrResourceLoader zkLoader = zkLoaderForConfigSet(configSet); - boolean trusted = isConfigSetTrusted(configSet); - - return SolrConfig.readFromResourceLoader(zkLoader, SOLR_CONFIG_XML, trusted, null); + return SolrConfig.readFromResourceLoader(zkLoader, SOLR_CONFIG_XML, null); } ManagedIndexSchema loadLatestSchema(String configSet) { diff --git a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerSettings.java b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerSettings.java index 4f16464fedd..0216d433ee5 100644 --- a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerSettings.java +++ b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerSettings.java @@ -144,8 +144,7 @@ public void setFieldGuessingEnabled(boolean enabled) { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof SchemaDesignerSettings)) return false; - SchemaDesignerSettings that = (SchemaDesignerSettings) o; + if (!(o instanceof SchemaDesignerSettings that)) return false; return isDisabled == that.isDisabled && dynamicFieldsEnabled == that.dynamicFieldsEnabled && nestedDocsEnabled == that.nestedDocsEnabled diff --git a/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java index e65b0e2e1a9..1269021aa2f 100644 --- a/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java +++ b/solr/core/src/java/org/apache/solr/handler/loader/JavabinLoader.java @@ -164,9 +164,8 @@ public List readIterator(DataInputInputStream fis) throws IOException { params = ((NamedList) o).toSolrParams(); } else { try { - if (o instanceof byte[]) { + if (o instanceof byte[] buf) { if (params != null) req.setParams(params); - byte[] buf = (byte[]) o; contentStreamLoader.load( req, rsp, new ContentStreamBase.ByteArrayStream(buf, null), processor); } else { diff --git a/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java index 48e095a092c..ff07a8fb5a9 100644 --- a/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java +++ b/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java @@ -92,8 +92,7 @@ public static SolrInputDocument buildDoc(Map m) { SolrInputDocument result = new SolrInputDocument(); for (Map.Entry e : m.entrySet()) { if (mapEntryIsChildDoc(e.getValue())) { // parse child documents - if (e.getValue() instanceof List) { - List value = (List) e.getValue(); + if (e.getValue() instanceof List value) { for (Object o : value) { if (o instanceof Map) { // retain the value as a list, even if the list contains a single value. @@ -114,8 +113,7 @@ public static SolrInputDocument buildDoc(Map m) { } private static boolean mapEntryIsChildDoc(Object val) { - if (val instanceof List) { - List listVal = (List) val; + if (val instanceof List listVal) { if (listVal.size() == 0) return false; return listVal.get(0) instanceof Map; } @@ -297,9 +295,8 @@ public void handle(Map record, String path) { private Map getDocMap( Map record, JSONParser parser, String srcField, boolean mapUniqueKeyOnly) { Map result = mapUniqueKeyOnly ? record : new LinkedHashMap<>(record); - if (srcField != null && parser instanceof RecordingJSONParser) { + if (srcField != null && parser instanceof RecordingJSONParser rjp) { // if srcFIeld specified extract it out first - RecordingJSONParser rjp = (RecordingJSONParser) parser; result.put(srcField, rjp.getBuf()); rjp.resetBuf(); } diff --git a/solr/core/src/java/org/apache/solr/handler/tagger/TaggerRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/tagger/TaggerRequestHandler.java index c85aeafedcb..2c4c07d0a13 100644 --- a/solr/core/src/java/org/apache/solr/handler/tagger/TaggerRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/tagger/TaggerRequestHandler.java @@ -351,8 +351,7 @@ private Bits computeDocCorpus(SolrQueryRequest req) throws SyntaxError, IOExcept private boolean fieldHasIndexedStopFilter(String field, SolrQueryRequest req) { FieldType fieldType = req.getSchema().getFieldType(field); Analyzer analyzer = fieldType.getIndexAnalyzer(); // index analyzer - if (analyzer instanceof TokenizerChain) { - TokenizerChain tokenizerChain = (TokenizerChain) analyzer; + if (analyzer instanceof TokenizerChain tokenizerChain) { TokenFilterFactory[] tokenFilterFactories = tokenizerChain.getTokenFilterFactories(); for (TokenFilterFactory tokenFilterFactory : tokenFilterFactories) { if (tokenFilterFactory instanceof StopFilterFactory) return true; diff --git a/solr/core/src/java/org/apache/solr/jersey/CatchAllExceptionMapper.java b/solr/core/src/java/org/apache/solr/jersey/CatchAllExceptionMapper.java index 3db1c921398..1110880529c 100644 --- a/solr/core/src/java/org/apache/solr/jersey/CatchAllExceptionMapper.java +++ b/solr/core/src/java/org/apache/solr/jersey/CatchAllExceptionMapper.java @@ -65,8 +65,7 @@ public Response toResponse(Exception exception) { (SolrQueryResponse) containerRequestContext.getProperty(SOLR_QUERY_RESPONSE); final SolrQueryRequest solrQueryRequest = (SolrQueryRequest) containerRequestContext.getProperty(SOLR_QUERY_REQUEST); - if (exception instanceof WebApplicationException) { - final WebApplicationException wae = (WebApplicationException) exception; + if (exception instanceof WebApplicationException wae) { final SolrException solrException = new SolrException(getErrorCode(wae.getResponse().getStatus()), wae.getMessage()); solrQueryResponse.setException(solrException); diff --git a/solr/core/src/java/org/apache/solr/jersey/MediaTypeOverridingFilter.java b/solr/core/src/java/org/apache/solr/jersey/MediaTypeOverridingFilter.java index 29695918224..e5a7f7150cc 100644 --- a/solr/core/src/java/org/apache/solr/jersey/MediaTypeOverridingFilter.java +++ b/solr/core/src/java/org/apache/solr/jersey/MediaTypeOverridingFilter.java @@ -29,7 +29,7 @@ import java.io.IOException; import java.util.List; import org.apache.solr.api.JerseyResource; -import org.apache.solr.handler.admin.ZookeeperReadAPI; +import org.apache.solr.handler.admin.ZookeeperRead; import org.apache.solr.handler.api.V2ApiUtils; import org.apache.solr.request.SolrQueryRequest; @@ -39,7 +39,7 @@ public class MediaTypeOverridingFilter implements ContainerResponseFilter { private static final List> EXEMPTED_RESOURCES = - List.of(ZookeeperReadAPI.class); + List.of(ZookeeperRead.class); @Context private ResourceInfo resourceInfo; diff --git a/solr/core/src/java/org/apache/solr/legacy/BBoxStrategy.java b/solr/core/src/java/org/apache/solr/legacy/BBoxStrategy.java index d572bbd7c75..4ce80840064 100644 --- a/solr/core/src/java/org/apache/solr/legacy/BBoxStrategy.java +++ b/solr/core/src/java/org/apache/solr/legacy/BBoxStrategy.java @@ -177,13 +177,12 @@ public BBoxStrategy(SpatialContext ctx, String fieldNamePrefix, FieldType fieldT numQuads++; } if (fieldType.indexOptions() != IndexOptions.NONE - && fieldType instanceof LegacyFieldType - && ((LegacyFieldType) fieldType).numericType() != null) { + && fieldType instanceof LegacyFieldType legacyType + && legacyType.numericType() != null) { if (hasPointVals) { throw new IllegalArgumentException( "pointValues and LegacyNumericType are mutually exclusive"); } - final LegacyFieldType legacyType = (LegacyFieldType) fieldType; if (legacyType.numericType() != LegacyNumericType.DOUBLE) { throw new IllegalArgumentException( getClass() + " does not support " + legacyType.numericType()); @@ -302,10 +301,9 @@ public DoubleValuesSource makeOverlapRatioValueSource( @Override public Query makeQuery(SpatialArgs args) { Shape shape = args.getShape(); - if (!(shape instanceof Rectangle)) + if (!(shape instanceof Rectangle bbox)) throw new UnsupportedOperationException("Can only query by Rectangle, not " + shape); - Rectangle bbox = (Rectangle) shape; Query spatial; // Useful for understanding Relations: diff --git a/solr/core/src/java/org/apache/solr/legacy/BBoxValueSource.java b/solr/core/src/java/org/apache/solr/legacy/BBoxValueSource.java index d04f8046004..b7b794917e4 100644 --- a/solr/core/src/java/org/apache/solr/legacy/BBoxValueSource.java +++ b/solr/core/src/java/org/apache/solr/legacy/BBoxValueSource.java @@ -88,9 +88,7 @@ public boolean isCacheable(LeafReaderContext ctx) { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof BBoxValueSource)) return false; - - BBoxValueSource that = (BBoxValueSource) o; + if (!(o instanceof BBoxValueSource that)) return false; return strategy.equals(that.strategy); } diff --git a/solr/core/src/java/org/apache/solr/legacy/LegacyFieldType.java b/solr/core/src/java/org/apache/solr/legacy/LegacyFieldType.java index 98199d62b6a..c038d804b7a 100644 --- a/solr/core/src/java/org/apache/solr/legacy/LegacyFieldType.java +++ b/solr/core/src/java/org/apache/solr/legacy/LegacyFieldType.java @@ -114,8 +114,7 @@ public boolean equals(Object obj) { if (!super.equals(obj)) { return false; } - if (!(obj instanceof LegacyFieldType)) return false; - LegacyFieldType other = (LegacyFieldType) obj; + if (!(obj instanceof LegacyFieldType other)) return false; if (numericPrecisionStep != other.numericPrecisionStep) return false; return numericType == other.numericType; } diff --git a/solr/core/src/java/org/apache/solr/legacy/LegacyNumericRangeQuery.java b/solr/core/src/java/org/apache/solr/legacy/LegacyNumericRangeQuery.java index 617dcfa57b4..0fbc78e3142 100644 --- a/solr/core/src/java/org/apache/solr/legacy/LegacyNumericRangeQuery.java +++ b/solr/core/src/java/org/apache/solr/legacy/LegacyNumericRangeQuery.java @@ -422,8 +422,7 @@ public String toString(final String field) { public final boolean equals(final Object o) { if (o == this) return true; if (!super.equals(o)) return false; - if (o instanceof LegacyNumericRangeQuery) { - final LegacyNumericRangeQuery q = (LegacyNumericRangeQuery) o; + if (o instanceof LegacyNumericRangeQuery q) { return Objects.equals(q.min, min) && Objects.equals(q.max, max) && minInclusive == q.minInclusive diff --git a/solr/core/src/java/org/apache/solr/legacy/LegacyNumericTokenStream.java b/solr/core/src/java/org/apache/solr/legacy/LegacyNumericTokenStream.java index 5f72628402c..9ac670fce22 100644 --- a/solr/core/src/java/org/apache/solr/legacy/LegacyNumericTokenStream.java +++ b/solr/core/src/java/org/apache/solr/legacy/LegacyNumericTokenStream.java @@ -254,8 +254,7 @@ public int hashCode() { @Override public boolean equals(Object obj) { if (this == obj) return true; - if (!(obj instanceof LegacyNumericTermAttributeImpl)) return false; - LegacyNumericTermAttributeImpl other = (LegacyNumericTermAttributeImpl) obj; + if (!(obj instanceof LegacyNumericTermAttributeImpl other)) return false; if (precisionStep != other.precisionStep) return false; if (shift != other.shift) return false; if (value != other.value) return false; diff --git a/solr/core/src/java/org/apache/solr/metrics/AggregateMetric.java b/solr/core/src/java/org/apache/solr/metrics/AggregateMetric.java index 97be390c6b0..638f2ebc644 100644 --- a/solr/core/src/java/org/apache/solr/metrics/AggregateMetric.java +++ b/solr/core/src/java/org/apache/solr/metrics/AggregateMetric.java @@ -90,10 +90,9 @@ public double getMax() { } Double res = null; for (Update u : values.values()) { - if (!(u.value instanceof Number)) { + if (!(u.value instanceof Number n)) { continue; } - Number n = (Number) u.value; if (res == null) { res = n.doubleValue(); continue; @@ -114,10 +113,9 @@ public double getMin() { } Double res = null; for (Update u : values.values()) { - if (!(u.value instanceof Number)) { + if (!(u.value instanceof Number n)) { continue; } - Number n = (Number) u.value; if (res == null) { res = n.doubleValue(); continue; @@ -138,10 +136,9 @@ public double getMean() { } double total = 0; for (Update u : values.values()) { - if (!(u.value instanceof Number)) { + if (!(u.value instanceof Number n)) { continue; } - Number n = (Number) u.value; total += n.doubleValue(); } return total / values.size(); @@ -156,11 +153,10 @@ public double getStdDev() { double sum = 0; int count = 0; for (Update u : values.values()) { - if (!(u.value instanceof Number)) { + if (!(u.value instanceof Number n)) { continue; } count++; - Number n = (Number) u.value; final double diff = n.doubleValue() - mean; sum += diff * diff; } @@ -177,10 +173,9 @@ public double getSum() { } double res = 0; for (Update u : values.values()) { - if (!(u.value instanceof Number)) { + if (!(u.value instanceof Number n)) { continue; } - Number n = (Number) u.value; res += n.doubleValue(); } return res; diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricInfo.java b/solr/core/src/java/org/apache/solr/metrics/SolrMetricInfo.java index 2c8b9aa95cd..a016fab7827 100644 --- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricInfo.java +++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricInfo.java @@ -94,9 +94,8 @@ public String toString() { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof SolrMetricInfo)) return false; + if (!(o instanceof SolrMetricInfo that)) return false; - SolrMetricInfo that = (SolrMetricInfo) o; return Objects.equals(name, that.name) && Objects.equals(scope, that.scope) && category == that.category; diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java index 06bea336dd1..52c468445cb 100644 --- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java +++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java @@ -835,8 +835,7 @@ public int unregisterGauges(String registryName, String tagSegment) { AtomicInteger removed = new AtomicInteger(); registry.removeMatching( (name, metric) -> { - if (metric instanceof GaugeWrapper) { - GaugeWrapper wrapper = (GaugeWrapper) metric; + if (metric instanceof GaugeWrapper wrapper) { boolean toRemove = wrapper.getTag().contains(tagSegment); if (toRemove) { removed.incrementAndGet(); diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/SolrPrometheusFormatter.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/SolrPrometheusFormatter.java index 8c1ba8652ab..5a8a2de4f04 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/SolrPrometheusFormatter.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/SolrPrometheusFormatter.java @@ -140,8 +140,7 @@ public void exportGauge( GaugeSnapshot.GaugeDataPointSnapshot dataPoint = createGaugeDatapoint(((Number) dropwizardMetric).doubleValue(), labels); collectGaugeDatapoint(metricName, dataPoint); - } else if (dropwizardMetric instanceof HashMap) { - HashMap itemsMap = (HashMap) dropwizardMetric; + } else if (dropwizardMetric instanceof HashMap itemsMap) { for (Object item : itemsMap.keySet()) { if (itemsMap.get(item) instanceof Number) { GaugeSnapshot.GaugeDataPointSnapshot dataPoint = diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/jmx/JmxMetricsReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/jmx/JmxMetricsReporter.java index 34059b562a6..bd9eb1b2b21 100644 --- a/solr/core/src/java/org/apache/solr/metrics/reporters/jmx/JmxMetricsReporter.java +++ b/solr/core/src/java/org/apache/solr/metrics/reporters/jmx/JmxMetricsReporter.java @@ -580,8 +580,7 @@ public void onGaugeAdded(String name, Gauge gauge) { if (filter.matches(name, gauge)) { final ObjectName objectName = createName("gauges", name); if (gauge instanceof SolrMetricManager.GaugeWrapper - && ((SolrMetricManager.GaugeWrapper) gauge).getGauge() instanceof MetricsMap) { - MetricsMap mm = (MetricsMap) ((SolrMetricManager.GaugeWrapper) gauge).getGauge(); + && ((SolrMetricManager.GaugeWrapper) gauge).getGauge() instanceof MetricsMap mm) { mm.setAttribute(new Attribute(INSTANCE_TAG, tag)); // don't wrap it in a JmxGauge, it already supports all necessary JMX attributes registerMBean(mm, objectName); diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java index a751d20472f..1d69a9c429d 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java @@ -40,6 +40,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import org.apache.solr.cli.CLIUtils; import org.apache.solr.cli.SolrCLI; import org.apache.solr.client.api.util.SolrVersion; import org.apache.solr.client.solrj.SolrClient; @@ -680,7 +681,7 @@ private boolean prompt(boolean noprompt) { boolean shouldExecute = true; if (!noprompt) { // show a prompt asking user to execute the setup command for the plugin PackageUtils.print( - SolrCLI.YELLOW, + CLIUtils.YELLOW, "Execute this command. (If you choose no, you can manually deploy/undeploy this plugin later) (y/n): "); try (Scanner scanner = new Scanner(System.in, StandardCharsets.UTF_8)) { String userInput = scanner.next(); @@ -954,7 +955,7 @@ public void deploy( shouldInstallClusterPlugins, parameters); PackageUtils.print( - res ? SolrCLI.GREEN : SolrCLI.RED, res ? "Deployment successful" : "Deployment failed"); + res ? CLIUtils.GREEN : CLIUtils.RED, res ? "Deployment successful" : "Deployment failed"); } /** Undeploys a package from given collections. */ diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java index 26b294b4906..1d1270a2b84 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java @@ -33,7 +33,7 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import org.apache.lucene.util.SuppressForbidden; -import org.apache.solr.cli.SolrCLI; +import org.apache.solr.cli.CLIUtils; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; @@ -236,7 +236,7 @@ public static String resolve( /** Console print using green color */ public static void formatGreen(StringBuilder sb, Object message) { - format(sb, SolrCLI.GREEN, message); + format(sb, CLIUtils.GREEN, message); } public static void format(StringBuilder sb, Object message) { diff --git a/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java b/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java index bdea4869e12..4160fd7a648 100644 --- a/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java +++ b/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java @@ -549,8 +549,7 @@ protected Query getFieldQuery(String field, String queryText, int slop) throws S // only set slop of the phrase query was a result of this parser // and not a sub-parser. if (subQParser == null) { - if (query instanceof PhraseQuery) { - PhraseQuery pq = (PhraseQuery) query; + if (query instanceof PhraseQuery pq) { Term[] terms = pq.getTerms(); int[] positions = pq.getPositions(); PhraseQuery.Builder builder = new PhraseQuery.Builder(); @@ -559,8 +558,7 @@ protected Query getFieldQuery(String field, String queryText, int slop) throws S } builder.setSlop(slop); query = builder.build(); - } else if (query instanceof MultiPhraseQuery) { - MultiPhraseQuery mpq = (MultiPhraseQuery) query; + } else if (query instanceof MultiPhraseQuery mpq) { if (slop != mpq.getSlop()) { query = new MultiPhraseQuery.Builder(mpq).setSlop(slop).build(); @@ -1011,9 +1009,8 @@ protected ReversedWildcardFilterFactory getReversedWildcardFilterFactory(FieldTy } Analyzer a = fieldType.getIndexAnalyzer(); - if (a instanceof TokenizerChain) { + if (a instanceof TokenizerChain tc) { // examine the indexing analysis chain if it supports leading wildcards - TokenizerChain tc = (TokenizerChain) a; TokenFilterFactory[] factories = tc.getTokenFilterFactories(); for (TokenFilterFactory factory : factories) { if (factory instanceof ReversedWildcardFilterFactory) { @@ -1053,8 +1050,7 @@ protected String analyzeIfMultitermTermText(String field, String part, FieldType // Create a "normal" query from a RawQuery (or just return the current query if it's not raw) Query rawToNormal(Query q) { Query normal = q; - if (q instanceof RawQuery) { - RawQuery rawq = (RawQuery) q; + if (q instanceof RawQuery rawq) { if (rawq.sfield.getType().isTokenized()) { normal = rawq.sfield.getType().getFieldQuery(parser, rawq.sfield, rawq.getJoinedExternalVal()); diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java index 24339feefe8..24c9800f3e8 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java @@ -198,8 +198,7 @@ public PkgVersion(PackagePayload.AddVersion addVersion) { @Override public boolean equals(Object obj) { - if (obj instanceof PkgVersion) { - PkgVersion that = (PkgVersion) obj; + if (obj instanceof PkgVersion that) { return Objects.equals(this.version, that.version) && Objects.equals(this.files, that.files); } return false; diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java index 65d46edd5ad..272de544b6e 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java @@ -105,6 +105,9 @@ public SolrPackageLoader.SolrPackage.Version findPackageVersion( return theVersion; } + // Allow method reference to return a reference to a functional interface (Mapwriter), + // rather than a reference to a PgkVersion object + @SuppressWarnings("UnnecessaryMethodReference") @Override public MapWriter getPackageVersion(PluginInfo.ClassName cName) { if (cName.pkg == null) return null; diff --git a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java index 1f1d2459818..b0731d05095 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java @@ -146,8 +146,7 @@ protected Object initNewInstance(SolrPackageLoader.SolrPackage.Version newest, S handleAwareCallbacks(newest.getLoader(), instance, core); T old = inst; inst = (T) instance; - if (old instanceof AutoCloseable) { - AutoCloseable closeable = (AutoCloseable) old; + if (old instanceof AutoCloseable closeable) { try { closeable.close(); } catch (Exception e) { @@ -158,8 +157,7 @@ protected Object initNewInstance(SolrPackageLoader.SolrPackage.Version newest, S } private void handleAwareCallbacks(SolrResourceLoader loader, Object instance, SolrCore core) { - if (instance instanceof SolrCoreAware) { - SolrCoreAware coreAware = (SolrCoreAware) instance; + if (instance instanceof SolrCoreAware coreAware) { if (!core.getResourceLoader().addToCoreAware(coreAware)) { coreAware.inform(core); } diff --git a/solr/core/src/java/org/apache/solr/query/FilterQuery.java b/solr/core/src/java/org/apache/solr/query/FilterQuery.java index 9039bc1f083..8af5a5a51ee 100644 --- a/solr/core/src/java/org/apache/solr/query/FilterQuery.java +++ b/solr/core/src/java/org/apache/solr/query/FilterQuery.java @@ -85,8 +85,7 @@ public int hashCode() { @Override public boolean equals(Object obj) { - if (!(obj instanceof FilterQuery)) return false; - FilterQuery fq = (FilterQuery) obj; + if (!(obj instanceof FilterQuery fq)) return false; return this.q.equals(fq.q); } @@ -119,13 +118,12 @@ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float bo throws IOException { // SolrRequestInfo reqInfo = SolrRequestInfo.getRequestInfo(); - if (!(searcher instanceof SolrIndexSearcher)) { + if (!(searcher instanceof SolrIndexSearcher solrSearcher)) { // delete-by-query won't have SolrIndexSearcher // note: CSQ has some optimizations so we wrap it even though unnecessary given 0 boost return new ConstantScoreQuery(q).createWeight(searcher, scoreMode, 0f); } - SolrIndexSearcher solrSearcher = (SolrIndexSearcher) searcher; DocSet docs = solrSearcher.getDocSet(q); // reqInfo.addCloseHook(docs); // needed for off-heap refcounting diff --git a/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java b/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java index 9aa323ea6ad..c9c6af65ce2 100644 --- a/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java +++ b/solr/core/src/java/org/apache/solr/query/SolrRangeQuery.java @@ -115,10 +115,9 @@ public boolean equals(Object obj) { if (this == obj) { return true; } - if (!(obj instanceof SolrRangeQuery)) { + if (!(obj instanceof SolrRangeQuery other)) { return false; } - SolrRangeQuery other = (SolrRangeQuery) obj; return (this.flags == other.flags) && (this.field.equals(other.field)) diff --git a/solr/core/src/java/org/apache/solr/request/SimpleFacets.java b/solr/core/src/java/org/apache/solr/request/SimpleFacets.java index 276626123f5..bc8c5e73080 100644 --- a/solr/core/src/java/org/apache/solr/request/SimpleFacets.java +++ b/solr/core/src/java/org/apache/solr/request/SimpleFacets.java @@ -234,8 +234,7 @@ protected DocSet computeDocSet(DocSet baseDocSet, List excludeTagList) // tagMap has entries of List>, but subject to change in the future if (!(olst instanceof Collection)) continue; for (Object o : (Collection) olst) { - if (!(o instanceof QParser)) continue; - QParser qp = (QParser) o; + if (!(o instanceof QParser qp)) continue; excludeSet.put(qp.getQuery(), Boolean.TRUE); } } @@ -1292,8 +1291,7 @@ public int hashCode() { @Override public boolean equals(Object o) { - if (!(o instanceof CountPair)) return false; - CountPair that = (CountPair) o; + if (!(o instanceof CountPair that)) return false; return (this.key.equals(that.key) && this.val.equals(that.val)); } diff --git a/solr/core/src/java/org/apache/solr/request/SolrRequestInfo.java b/solr/core/src/java/org/apache/solr/request/SolrRequestInfo.java index 61b51a6af20..444867b430b 100644 --- a/solr/core/src/java/org/apache/solr/request/SolrRequestInfo.java +++ b/solr/core/src/java/org/apache/solr/request/SolrRequestInfo.java @@ -253,10 +253,12 @@ private void initQueryLimits() { */ public QueryLimits getLimits() { // make sure the ThreadCpuTime is always initialized - return req == null || rsp == null - ? QueryLimits.NONE - : (QueryLimits) - req.getContext().computeIfAbsent(LIMITS_KEY, (k) -> new QueryLimits(req, rsp)); + return req == null || rsp == null ? QueryLimits.NONE : getQueryLimits(req, rsp); + } + + public static QueryLimits getQueryLimits(SolrQueryRequest request, SolrQueryResponse response) { + return (QueryLimits) + request.getContext().computeIfAbsent(LIMITS_KEY, (k) -> new QueryLimits(request, response)); } public SolrDispatchFilter.Action getAction() { diff --git a/solr/core/src/java/org/apache/solr/request/json/JsonQueryConverter.java b/solr/core/src/java/org/apache/solr/request/json/JsonQueryConverter.java index 7c0baedd9d7..e612f99221a 100644 --- a/solr/core/src/java/org/apache/solr/request/json/JsonQueryConverter.java +++ b/solr/core/src/java/org/apache/solr/request/json/JsonQueryConverter.java @@ -145,14 +145,13 @@ private void buildLocalParams( } else { for (Map.Entry entry : map.entrySet()) { String key = entry.getKey(); - if (entry.getValue() instanceof List) { + if (entry.getValue() instanceof List l) { if (key.equals("query")) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Error when parsing json query, value of query field should not be a list, found : " + entry.getValue()); } - List l = (List) entry.getValue(); for (Object subVal : l) { builder.append(key).append("="); buildLocalParams(builder, subVal, true, additionalParams); diff --git a/solr/core/src/java/org/apache/solr/request/json/RequestUtil.java b/solr/core/src/java/org/apache/solr/request/json/RequestUtil.java index e6367612951..3cba7e0ec2c 100644 --- a/solr/core/src/java/org/apache/solr/request/json/RequestUtil.java +++ b/solr/core/src/java/org/apache/solr/request/json/RequestUtil.java @@ -78,7 +78,7 @@ public static void processParams( for (ContentStream cs : req.getContentStreams()) { // if BinaryResponseParser.BINARY_CONTENT_TYPE, let the following fail below - we may have // adjusted the content without updating the content type - // problem in this case happens in a few tests, one seems to happen with kerberos and remote + // problem in this case happens in a few tests including a remote // node query (HttpSolrCall's request proxy) String contentType = cs.getContentType(); @@ -363,8 +363,7 @@ private static void getParamsFromJSON(Map params, String json) if (val == null) { params.remove(key); - } else if (val instanceof List) { - List lst = (List) val; + } else if (val instanceof List lst) { String[] vals = new String[lst.size()]; for (int i = 0; i < vals.length; i++) { vals[i] = lst.get(i).toString(); diff --git a/solr/core/src/java/org/apache/solr/response/BinaryResponseWriter.java b/solr/core/src/java/org/apache/solr/response/BinaryResponseWriter.java index 7b72c7e78f6..317412802d1 100644 --- a/solr/core/src/java/org/apache/solr/response/BinaryResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/BinaryResponseWriter.java @@ -83,9 +83,8 @@ public Resolver(SolrQueryRequest req, ReturnFields returnFields) { @Override public Object resolve(Object o, JavaBinCodec codec) throws IOException { - if (o instanceof ResultContext) { + if (o instanceof ResultContext res) { ReturnFields orig = returnFields; - ResultContext res = (ResultContext) o; if (res.getReturnFields() != null) { returnFields = res.getReturnFields(); } @@ -111,10 +110,9 @@ public Object resolve(Object o, JavaBinCodec codec) throws IOException { writeResults(ctx, codec); return null; // null means we completely handled it } - if (o instanceof IndexableField) { + if (o instanceof IndexableField f) { if (schema == null) schema = solrQueryRequest.getSchema(); - IndexableField f = (IndexableField) o; SchemaField sf = schema.getFieldOrNull(f.name()); try { o = DocsStreamer.getValue(sf, f); @@ -262,8 +260,7 @@ public Entry next() { @Override public Object getFirstValue(String name) { Object v = _fields.get(name); - if (v == null || !(v instanceof Collection)) return convertCharSeq(v); - Collection c = (Collection) v; + if (v == null || !(v instanceof Collection c)) return convertCharSeq(v); if (c.size() > 0) { return convertCharSeq(c.iterator().next()); } diff --git a/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java b/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java index 05df434bf37..8cb30080fd7 100644 --- a/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/CSVResponseWriter.java @@ -419,8 +419,7 @@ public void writeSolrDocument(String name, SolrDocument doc, ReturnFields return } else { // normalize to first value - if (val instanceof Collection) { - Collection values = (Collection) val; + if (val instanceof Collection values) { val = values.iterator().next(); } // if field is polyfield, use the multi-valued printer to apply appropriate escaping diff --git a/solr/core/src/java/org/apache/solr/response/GraphMLResponseWriter.java b/solr/core/src/java/org/apache/solr/response/GraphMLResponseWriter.java index 6a49f92fdc3..11a04d10240 100644 --- a/solr/core/src/java/org/apache/solr/response/GraphMLResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/GraphMLResponseWriter.java @@ -46,8 +46,7 @@ public void write(Writer writer, SolrQueryRequest req, SolrQueryResponse res) th TupleStream stream = (TupleStream) req.getContext().get("stream"); - if (stream instanceof GraphHandler.DummyErrorStream) { - GraphHandler.DummyErrorStream d = (GraphHandler.DummyErrorStream) stream; + if (stream instanceof GraphHandler.DummyErrorStream d) { Exception e = d.getException(); e.printStackTrace(new PrintWriter(writer)); return; diff --git a/solr/core/src/java/org/apache/solr/response/QueryResponseWriterUtil.java b/solr/core/src/java/org/apache/solr/response/QueryResponseWriterUtil.java index 44455a8cf36..1ea0642738d 100644 --- a/solr/core/src/java/org/apache/solr/response/QueryResponseWriterUtil.java +++ b/solr/core/src/java/org/apache/solr/response/QueryResponseWriterUtil.java @@ -49,13 +49,11 @@ public static void writeQueryResponse( String contentType) throws IOException { - if (responseWriter instanceof JacksonJsonWriter) { - JacksonJsonWriter binWriter = (JacksonJsonWriter) responseWriter; + if (responseWriter instanceof JacksonJsonWriter binWriter) { BufferedOutputStream bos = new BufferedOutputStream(new NonFlushingStream(outputStream)); binWriter.write(bos, solrRequest, solrResponse); bos.flush(); - } else if (responseWriter instanceof BinaryQueryResponseWriter) { - BinaryQueryResponseWriter binWriter = (BinaryQueryResponseWriter) responseWriter; + } else if (responseWriter instanceof BinaryQueryResponseWriter binWriter) { binWriter.write(outputStream, solrRequest, solrResponse); } else { OutputStream out = new NonFlushingStream(outputStream); diff --git a/solr/core/src/java/org/apache/solr/response/RawResponseWriter.java b/solr/core/src/java/org/apache/solr/response/RawResponseWriter.java index 34d7e112ed2..d20128ae4e0 100644 --- a/solr/core/src/java/org/apache/solr/response/RawResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/RawResponseWriter.java @@ -98,9 +98,8 @@ public String getContentType(SolrQueryRequest request, SolrQueryResponse respons public void write(Writer writer, SolrQueryRequest request, SolrQueryResponse response) throws IOException { Object obj = response.getValues().get(CONTENT); - if (obj != null && (obj instanceof ContentStream)) { + if (obj != null && (obj instanceof ContentStream content)) { // copy the contents to the writer... - ContentStream content = (ContentStream) obj; try (Reader reader = content.getReader()) { reader.transferTo(writer); } @@ -113,14 +112,12 @@ public void write(Writer writer, SolrQueryRequest request, SolrQueryResponse res public void write(OutputStream out, SolrQueryRequest request, SolrQueryResponse response) throws IOException { Object obj = response.getValues().get(CONTENT); - if (obj != null && (obj instanceof ContentStream)) { + if (obj != null && (obj instanceof ContentStream content)) { // copy the contents to the writer... - ContentStream content = (ContentStream) obj; try (InputStream in = content.getStream()) { in.transferTo(out); } - } else if (obj != null && (obj instanceof SolrCore.RawWriter)) { - final var rawWriter = (SolrCore.RawWriter) obj; + } else if (obj != null && (obj instanceof SolrCore.RawWriter rawWriter)) { rawWriter.write(out); if (rawWriter instanceof Closeable) ((Closeable) rawWriter).close(); } else { diff --git a/solr/core/src/java/org/apache/solr/response/SolrQueryResponse.java b/solr/core/src/java/org/apache/solr/response/SolrQueryResponse.java index b399ce57b79..5676a63fe10 100644 --- a/solr/core/src/java/org/apache/solr/response/SolrQueryResponse.java +++ b/solr/core/src/java/org/apache/solr/response/SolrQueryResponse.java @@ -207,7 +207,10 @@ public static boolean haveCompleteResults(NamedList header) { */ public void addPartialResponseDetail(Object detail) { NamedList header = getResponseHeader(); - if (header != null && detail != null) { + // never overwrite the original detail message. The first limit violation is the important one. + if (header != null + && detail != null + && header.get(RESPONSE_HEADER_PARTIAL_RESULTS_DETAILS_KEY) == null) { header.add(RESPONSE_HEADER_PARTIAL_RESULTS_DETAILS_KEY, detail); } } diff --git a/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java b/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java index 734dd2bea1b..a7017f5d4fb 100644 --- a/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/TextResponseWriter.java @@ -176,8 +176,7 @@ public final void writeVal(String name, Object val, boolean raw) throws IOExcept return; } - if (val instanceof IndexableField) { - IndexableField f = (IndexableField) val; + if (val instanceof IndexableField f) { SchemaField sf = schema.getFieldOrNull(f.name()); if (sf != null) { sf.getType().write(raw ? rawShim : this, name, f); @@ -206,8 +205,7 @@ public final void writeVal(String name, Object val, boolean raw) throws IOExcept // restricts the fields to write...? } else if (val instanceof SolrDocumentList) { writeSolrDocumentList(name, (SolrDocumentList) val, returnFields); - } else if (val instanceof BytesRef) { - BytesRef arr = (BytesRef) val; + } else if (val instanceof BytesRef arr) { writeByteArr(name, arr.bytes, arr.offset, arr.length); } else { TextWriter.super.writeVal(name, val, raw); diff --git a/solr/core/src/java/org/apache/solr/response/transform/BaseEditorialTransformer.java b/solr/core/src/java/org/apache/solr/response/transform/BaseEditorialTransformer.java index 8382f7c0c59..3ff7c639f5d 100644 --- a/solr/core/src/java/org/apache/solr/response/transform/BaseEditorialTransformer.java +++ b/solr/core/src/java/org/apache/solr/response/transform/BaseEditorialTransformer.java @@ -59,8 +59,7 @@ public void transform(SolrDocument doc, int docid) { protected BytesRef getKey(SolrDocument doc) { Object obj = doc.get(idFieldName); - if (obj instanceof IndexableField) { - IndexableField f = (IndexableField) obj; + if (obj instanceof IndexableField f) { BytesRefBuilder bytesRefBuilder = new BytesRefBuilder(); Number n = f.numericValue(); if (n != null) { diff --git a/solr/core/src/java/org/apache/solr/response/transform/GeoTransformerFactory.java b/solr/core/src/java/org/apache/solr/response/transform/GeoTransformerFactory.java index 443426a1d68..725a2d2b52a 100644 --- a/solr/core/src/java/org/apache/solr/response/transform/GeoTransformerFactory.java +++ b/solr/core/src/java/org/apache/solr/response/transform/GeoTransformerFactory.java @@ -95,7 +95,7 @@ public DocTransformer create( ErrorCode.BAD_REQUEST, this.getClass().getSimpleName() + " using unknown field: " + fname); } - if (!(sf.getType() instanceof AbstractSpatialFieldType)) { + if (!(sf.getType() instanceof AbstractSpatialFieldType sdv)) { throw new SolrException( ErrorCode.BAD_REQUEST, "GeoTransformer requested non-spatial field: " @@ -111,7 +111,6 @@ public DocTransformer create( updater.display_error = display + "_error"; final ShapeValuesSource shapes; - AbstractSpatialFieldType sdv = (AbstractSpatialFieldType) sf.getType(); SpatialStrategy strategy = sdv.getStrategy(fname); if (strategy instanceof CompositeSpatialStrategy) { shapes = ((CompositeSpatialStrategy) strategy).getGeometryStrategy().makeShapeValueSource(); diff --git a/solr/core/src/java/org/apache/solr/response/transform/SubQueryAugmenterFactory.java b/solr/core/src/java/org/apache/solr/response/transform/SubQueryAugmenterFactory.java index 713dff67053..a750e6fcde5 100644 --- a/solr/core/src/java/org/apache/solr/response/transform/SubQueryAugmenterFactory.java +++ b/solr/core/src/java/org/apache/solr/response/transform/SubQueryAugmenterFactory.java @@ -274,8 +274,7 @@ private Collection mapToDocField(String param) { private String convertFieldValue(Object val) { - if (val instanceof IndexableField) { - IndexableField f = (IndexableField) val; + if (val instanceof IndexableField f) { return f.stringValue(); } return val.toString(); diff --git a/solr/core/src/java/org/apache/solr/rest/ManagedResourceStorage.java b/solr/core/src/java/org/apache/solr/rest/ManagedResourceStorage.java index af04c36eaab..038b9c5f421 100644 --- a/solr/core/src/java/org/apache/solr/rest/ManagedResourceStorage.java +++ b/solr/core/src/java/org/apache/solr/rest/ManagedResourceStorage.java @@ -172,16 +172,23 @@ public void configure(SolrResourceLoader loader, NamedList initArgs) throw new IllegalArgumentException( "Required configuration parameter '" + STORAGE_DIR_INIT_ARG + "' not provided!"); - File dir = new File(storageDirArg); - if (!dir.isDirectory()) dir.mkdirs(); + Path dir = Path.of(storageDirArg); + if (!Files.isDirectory(dir)) { + try { + Files.createDirectories(dir); + } catch (IOException e) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, "Unable to create storage directory " + dir, e); + } + } - storageDir = dir.getAbsolutePath(); + storageDir = dir.toAbsolutePath().toString(); log.info("File-based storage initialized to use dir: {}", storageDir); } @Override public boolean exists(String storedResourceId) throws IOException { - return (new File(storageDir, storedResourceId)).exists(); + return Files.exists(Path.of(storageDir, storedResourceId)); } @Override @@ -196,18 +203,19 @@ public OutputStream openOutputStream(String storedResourceId) throws IOException @Override public boolean delete(String storedResourceId) throws IOException { + // TODO SOLR-8282 move to PATH File storedFile = new File(storageDir, storedResourceId); - return deleteIfFile(storedFile); + return deleteIfFile(storedFile.toPath()); } // TODO: this interface should probably be changed, this simulates the old behavior, // only throw security exception, just return false otherwise - private boolean deleteIfFile(File f) { - if (!f.isFile()) { + private boolean deleteIfFile(Path p) { + if (!Files.isRegularFile(p)) { return false; } try { - Files.delete(f.toPath()); + Files.delete(p); return true; } catch (IOException cause) { return false; diff --git a/solr/core/src/java/org/apache/solr/rest/schema/analysis/ManagedSynonymFilterFactory.java b/solr/core/src/java/org/apache/solr/rest/schema/analysis/ManagedSynonymFilterFactory.java index a004c907f20..94fd01114fe 100644 --- a/solr/core/src/java/org/apache/solr/rest/schema/analysis/ManagedSynonymFilterFactory.java +++ b/solr/core/src/java/org/apache/solr/rest/schema/analysis/ManagedSynonymFilterFactory.java @@ -221,8 +221,7 @@ protected boolean applyMapUpdates(Map jsonMap, boolean ignoreCas Set output = cpsm.mappings.get(origTerm); Object val = jsonMap.get(origTerm); // IMPORTANT: use the original - if (val instanceof String) { - String strVal = (String) val; + if (val instanceof String strVal) { if (output == null) { output = new TreeSet<>(); diff --git a/solr/core/src/java/org/apache/solr/rest/schema/analysis/ManagedSynonymGraphFilterFactory.java b/solr/core/src/java/org/apache/solr/rest/schema/analysis/ManagedSynonymGraphFilterFactory.java index 7863ed6fc3b..1d91fb8686f 100644 --- a/solr/core/src/java/org/apache/solr/rest/schema/analysis/ManagedSynonymGraphFilterFactory.java +++ b/solr/core/src/java/org/apache/solr/rest/schema/analysis/ManagedSynonymGraphFilterFactory.java @@ -217,8 +217,7 @@ protected boolean applyMapUpdates(Map jsonMap, boolean ignoreCas Set output = cpsm.mappings.get(origTerm); Object val = jsonMap.get(origTerm); // IMPORTANT: use the original - if (val instanceof String) { - String strVal = (String) val; + if (val instanceof String strVal) { if (output == null) { output = new TreeSet<>(); diff --git a/solr/core/src/java/org/apache/solr/schema/BinaryField.java b/solr/core/src/java/org/apache/solr/schema/BinaryField.java index 94bc94fd86f..1659e51f870 100644 --- a/solr/core/src/java/org/apache/solr/schema/BinaryField.java +++ b/solr/core/src/java/org/apache/solr/schema/BinaryField.java @@ -112,8 +112,7 @@ private static BytesRef getBytesRef(Object val) { if (val instanceof byte[]) { buf = (byte[]) val; len = buf.length; - } else if (val instanceof ByteBuffer && ((ByteBuffer) val).hasArray()) { - ByteBuffer byteBuf = (ByteBuffer) val; + } else if (val instanceof ByteBuffer byteBuf && byteBuf.hasArray()) { buf = byteBuf.array(); offset = byteBuf.arrayOffset() + byteBuf.position(); len = byteBuf.limit() - byteBuf.position(); @@ -156,8 +155,7 @@ protected void checkSupportsDocValues() { // we support DocValues public Object toNativeType(Object val) { if (val instanceof byte[]) { return ByteBuffer.wrap((byte[]) val); - } else if (val instanceof CharSequence) { - final CharSequence valAsCharSequence = (CharSequence) val; + } else if (val instanceof CharSequence valAsCharSequence) { return ByteBuffer.wrap(Base64.getDecoder().decode(valAsCharSequence.toString())); } return super.toNativeType(val); diff --git a/solr/core/src/java/org/apache/solr/schema/CurrencyFieldType.java b/solr/core/src/java/org/apache/solr/schema/CurrencyFieldType.java index dc4cc1eac79..1b4bf4574ba 100644 --- a/solr/core/src/java/org/apache/solr/schema/CurrencyFieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/CurrencyFieldType.java @@ -515,9 +515,7 @@ public String description() { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof ConvertedCurrencyValueSource)) return false; - - ConvertedCurrencyValueSource that = (ConvertedCurrencyValueSource) o; + if (!(o instanceof ConvertedCurrencyValueSource that)) return false; return Objects.equals(source, that.source) && (rate == that.rate) @@ -728,9 +726,7 @@ public String description() { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof RawCurrencyValueSource)) return false; - - RawCurrencyValueSource that = (RawCurrencyValueSource) o; + if (!(o instanceof RawCurrencyValueSource that)) return false; return Objects.equals(amountValues, that.amountValues) && Objects.equals(currencyValues, that.currencyValues) diff --git a/solr/core/src/java/org/apache/solr/schema/DateRangeField.java b/solr/core/src/java/org/apache/solr/schema/DateRangeField.java index d7a91a41383..880473d68ab 100644 --- a/solr/core/src/java/org/apache/solr/schema/DateRangeField.java +++ b/solr/core/src/java/org/apache/solr/schema/DateRangeField.java @@ -75,8 +75,7 @@ public List createFields(SchemaField field, Object val) { protected String getStoredValue(Shape shape, String shapeStr) { // even if shapeStr is set, it might have included some dateMath, so see if we can resolve it // first: - if (shape instanceof UnitNRShape) { - UnitNRShape unitShape = (UnitNRShape) shape; + if (shape instanceof UnitNRShape unitShape) { if (unitShape.getLevel() == tree.getMaxLevels()) { // fully precise date. We can be fully compatible with DatePointField (incl. 'Z') return shape.toString() + 'Z'; diff --git a/solr/core/src/java/org/apache/solr/schema/ExternalFileFieldReloader.java b/solr/core/src/java/org/apache/solr/schema/ExternalFileFieldReloader.java index 63d546d645a..10f4945ed40 100644 --- a/solr/core/src/java/org/apache/solr/schema/ExternalFileFieldReloader.java +++ b/solr/core/src/java/org/apache/solr/schema/ExternalFileFieldReloader.java @@ -79,8 +79,7 @@ public void cacheFieldSources(IndexSchema schema) { fieldSources.clear(); for (SchemaField field : schema.getFields().values()) { FieldType type = field.getType(); - if (type instanceof ExternalFileField) { - ExternalFileField eff = (ExternalFileField) type; + if (type instanceof ExternalFileField eff) { fieldSources.add(eff.getFileFloatSource(field, datadir)); if (log.isInfoEnabled()) { log.info("Adding ExternalFileFieldReloader listener for field {}", field.getName()); diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java index cb60b1f332a..472f6b4775c 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java @@ -1311,9 +1311,8 @@ protected Map getNonFieldPropertyArgs() { protected static SimpleOrderedMap getAnalyzerProperties(Analyzer analyzer) { SimpleOrderedMap analyzerProps = new SimpleOrderedMap<>(); - if (analyzer instanceof TokenizerChain) { + if (analyzer instanceof TokenizerChain tokenizerChain) { Map factoryArgs; - TokenizerChain tokenizerChain = (TokenizerChain) analyzer; CharFilterFactory[] charFilterFactories = tokenizerChain.getCharFilterFactories(); if (0 < charFilterFactories.length) { List> charFilterProps = new ArrayList<>(); diff --git a/solr/core/src/java/org/apache/solr/schema/FileExchangeRateProvider.java b/solr/core/src/java/org/apache/solr/schema/FileExchangeRateProvider.java index 18318770c1c..62161d87dcc 100644 --- a/solr/core/src/java/org/apache/solr/schema/FileExchangeRateProvider.java +++ b/solr/core/src/java/org/apache/solr/schema/FileExchangeRateProvider.java @@ -132,9 +132,7 @@ private void addRate( @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof FileExchangeRateProvider)) return false; - - FileExchangeRateProvider that = (FileExchangeRateProvider) o; + if (!(o instanceof FileExchangeRateProvider that)) return false; return Objects.equals(rates, that.rates); } diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java index 06dad3175dc..6f4f131de8f 100644 --- a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java +++ b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java @@ -526,7 +526,7 @@ protected void readSchema(ConfigSetService.ConfigResource is) { log.info("{}", sb); } - version = Float.parseFloat(rootNode.attributes().get("version", "1.0f")); + version = Float.parseFloat(rootNode.attributes().getOrDefault("version", "1.0f")); // load the Field Types final FieldTypePluginLoader typeLoader = @@ -2017,8 +2017,7 @@ private void informResourceLoaderAwareObjectsForFieldType(FieldType fieldType) { informResourceLoaderAwareObjectsInChain((TokenizerChain) queryAnalyzer); // if fieldType is a TextField, it might have a multi-term analyzer - if (fieldType instanceof TextField) { - TextField textFieldType = (TextField) fieldType; + if (fieldType instanceof TextField textFieldType) { Analyzer multiTermAnalyzer = textFieldType.getMultiTermAnalyzer(); if (multiTermAnalyzer != null && multiTermAnalyzer != indexAnalyzer diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchemaFactory.java b/solr/core/src/java/org/apache/solr/schema/IndexSchemaFactory.java index 0300f9bba03..8a984874538 100644 --- a/solr/core/src/java/org/apache/solr/schema/IndexSchemaFactory.java +++ b/solr/core/src/java/org/apache/solr/schema/IndexSchemaFactory.java @@ -168,8 +168,7 @@ public static VersionedConfig getFromCache( return c.get(); }; - if (loader instanceof ZkSolrResourceLoader) { - ZkSolrResourceLoader zkLoader = (ZkSolrResourceLoader) loader; + if (loader instanceof ZkSolrResourceLoader zkLoader) { ObjectCache objectCache = objectCacheSupplier.get(); if (objectCache == null) return cfgLoader.get(); Map confCache = diff --git a/solr/core/src/java/org/apache/solr/schema/LatLonPointSpatialField.java b/solr/core/src/java/org/apache/solr/schema/LatLonPointSpatialField.java index 5010ba27075..f484c28d330 100644 --- a/solr/core/src/java/org/apache/solr/schema/LatLonPointSpatialField.java +++ b/solr/core/src/java/org/apache/solr/schema/LatLonPointSpatialField.java @@ -138,12 +138,11 @@ public LatLonPointSpatialStrategy( @Override public Field[] createIndexableFields(Shape shape) { - if (!(shape instanceof Point)) { + if (!(shape instanceof Point point)) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, getClass().getSimpleName() + " only supports indexing points; got: " + shape); } - Point point = (Point) shape; int fieldsLen = (indexed ? 1 : 0) + (docValues ? 1 : 0); Field[] fields = new Field[fieldsLen]; @@ -179,17 +178,14 @@ public Query makeQuery(SpatialArgs args) { // Uses LatLonPoint protected Query makeQueryFromIndex(Shape shape) { // note: latitude then longitude order for LLP's methods - if (shape instanceof Circle) { - Circle circle = (Circle) shape; + if (shape instanceof Circle circle) { double radiusMeters = circle.getRadius() * DistanceUtils.DEG_TO_KM * 1000; return LatLonPoint.newDistanceQuery( getFieldName(), circle.getCenter().getY(), circle.getCenter().getX(), radiusMeters); - } else if (shape instanceof Rectangle) { - Rectangle rect = (Rectangle) shape; + } else if (shape instanceof Rectangle rect) { return LatLonPoint.newBoxQuery( getFieldName(), rect.getMinY(), rect.getMaxY(), rect.getMinX(), rect.getMaxX()); - } else if (shape instanceof Point) { - Point point = (Point) shape; + } else if (shape instanceof Point point) { return LatLonPoint.newDistanceQuery(getFieldName(), point.getY(), point.getX(), 0); } else { throw new UnsupportedOperationException( @@ -204,17 +200,14 @@ protected Query makeQueryFromIndex(Shape shape) { // Uses DocValuesField (otherwise identical to above) protected Query makeQueryFromDocValues(Shape shape) { // note: latitude then longitude order for LLP's methods - if (shape instanceof Circle) { - Circle circle = (Circle) shape; + if (shape instanceof Circle circle) { double radiusMeters = circle.getRadius() * DistanceUtils.DEG_TO_KM * 1000; return LatLonDocValuesField.newSlowDistanceQuery( getFieldName(), circle.getCenter().getY(), circle.getCenter().getX(), radiusMeters); - } else if (shape instanceof Rectangle) { - Rectangle rect = (Rectangle) shape; + } else if (shape instanceof Rectangle rect) { return LatLonDocValuesField.newSlowBoxQuery( getFieldName(), rect.getMinY(), rect.getMaxY(), rect.getMinX(), rect.getMaxX()); - } else if (shape instanceof Point) { - Point point = (Point) shape; + } else if (shape instanceof Point point) { return LatLonDocValuesField.newSlowDistanceQuery( getFieldName(), point.getY(), point.getX(), 0); } else { @@ -258,8 +251,7 @@ private static class DistanceSortValueSource extends DoubleValuesSource { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof DistanceSortValueSource)) return false; - DistanceSortValueSource that = (DistanceSortValueSource) o; + if (!(o instanceof DistanceSortValueSource that)) return false; return Double.compare(that.multiplier, multiplier) == 0 && Objects.equals(fieldName, that.fieldName) && Objects.equals(queryPoint, that.queryPoint); diff --git a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchemaFactory.java b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchemaFactory.java index a4e737d364f..df44cda12e2 100644 --- a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchemaFactory.java +++ b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchemaFactory.java @@ -209,12 +209,11 @@ public ManagedIndexSchema create( } int schemaZkVersion = -1; - if (!(loader instanceof ZkSolrResourceLoader)) { + if (!(loader instanceof ZkSolrResourceLoader zkLoader)) { Entry localSchemaInput = readSchemaLocally(); loadedResource = localSchemaInput.getKey(); schemaInputStream = localSchemaInput.getValue(); } else { // ZooKeeper - final ZkSolrResourceLoader zkLoader = (ZkSolrResourceLoader) loader; final SolrZkClient zkClient = zkLoader.getZkController().getZkClient(); final String managedSchemaPath = lookupZKManagedSchemaPath(); managedSchemaResourceName = @@ -347,8 +346,7 @@ private void warnIfNonManagedSchemaExists() { if (!resourceName.equals(managedSchemaResourceName)) { boolean exists = false; SolrResourceLoader loader = config.getResourceLoader(); - if (loader instanceof ZkSolrResourceLoader) { - ZkSolrResourceLoader zkLoader = (ZkSolrResourceLoader) loader; + if (loader instanceof ZkSolrResourceLoader zkLoader) { String nonManagedSchemaPath = zkLoader.getConfigSetZkPath() + "/" + resourceName; try { exists = zkLoader.getZkController().pathExists(nonManagedSchemaPath); @@ -403,7 +401,8 @@ private void upgradeToManagedSchema() { "On upgrading to managed schema, did not rename non-managed schema '{}' because it's the same as the managed schema's name.", resourceName); } else { - final File nonManagedSchemaFile = locateConfigFile(resourceName); + // TODO SOLR-8282 move to PATH + final File nonManagedSchemaFile = locateConfigFile(resourceName).toFile(); if (null == nonManagedSchemaFile) { // Don't throw an exception for failure to rename the non-managed schema log.warn( @@ -443,11 +442,11 @@ private void upgradeToManagedSchema() { * * @return the File for the named resource, or null if it can't be found */ - private File locateConfigFile(String resource) { + private Path locateConfigFile(String resource) { String location = config.getResourceLoader().resourceLocation(resource); if (location == null || location.equals(resource) || location.startsWith("classpath:")) return null; - return new File(location); + return Path.of(location); } /** @@ -544,9 +543,8 @@ public Object getSchemaUpdateLock() { @Override public void inform(SolrCore core) { this.core = core; - if (loader instanceof ZkSolrResourceLoader) { + if (loader instanceof ZkSolrResourceLoader zkLoader) { this.zkIndexSchemaReader = new ZkIndexSchemaReader(this, core); - ZkSolrResourceLoader zkLoader = (ZkSolrResourceLoader) loader; zkLoader.setZkIndexSchemaReader(this.zkIndexSchemaReader); try { zkIndexSchemaReader.refreshSchemaFromZk(-1); // update immediately if newer is available diff --git a/solr/core/src/java/org/apache/solr/schema/OpenExchangeRatesOrgProvider.java b/solr/core/src/java/org/apache/solr/schema/OpenExchangeRatesOrgProvider.java index fcc64b111eb..a17dd6fc78b 100644 --- a/solr/core/src/java/org/apache/solr/schema/OpenExchangeRatesOrgProvider.java +++ b/solr/core/src/java/org/apache/solr/schema/OpenExchangeRatesOrgProvider.java @@ -122,9 +122,8 @@ private void reloadIfExpired() { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof OpenExchangeRatesOrgProvider)) return false; + if (!(o instanceof OpenExchangeRatesOrgProvider that)) return false; - OpenExchangeRatesOrgProvider that = (OpenExchangeRatesOrgProvider) o; return Objects.equals(rates, that.rates); } diff --git a/solr/core/src/java/org/apache/solr/schema/RandomSortField.java b/solr/core/src/java/org/apache/solr/schema/RandomSortField.java index 1330832676e..5942e69a684 100644 --- a/solr/core/src/java/org/apache/solr/schema/RandomSortField.java +++ b/solr/core/src/java/org/apache/solr/schema/RandomSortField.java @@ -192,8 +192,7 @@ public int intVal(int doc) { @Override public boolean equals(Object o) { - if (!(o instanceof RandomValueSource)) return false; - RandomValueSource other = (RandomValueSource) o; + if (!(o instanceof RandomValueSource other)) return false; return this.field.equals(other.field); } diff --git a/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java b/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java index ce9a7e04669..ddd814121fa 100644 --- a/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java +++ b/solr/core/src/java/org/apache/solr/schema/RptWithGeometrySpatialField.java @@ -134,9 +134,8 @@ public String toString() { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof CachingShapeValuesource)) return false; + if (!(o instanceof CachingShapeValuesource that)) return false; - CachingShapeValuesource that = (CachingShapeValuesource) o; return Objects.equals(targetValueSource, that.targetValueSource) && Objects.equals(fieldName, that.fieldName); } @@ -217,9 +216,7 @@ private PerSegCacheKey(Object segCoreKey, int docId) { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof PerSegCacheKey)) return false; - - PerSegCacheKey that = (PerSegCacheKey) o; + if (!(o instanceof PerSegCacheKey that)) return false; if (docId != that.docId) return false; diff --git a/solr/core/src/java/org/apache/solr/schema/SchemaManager.java b/solr/core/src/java/org/apache/solr/schema/SchemaManager.java index 53a3bbff40c..b30293efb1a 100644 --- a/solr/core/src/java/org/apache/solr/schema/SchemaManager.java +++ b/solr/core/src/java/org/apache/solr/schema/SchemaManager.java @@ -120,8 +120,7 @@ private List> doOperations(List operations errors = CommandOperation.captureErrors(operations); if (!errors.isEmpty()) break; SolrResourceLoader loader = req.getCore().getResourceLoader(); - if (loader instanceof ZkSolrResourceLoader) { - ZkSolrResourceLoader zkLoader = (ZkSolrResourceLoader) loader; + if (loader instanceof ZkSolrResourceLoader zkLoader) { StringWriter sw = new StringWriter(); try { managedIndexSchema.persist(sw); @@ -484,8 +483,7 @@ private ManagedIndexSchema getFreshManagedSchema(SolrCore core) SolrResourceLoader resourceLoader = core.getResourceLoader(); String schemaResourceName = core.getLatestSchema().getResourceName(); - if (resourceLoader instanceof ZkSolrResourceLoader) { - final ZkSolrResourceLoader zkLoader = (ZkSolrResourceLoader) resourceLoader; + if (resourceLoader instanceof ZkSolrResourceLoader zkLoader) { SolrZkClient zkClient = zkLoader.getZkController().getZkClient(); String managedSchemaPath = zkLoader.getConfigSetZkPath() + "/" + schemaResourceName; try { diff --git a/solr/core/src/java/org/apache/solr/schema/SortableTextField.java b/solr/core/src/java/org/apache/solr/schema/SortableTextField.java index 9bc6a3f7cb9..6e120dddb5c 100644 --- a/solr/core/src/java/org/apache/solr/schema/SortableTextField.java +++ b/solr/core/src/java/org/apache/solr/schema/SortableTextField.java @@ -101,8 +101,7 @@ public List createFields(SchemaField field, Object value) { if (!field.hasDocValues()) { return Collections.singletonList(f); } - if (value instanceof ByteArrayUtf8CharSequence) { - ByteArrayUtf8CharSequence utf8 = (ByteArrayUtf8CharSequence) value; + if (value instanceof ByteArrayUtf8CharSequence utf8) { if (utf8.size() < maxCharsForDocValues) { BytesRef bytes = new BytesRef(utf8.getBuf(), utf8.offset(), utf8.size()); return getIndexableFields(field, f, bytes); diff --git a/solr/core/src/java/org/apache/solr/schema/StrField.java b/solr/core/src/java/org/apache/solr/schema/StrField.java index 114de6bcaca..a5f7b4643f9 100644 --- a/solr/core/src/java/org/apache/solr/schema/StrField.java +++ b/solr/core/src/java/org/apache/solr/schema/StrField.java @@ -69,8 +69,7 @@ public List createFields(SchemaField field, Object value) { } public static BytesRef getBytesRef(Object value) { - if (value instanceof ByteArrayUtf8CharSequence) { - ByteArrayUtf8CharSequence utf8 = (ByteArrayUtf8CharSequence) value; + if (value instanceof ByteArrayUtf8CharSequence utf8) { return new BytesRef(utf8.getBuf(), utf8.offset(), utf8.size()); } else return new BytesRef(value.toString()); } diff --git a/solr/core/src/java/org/apache/solr/schema/TrieField.java b/solr/core/src/java/org/apache/solr/schema/TrieField.java index 946af8c8eb9..02926704c21 100644 --- a/solr/core/src/java/org/apache/solr/schema/TrieField.java +++ b/solr/core/src/java/org/apache/solr/schema/TrieField.java @@ -697,8 +697,7 @@ public List createFields(SchemaField sf, Object value) { * prefix of the main value of a trie field that indexes multiple precisions per value. */ public static String getMainValuePrefix(org.apache.solr.schema.FieldType ft) { - if (ft instanceof TrieField) { - final TrieField trie = (TrieField) ft; + if (ft instanceof TrieField trie) { if (trie.precisionStep == Integer.MAX_VALUE) return null; switch (trie.type) { case INTEGER: diff --git a/solr/core/src/java/org/apache/solr/schema/ZkIndexSchemaReader.java b/solr/core/src/java/org/apache/solr/schema/ZkIndexSchemaReader.java index faf496789f8..3112bdced58 100644 --- a/solr/core/src/java/org/apache/solr/schema/ZkIndexSchemaReader.java +++ b/solr/core/src/java/org/apache/solr/schema/ZkIndexSchemaReader.java @@ -257,8 +257,7 @@ public int hashCode() { public boolean equals(Object other) { if (other == null) return false; if (other == this) return true; - if (!(other instanceof ZkIndexSchemaReader)) return false; - ZkIndexSchemaReader that = (ZkIndexSchemaReader) other; + if (!(other instanceof ZkIndexSchemaReader that)) return false; return this.managedSchemaPath.equals(that.managedSchemaPath) && this.uniqueCoreId.equals(that.uniqueCoreId); } diff --git a/solr/core/src/java/org/apache/solr/search/BitDocSet.java b/solr/core/src/java/org/apache/solr/search/BitDocSet.java index 831de78cb47..c45cbd5d893 100644 --- a/solr/core/src/java/org/apache/solr/search/BitDocSet.java +++ b/solr/core/src/java/org/apache/solr/search/BitDocSet.java @@ -224,8 +224,7 @@ protected static void andNot(FixedBitSet bits, DocSet other) { @Override public DocSet union(DocSet other) { FixedBitSet newbits = bits.clone(); - if (other instanceof BitDocSet) { - BitDocSet otherDocSet = (BitDocSet) other; + if (other instanceof BitDocSet otherDocSet) { newbits = FixedBitSet.ensureCapacity(newbits, otherDocSet.bits.length()); newbits.or(otherDocSet.bits); } else { diff --git a/solr/core/src/java/org/apache/solr/search/CacheConfig.java b/solr/core/src/java/org/apache/solr/search/CacheConfig.java index 83ce72af836..da05152a86d 100644 --- a/solr/core/src/java/org/apache/solr/search/CacheConfig.java +++ b/solr/core/src/java/org/apache/solr/search/CacheConfig.java @@ -87,7 +87,7 @@ public static Map getMultipleConfigs( for (ConfigNode node : nodes) { if (node.boolAttr("enabled", true)) { CacheConfig config = - getConfig(loader, solrConfig, node.name(), node.attributes().asMap(), configPath); + getConfig(loader, solrConfig, node.name(), node.attributes(), configPath); result.put(config.args.get(NAME), config); } } @@ -98,7 +98,7 @@ public static CacheConfig getConfig(SolrConfig solrConfig, ConfigNode node, Stri if (!node.boolAttr("enabled", true) || !node.exists()) { return null; } - return getConfig(solrConfig, node.name(), node.attributes().asMap(), xpath); + return getConfig(solrConfig, node.name(), node.attributes(), xpath); } public static CacheConfig getConfig( diff --git a/solr/core/src/java/org/apache/solr/search/CaffeineCache.java b/solr/core/src/java/org/apache/solr/search/CaffeineCache.java index 1d921030ff7..091dd984fa5 100644 --- a/solr/core/src/java/org/apache/solr/search/CaffeineCache.java +++ b/solr/core/src/java/org/apache/solr/search/CaffeineCache.java @@ -232,7 +232,7 @@ private V computeAsync(K key, IOFunction mappingFunction // We reserved the slot, so we do the work V value = mappingFunction.apply(key); future.complete(value); // This will update the weight and expiration - recordRamBytes(key, null, value); + recordRamBytes(key, value); inserts.increment(); return value; } catch (Error | RuntimeException | IOException e) { @@ -262,7 +262,7 @@ public V computeIfAbsent(K key, IOFunction mappingFuncti if (value == null) { return null; } - recordRamBytes(key, null, value); + recordRamBytes(key, value); inserts.increment(); return value; }); @@ -275,30 +275,34 @@ public V computeIfAbsent(K key, IOFunction mappingFuncti public V put(K key, V val) { inserts.increment(); V old = cache.asMap().put(key, val); - recordRamBytes(key, old, val); + // ramBytes decrement for `old` happens via #onRemoval + if (val != old) { + // NOTE: this is conditional on `val != old` in order to work around a + // behavior in the Caffeine library: where there is reference equality + // between `val` and `old`, caffeine does _not_ invoke RemovalListener, + // so the entry is not decremented for the replaced value (hence we + // don't need to increment ram bytes for the entry either). + recordRamBytes(key, val); + } return old; } /** - * Update the estimate of used memory + * Update the estimate of used memory. + * + *

    NOTE: old value (in the event of replacement) adjusts {@link #ramBytes} via {@link + * #onRemoval(Object, Object, RemovalCause)} * * @param key the cache key - * @param oldValue the old cached value to decrement estimate (can be null) * @param newValue the new cached value to increment estimate */ - private void recordRamBytes(K key, V oldValue, V newValue) { + private void recordRamBytes(K key, V newValue) { ramBytes.add( RamUsageEstimator.sizeOfObject(newValue, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED)); - if (oldValue == null) { - ramBytes.add( - RamUsageEstimator.sizeOfObject(key, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED)); - ramBytes.add(RamUsageEstimator.LINKED_HASHTABLE_RAM_BYTES_PER_ENTRY); - if (async) ramBytes.add(RAM_BYTES_PER_FUTURE); - } else { - ramBytes.add( - -RamUsageEstimator.sizeOfObject( - oldValue, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED)); - } + ramBytes.add( + RamUsageEstimator.sizeOfObject(key, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED)); + ramBytes.add(RamUsageEstimator.LINKED_HASHTABLE_RAM_BYTES_PER_ENTRY); + if (async) ramBytes.add(RAM_BYTES_PER_FUTURE); } @Override diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index 221f2c8b06b..119cc422a1e 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -249,8 +249,7 @@ private GroupHeadSelector(String s, GroupHeadSelectorType type) { @Override public boolean equals(final Object other) { - if (other instanceof GroupHeadSelector) { - final GroupHeadSelector that = (GroupHeadSelector) other; + if (other instanceof GroupHeadSelector that) { return (this.type == that.type) && this.selectorText.equals(that.selectorText); } return false; diff --git a/solr/core/src/java/org/apache/solr/search/DisMaxQParser.java b/solr/core/src/java/org/apache/solr/search/DisMaxQParser.java index b6ae705704e..9f90831e820 100644 --- a/solr/core/src/java/org/apache/solr/search/DisMaxQParser.java +++ b/solr/core/src/java/org/apache/solr/search/DisMaxQParser.java @@ -151,8 +151,7 @@ protected void addBoostQuery(BooleanQuery.Builder query, SolrParams solrParams) if (1 == boostQueries.size() && 1 == boostParams.length) { /* legacy logic */ Query f = boostQueries.get(0); - while (f instanceof BoostQuery) { - BoostQuery bq = (BoostQuery) f; + while (f instanceof BoostQuery bq) { if (bq.getBoost() == 1f) { f = bq.getQuery(); } else { diff --git a/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java b/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java index 3d593669ed5..535e2c493bf 100644 --- a/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java +++ b/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java @@ -1236,8 +1236,7 @@ && allSameQueryStructure(lst)) { subs.clear(); // Make a dismax query for each clause position in the boolean per-field queries. for (int n = 0; n < lst.size(); ++n) { - if (lst.get(n) instanceof BoostQuery) { - BoostQuery boostQuery = (BoostQuery) lst.get(n); + if (lst.get(n) instanceof BoostQuery boostQuery) { BooleanQuery booleanQuery = (BooleanQuery) boostQuery.getQuery(); subs.add( new BoostQuery( @@ -1440,8 +1439,7 @@ private Query getQuery() { BooleanQuery bq = (BooleanQuery) query; query = SolrPluginUtils.setMinShouldMatch(bq, minShouldMatch, false); } - } else if (query instanceof PhraseQuery) { - PhraseQuery pq = (PhraseQuery) query; + } else if (query instanceof PhraseQuery pq) { if (minClauseSize > 1 && pq.getTerms().length < minClauseSize) return null; PhraseQuery.Builder builder = new PhraseQuery.Builder(); Term[] terms = pq.getTerms(); @@ -1451,8 +1449,7 @@ private Query getQuery() { } builder.setSlop(slop); query = builder.build(); - } else if (query instanceof MultiPhraseQuery) { - MultiPhraseQuery mpq = (MultiPhraseQuery) query; + } else if (query instanceof MultiPhraseQuery mpq) { if (minClauseSize > 1 && mpq.getTermArrays().length < minClauseSize) return null; if (slop != mpq.getSlop()) { query = new MultiPhraseQuery.Builder(mpq).setSlop(slop).build(); @@ -1485,16 +1482,14 @@ private Query getQuery() { private Analyzer noStopwordFilterAnalyzer(String fieldName) { FieldType ft = parser.getReq().getSchema().getFieldType(fieldName); Analyzer qa = ft.getQueryAnalyzer(); - if (!(qa instanceof TokenizerChain)) { + if (!(qa instanceof TokenizerChain tcq)) { return qa; } - TokenizerChain tcq = (TokenizerChain) qa; Analyzer ia = ft.getIndexAnalyzer(); - if (ia == qa || !(ia instanceof TokenizerChain)) { + if (ia == qa || !(ia instanceof TokenizerChain tci)) { return qa; } - TokenizerChain tci = (TokenizerChain) ia; // make sure that there isn't a stop filter in the indexer for (TokenFilterFactory tf : tci.getTokenFilterFactories()) { diff --git a/solr/core/src/java/org/apache/solr/search/FloatPayloadValueSource.java b/solr/core/src/java/org/apache/solr/search/FloatPayloadValueSource.java index d6572edd31d..a8f5b5112ea 100644 --- a/solr/core/src/java/org/apache/solr/search/FloatPayloadValueSource.java +++ b/solr/core/src/java/org/apache/solr/search/FloatPayloadValueSource.java @@ -221,9 +221,8 @@ public String description() { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof FloatPayloadValueSource)) return false; + if (!(o instanceof FloatPayloadValueSource that)) return false; - FloatPayloadValueSource that = (FloatPayloadValueSource) o; return Objects.equals(indexedField, that.indexedField) && Objects.equals(indexedBytes, that.indexedBytes) && Objects.equals(decoder, that.decoder) diff --git a/solr/core/src/java/org/apache/solr/search/FunctionQParser.java b/solr/core/src/java/org/apache/solr/search/FunctionQParser.java index 2a3dcd9dc68..97b1fb35d22 100644 --- a/solr/core/src/java/org/apache/solr/search/FunctionQParser.java +++ b/solr/core/src/java/org/apache/solr/search/FunctionQParser.java @@ -424,8 +424,7 @@ protected ValueSource parseValueSource(int flags) throws SyntaxError { valueSource = new FieldNameValueSource(val); } else { QParser subParser = subQuery(val, "func"); - if (subParser instanceof FunctionQParser) { - FunctionQParser subFunc = (FunctionQParser) subParser; + if (subParser instanceof FunctionQParser subFunc) { subFunc.setParseMultipleSources(true); subFunc.setFlags(flags); } diff --git a/solr/core/src/java/org/apache/solr/search/HashQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/HashQParserPlugin.java index 89c012654ee..6ba9a7d66a4 100644 --- a/solr/core/src/java/org/apache/solr/search/HashQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/HashQParserPlugin.java @@ -193,8 +193,7 @@ public boolean needsScores() { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof HashCodeValuesSource)) return false; - HashCodeValuesSource that = (HashCodeValuesSource) o; + if (!(o instanceof HashCodeValuesSource that)) return false; return Arrays.equals(fields, that.fields); } @@ -240,8 +239,7 @@ public boolean test(double hashAsDouble) { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof HashPartitionPredicate)) return false; - HashPartitionPredicate that = (HashPartitionPredicate) o; + if (!(o instanceof HashPartitionPredicate that)) return false; return workers == that.workers && worker == that.worker; } diff --git a/solr/core/src/java/org/apache/solr/search/IGainTermsQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/IGainTermsQParserPlugin.java index 011e22c0f6c..eba8327360a 100644 --- a/solr/core/src/java/org/apache/solr/search/IGainTermsQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/IGainTermsQParserPlugin.java @@ -230,8 +230,7 @@ public int hashCode() { @Override public boolean equals(Object obj) { - if (!(obj instanceof TermWithScore)) return false; - TermWithScore other = (TermWithScore) obj; + if (!(obj instanceof TermWithScore other)) return false; return Objects.equals(this.term, other.term); } diff --git a/solr/core/src/java/org/apache/solr/search/MaxScoreQParser.java b/solr/core/src/java/org/apache/solr/search/MaxScoreQParser.java index 8cbceeefad5..eef5813e953 100644 --- a/solr/core/src/java/org/apache/solr/search/MaxScoreQParser.java +++ b/solr/core/src/java/org/apache/solr/search/MaxScoreQParser.java @@ -54,8 +54,7 @@ public MaxScoreQParser( public Query parse() throws SyntaxError { Query q = super.parse(); float boost = 1f; - if (q instanceof BoostQuery) { - BoostQuery bq = (BoostQuery) q; + if (q instanceof BoostQuery bq) { boost = bq.getBoost(); q = bq.getQuery(); } diff --git a/solr/core/src/java/org/apache/solr/search/MultiThreadedSearcher.java b/solr/core/src/java/org/apache/solr/search/MultiThreadedSearcher.java index c4bdd921e7a..b7b4d7a0325 100644 --- a/solr/core/src/java/org/apache/solr/search/MultiThreadedSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/MultiThreadedSearcher.java @@ -280,8 +280,7 @@ public Collector newCollector() throws IOException { public Object reduce(Collection collectors) throws IOException { final FixedBitSet reduced = new FixedBitSet(maxDoc); for (Object collector : collectors) { - if (collector instanceof FixedBitSetCollector) { - FixedBitSetCollector fixedBitSetCollector = (FixedBitSetCollector) collector; + if (collector instanceof FixedBitSetCollector fixedBitSetCollector) { fixedBitSetCollector.update(reduced); } } diff --git a/solr/core/src/java/org/apache/solr/search/MutableBitDocSet.java b/solr/core/src/java/org/apache/solr/search/MutableBitDocSet.java index 8b1f7301ed2..73b26770353 100644 --- a/solr/core/src/java/org/apache/solr/search/MutableBitDocSet.java +++ b/solr/core/src/java/org/apache/solr/search/MutableBitDocSet.java @@ -50,8 +50,7 @@ public static MutableBitDocSet fromBitDocSet(BitDocSet bitDocSet) { * @return Unwrapped DocSet that is not mutable */ public static DocSet unwrapIfMutable(DocSet docSet) { - if (docSet instanceof MutableBitDocSet) { - MutableBitDocSet mutableBitDocSet = (MutableBitDocSet) docSet; + if (docSet instanceof MutableBitDocSet mutableBitDocSet) { // don't call size() since we just want to pass through the size // instead of computing it if it was already computed return new BitDocSet(mutableBitDocSet.getBits(), mutableBitDocSet.size); diff --git a/solr/core/src/java/org/apache/solr/search/QueryLimits.java b/solr/core/src/java/org/apache/solr/search/QueryLimits.java index 7aee5657678..e6e0db5eed9 100644 --- a/solr/core/src/java/org/apache/solr/search/QueryLimits.java +++ b/solr/core/src/java/org/apache/solr/search/QueryLimits.java @@ -16,12 +16,14 @@ */ package org.apache.solr.search; +import static org.apache.solr.response.SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_DETAILS_KEY; import static org.apache.solr.search.CpuAllowedLimit.hasCpuLimit; import static org.apache.solr.search.TimeAllowedLimit.hasTimeLimit; import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.function.Supplier; import org.apache.lucene.index.QueryTimeout; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CommonParams; @@ -113,7 +115,8 @@ public String formatExceptionMessage(String label) { * @throws QueryLimitsExceededException if {@link #allowPartialResults} is false and limits have * been reached. */ - public boolean maybeExitWithPartialResults(String label) throws QueryLimitsExceededException { + public boolean maybeExitWithPartialResults(Supplier label) + throws QueryLimitsExceededException { if (isLimitsEnabled() && shouldExit()) { if (allowPartialResults) { if (rsp != null) { @@ -124,17 +127,25 @@ public boolean maybeExitWithPartialResults(String label) throws QueryLimitsExcee "No request active, but attempting to exit with partial results?"); } rsp.setPartialResults(requestInfo.getReq()); - rsp.addPartialResponseDetail(formatExceptionMessage(label)); + if (rsp.getResponseHeader().get(RESPONSE_HEADER_PARTIAL_RESULTS_DETAILS_KEY) == null) { + // don't want to add duplicate keys. Although technically legal, there's a strong risk + // that clients won't anticipate it and break. + rsp.addPartialResponseDetail(formatExceptionMessage(label.get())); + } } return true; } else { - throw new QueryLimitsExceededException(formatExceptionMessage(label)); + throw new QueryLimitsExceededException(formatExceptionMessage(label.get())); } } else { return false; } } + public boolean maybeExitWithPartialResults(String label) throws QueryLimitsExceededException { + return maybeExitWithPartialResults(() -> label); + } + /** * Method to diagnose limit exceeded. Note that while this should always list the exceeded limit, * it may also nominate additional limits that have been exceeded since the actual check that diff --git a/solr/core/src/java/org/apache/solr/search/QueryParsing.java b/solr/core/src/java/org/apache/solr/search/QueryParsing.java index 2abe8b8f6d3..5b635029c6f 100644 --- a/solr/core/src/java/org/apache/solr/search/QueryParsing.java +++ b/solr/core/src/java/org/apache/solr/search/QueryParsing.java @@ -230,13 +230,11 @@ public static void toString(Query query, IndexSchema schema, Appendable out, int // clear the boosted / is clause flags for recursion int subflag = flags & ~(FLAG_BOOSTED | FLAG_IS_CLAUSE); - if (query instanceof TermQuery) { - TermQuery q = (TermQuery) query; + if (query instanceof TermQuery q) { Term t = q.getTerm(); FieldType ft = writeFieldName(t.field(), schema, out, flags); writeFieldVal(t.bytes(), ft, out, flags); - } else if (query instanceof TermRangeQuery) { - TermRangeQuery q = (TermRangeQuery) query; + } else if (query instanceof TermRangeQuery q) { String fname = q.getField(); FieldType ft = writeFieldName(fname, schema, out, flags); out.append(q.includesLower() ? '[' : '{'); @@ -257,8 +255,7 @@ public static void toString(Query query, IndexSchema schema, Appendable out, int } out.append(q.includesUpper() ? ']' : '}'); - } else if (query instanceof LegacyNumericRangeQuery) { - LegacyNumericRangeQuery q = (LegacyNumericRangeQuery) query; + } else if (query instanceof LegacyNumericRangeQuery q) { String fname = q.getField(); FieldType ft = writeFieldName(fname, schema, out, flags); out.append(q.includesMin() ? '[' : '{'); @@ -279,8 +276,7 @@ public static void toString(Query query, IndexSchema schema, Appendable out, int } out.append(q.includesMax() ? ']' : '}'); - } else if (query instanceof BooleanQuery) { - BooleanQuery q = (BooleanQuery) query; + } else if (query instanceof BooleanQuery q) { boolean needParens = false; if (q.getMinimumNumberShouldMatch() != 0 || (flags & (FLAG_IS_CLAUSE | FLAG_BOOSTED)) != 0) { @@ -315,8 +311,7 @@ public static void toString(Query query, IndexSchema schema, Appendable out, int out.append(Integer.toString(q.getMinimumNumberShouldMatch())); } - } else if (query instanceof PrefixQuery) { - PrefixQuery q = (PrefixQuery) query; + } else if (query instanceof PrefixQuery q) { Term prefix = q.getPrefix(); FieldType ft = writeFieldName(prefix.field(), schema, out, flags); out.append(prefix.text()); @@ -327,12 +322,10 @@ public static void toString(Query query, IndexSchema schema, Appendable out, int out.append(query.toString()); } else if (query instanceof ConstantScoreQuery) { out.append(query.toString()); - } else if (query instanceof WrappedQuery) { - WrappedQuery q = (WrappedQuery) query; + } else if (query instanceof WrappedQuery q) { out.append(q.getOptions()); toString(q.getWrappedQuery(), schema, out, subflag); - } else if (query instanceof BoostQuery) { - BoostQuery q = (BoostQuery) query; + } else if (query instanceof BoostQuery q) { toString(q.getQuery(), schema, out, subflag | FLAG_BOOSTED); out.append('^'); out.append(Float.toString(q.getBoost())); diff --git a/solr/core/src/java/org/apache/solr/search/QueryResultKey.java b/solr/core/src/java/org/apache/solr/search/QueryResultKey.java index 8d75c5bb6a6..671da8cda54 100644 --- a/solr/core/src/java/org/apache/solr/search/QueryResultKey.java +++ b/solr/core/src/java/org/apache/solr/search/QueryResultKey.java @@ -110,8 +110,7 @@ public int hashCode() { @Override public boolean equals(Object o) { if (o == this) return true; - if (!(o instanceof QueryResultKey)) return false; - QueryResultKey other = (QueryResultKey) o; + if (!(o instanceof QueryResultKey other)) return false; // fast check of the whole hash code... most hash tables will only use // some of the bits, so if this is a hash collision, it's still likely diff --git a/solr/core/src/java/org/apache/solr/search/QueryUtils.java b/solr/core/src/java/org/apache/solr/search/QueryUtils.java index ba02c34819c..954ed91b0d4 100644 --- a/solr/core/src/java/org/apache/solr/search/QueryUtils.java +++ b/solr/core/src/java/org/apache/solr/search/QueryUtils.java @@ -44,8 +44,7 @@ public class QueryUtils { /** return true if this query has no positive components */ public static boolean isNegative(Query q) { - if (!(q instanceof BooleanQuery)) return false; - BooleanQuery bq = (BooleanQuery) q; + if (!(q instanceof BooleanQuery bq)) return false; Collection clauses = bq.clauses(); if (clauses.size() == 0) return false; for (BooleanClause clause : clauses) { @@ -140,8 +139,7 @@ public static void ensurePrefixQueryObeysMinimumPrefixLength( * @return Absolute version of the Query */ public static Query getAbs(Query q) { - if (q instanceof BoostQuery) { - BoostQuery bq = (BoostQuery) q; + if (q instanceof BoostQuery bq) { Query subQ = bq.getQuery(); Query absSubQ = getAbs(subQ); if (absSubQ.equals(subQ)) return q; @@ -155,8 +153,7 @@ public static Query getAbs(Query q) { return new WrappedQuery(absSubQ); } - if (!(q instanceof BooleanQuery)) return q; - BooleanQuery bq = (BooleanQuery) q; + if (!(q instanceof BooleanQuery bq)) return q; Collection clauses = bq.clauses(); if (clauses.size() == 0) return q; @@ -198,8 +195,7 @@ public static Query makeQueryable(Query q) { */ public static Query fixNegativeQuery(Query q) { float boost = 1f; - if (q instanceof BoostQuery) { - BoostQuery bq = (BoostQuery) q; + if (q instanceof BoostQuery bq) { boost = bq.getBoost(); q = bq.getQuery(); } @@ -326,8 +322,7 @@ public static Set getTaggedQueries(SolrQueryRequest req, Collection) tagVal) { - if (!(obj instanceof QParser)) continue; - QParser qParser = (QParser) obj; + if (!(obj instanceof QParser qParser)) continue; Query query; try { query = qParser.getQuery(); diff --git a/solr/core/src/java/org/apache/solr/search/ReRankScaler.java b/solr/core/src/java/org/apache/solr/search/ReRankScaler.java index 8410467ddf8..3a03e13fc65 100644 --- a/solr/core/src/java/org/apache/solr/search/ReRankScaler.java +++ b/solr/core/src/java/org/apache/solr/search/ReRankScaler.java @@ -84,8 +84,7 @@ public int hashCode() { @Override public boolean equals(Object o) { - if (o instanceof ReRankScaler) { - ReRankScaler _reRankScaler = (ReRankScaler) o; + if (o instanceof ReRankScaler _reRankScaler) { if (mainQueryMin == _reRankScaler.mainQueryMin && mainQueryMax == _reRankScaler.mainQueryMax && reRankQueryMin == _reRankScaler.reRankQueryMin diff --git a/solr/core/src/java/org/apache/solr/search/SignificantTermsQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/SignificantTermsQParserPlugin.java index 4e6c09f38f6..3fdb46ee5a2 100644 --- a/solr/core/src/java/org/apache/solr/search/SignificantTermsQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/SignificantTermsQParserPlugin.java @@ -291,8 +291,7 @@ public int hashCode() { @Override public boolean equals(Object obj) { - if (!(obj instanceof TermWithScore)) return false; - TermWithScore other = (TermWithScore) obj; + if (!(obj instanceof TermWithScore other)) return false; return Objects.equals(this.term, other.term); } diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index 3c8afde39ff..19f93cc49fa 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -1230,8 +1230,7 @@ public ProcessedFilter getProcessedFilter(List queries) throws IOExceptio int end = 0; // size of "sets" and "neg"; parallel arrays for (Query q : queries) { - if (q instanceof ExtendedQuery) { - ExtendedQuery eq = (ExtendedQuery) q; + if (q instanceof ExtendedQuery eq) { if (!eq.getCache()) { if (eq.getCost() >= 100 && eq instanceof PostFilter) { if (postFilters == null) postFilters = new ArrayList<>(sets.length - end); @@ -1608,8 +1607,7 @@ private QueryResult getDocListC(QueryResult qr, QueryCommand cmd) throws IOExcep int flags = cmd.getFlags(); Query q = cmd.getQuery(); - if (q instanceof ExtendedQuery) { - ExtendedQuery eq = (ExtendedQuery) q; + if (q instanceof ExtendedQuery eq) { if (!eq.getCache()) { flags |= (NO_CHECK_QCACHE | NO_SET_QCACHE | NO_CHECK_FILTERCACHE); } @@ -1858,8 +1856,7 @@ TopDocsCollector buildTopDocsCollector(int len, QueryCommand throws IOException { int minNumFound = cmd.getMinExactCount(); Query q = cmd.getQuery(); - if (q instanceof RankQuery) { - RankQuery rq = (RankQuery) q; + if (q instanceof RankQuery rq) { return rq.getTopDocsCollector(len, cmd, this); } diff --git a/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java b/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java index af35245af15..baaaeb270ef 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java +++ b/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java @@ -381,8 +381,7 @@ private void add( ValueSource vs = null; try { - if (parser instanceof FunctionQParser) { - FunctionQParser fparser = (FunctionQParser) parser; + if (parser instanceof FunctionQParser fparser) { fparser.setParseMultipleSources(false); fparser.setParseToEnd(false); diff --git a/solr/core/src/java/org/apache/solr/search/SortSpecParsing.java b/solr/core/src/java/org/apache/solr/search/SortSpecParsing.java index 6b71f139133..ef012a5fed6 100644 --- a/solr/core/src/java/org/apache/solr/search/SortSpecParsing.java +++ b/solr/core/src/java/org/apache/solr/search/SortSpecParsing.java @@ -115,8 +115,7 @@ private static SortSpec parseSortSpecImpl( QParser parser = QParser.getParser(funcStr, FunctionQParserPlugin.NAME, optionalReq); Query q = null; try { - if (parser instanceof FunctionQParser) { - FunctionQParser fparser = (FunctionQParser) parser; + if (parser instanceof FunctionQParser fparser) { fparser.setParseMultipleSources(false); fparser.setParseToEnd(false); diff --git a/solr/core/src/java/org/apache/solr/search/SpatialFilterQParser.java b/solr/core/src/java/org/apache/solr/search/SpatialFilterQParser.java index 1bb1ca343b0..97224284122 100644 --- a/solr/core/src/java/org/apache/solr/search/SpatialFilterQParser.java +++ b/solr/core/src/java/org/apache/solr/search/SpatialFilterQParser.java @@ -73,8 +73,7 @@ public Query parse() throws SyntaxError { SchemaField sf = req.getSchema().getField(fields[0]); FieldType type = sf.getType(); - if (type instanceof SpatialQueryable) { - SpatialQueryable queryable = ((SpatialQueryable) type); + if (type instanceof SpatialQueryable queryable) { double radius = localParams.getDouble(SpatialParams.SPHERE_RADIUS, queryable.getSphereRadius()); SpatialOptions opts = new SpatialOptions(pointStr, dist, sf, measStr, radius); diff --git a/solr/core/src/java/org/apache/solr/search/TopLevelJoinQuery.java b/solr/core/src/java/org/apache/solr/search/TopLevelJoinQuery.java index ea7b55a2e75..ce05a02eb95 100644 --- a/solr/core/src/java/org/apache/solr/search/TopLevelJoinQuery.java +++ b/solr/core/src/java/org/apache/solr/search/TopLevelJoinQuery.java @@ -56,14 +56,13 @@ public TopLevelJoinQuery(String fromField, String toField, String coreName, Quer @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { - if (!(searcher instanceof SolrIndexSearcher)) { + if (!(searcher instanceof SolrIndexSearcher solrSearcher)) { log.debug( "Falling back to JoinQueryWeight because searcher [{}] is not the required SolrIndexSearcher", searcher); return super.createWeight(searcher, scoreMode, boost); } - final SolrIndexSearcher solrSearcher = (SolrIndexSearcher) searcher; final JoinQueryWeight weight = new JoinQueryWeight(solrSearcher, ScoreMode.COMPLETE_NO_SCORES, 1.0f); final SolrIndexSearcher fromSearcher = weight.fromSearcher; diff --git a/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java b/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java index 3e4a720e5ce..2cd071a8938 100644 --- a/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java +++ b/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java @@ -541,12 +541,11 @@ public ValueSource parse(FunctionQParser fp) throws SyntaxError { String fieldName = fp.parseArg(); SchemaField f = fp.getReq().getSchema().getField(fieldName); - if (!(f.getType() instanceof CurrencyFieldType)) { + if (!(f.getType() instanceof CurrencyFieldType ft)) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Currency function input must be the name of a CurrencyFieldType: " + fieldName); } - CurrencyFieldType ft = (CurrencyFieldType) f.getType(); String code = fp.hasMoreArguments() ? fp.parseArg() : null; return ft.getConvertedValueSource(code, ft.getValueSource(f, fp)); } @@ -1606,8 +1605,7 @@ public int hashCode() { @Override public boolean equals(Object o) { - if (!(o instanceof LongConstValueSource)) return false; - LongConstValueSource other = (LongConstValueSource) o; + if (!(o instanceof LongConstValueSource other)) return false; return this.constant == other.constant; } @@ -1759,8 +1757,7 @@ public int hashCode() { @Override public boolean equals(Object o) { - if (!(o instanceof Function)) return false; - Function other = (Function) o; + if (!(o instanceof Function other)) return false; return this.a.equals(other.a) && this.b.equals(other.b); } } @@ -1799,8 +1796,7 @@ public int hashCode() { @Override public boolean equals(Object o) { - if (!(o instanceof BoolConstValueSource)) return false; - BoolConstValueSource other = (BoolConstValueSource) o; + if (!(o instanceof BoolConstValueSource other)) return false; return this.constant == other.constant; } diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java index ff3a2553ca4..3e8d3020294 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java @@ -895,8 +895,8 @@ public SlotAcc registerSweepingAccs(SweepingCountSlotAcc baseSweepingAcc) { * @see SweepingCountSlotAcc */ protected boolean registerSweepingAccIfSupportedByCollectAcc() { - if (countAcc instanceof SweepingCountSlotAcc && collectAcc instanceof SweepableSlotAcc) { - final SweepingCountSlotAcc sweepingCountAcc = (SweepingCountSlotAcc) countAcc; + if (countAcc instanceof SweepingCountSlotAcc sweepingCountAcc + && collectAcc instanceof SweepableSlotAcc) { collectAcc = ((SweepableSlotAcc) collectAcc).registerSweepingAccs(sweepingCountAcc); if (allBucketsAcc != null) { allBucketsAcc.collectAcc = collectAcc; diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArrayDV.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArrayDV.java index cb7d5492a91..6dde1e24c85 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArrayDV.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArrayDV.java @@ -225,9 +225,7 @@ private void collectPerSeg(SortedDocValues singleDv, SweepDISI disi, LongValues // calculate segment-local counts int doc; - if (singleDv instanceof FieldCacheImpl.SortedDocValuesImpl.Iter) { - FieldCacheImpl.SortedDocValuesImpl.Iter fc = - (FieldCacheImpl.SortedDocValuesImpl.Iter) singleDv; + if (singleDv instanceof FieldCacheImpl.SortedDocValuesImpl.Iter fc) { while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { final int segOrd = fc.getOrd(doc); if (segOrd >= 0) { @@ -340,10 +338,8 @@ private void collectCounts(SortedDocValues singleDv, SweepDISI disi, LongValues throws IOException { final SegCountGlobal segCounter = getSegCountGlobal(disi, singleDv); int doc; - if (singleDv instanceof FieldCacheImpl.SortedDocValuesImpl.Iter) { + if (singleDv instanceof FieldCacheImpl.SortedDocValuesImpl.Iter fc) { - FieldCacheImpl.SortedDocValuesImpl.Iter fc = - (FieldCacheImpl.SortedDocValuesImpl.Iter) singleDv; while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { int segOrd = fc.getOrd(doc); if (segOrd < 0) continue; diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashDV.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashDV.java index 473f70c9d2d..a61230e9242 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashDV.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashDV.java @@ -369,9 +369,7 @@ public ScoreMode scoreMode() { @Override protected void doSetNextReader(LeafReaderContext ctx) throws IOException { setNextReaderFirstPhase(ctx); - if (globalDocValues instanceof MultiDocValues.MultiSortedDocValues) { - MultiDocValues.MultiSortedDocValues multiDocValues = - (MultiDocValues.MultiSortedDocValues) globalDocValues; + if (globalDocValues instanceof MultiDocValues.MultiSortedDocValues multiDocValues) { docValues = multiDocValues.values[ctx.ord]; toGlobal = multiDocValues.mapping.getGlobalOrds(ctx.ord); } diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetHeatmap.java b/solr/core/src/java/org/apache/solr/search/facet/FacetHeatmap.java index 4a13ddc51af..57d69d47c52 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetHeatmap.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetHeatmap.java @@ -129,13 +129,10 @@ public FacetHeatmap parse(Object argsObj) { final DistanceUnits distanceUnits; // note: the two instanceof conditions is not ideal, versus one. If we start needing to add // more then refactor. - if ((type instanceof AbstractSpatialPrefixTreeFieldType)) { - AbstractSpatialPrefixTreeFieldType rptType = - (AbstractSpatialPrefixTreeFieldType) type; + if ((type instanceof AbstractSpatialPrefixTreeFieldType rptType)) { strategy = rptType.getStrategy(fieldName); distanceUnits = rptType.getDistanceUnits(); - } else if (type instanceof RptWithGeometrySpatialField) { - RptWithGeometrySpatialField rptSdvType = (RptWithGeometrySpatialField) type; + } else if (type instanceof RptWithGeometrySpatialField rptSdvType) { strategy = rptSdvType.getStrategy(fieldName).getIndexStrategy(); distanceUnits = rptSdvType.getDistanceUnits(); } else { diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetParser.java b/solr/core/src/java/org/apache/solr/search/facet/FacetParser.java index ce1276788d5..92d6ef2fb7d 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetParser.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetParser.java @@ -612,8 +612,7 @@ private static FacetRequest.FacetSort parseAndValidateSort( FacetRequest.FacetSort facetSort = null; - if (sort instanceof String) { - String sortStr = (String) sort; + if (sort instanceof String sortStr) { if (sortStr.endsWith(" asc")) { facetSort = new FacetRequest.FacetSort( diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetRangeProcessor.java b/solr/core/src/java/org/apache/solr/search/facet/FacetRangeProcessor.java index 3bcb5349ff2..554f6f762b3 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetRangeProcessor.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetRangeProcessor.java @@ -297,12 +297,11 @@ private void createRangeList() { * @return list of {@link Range} */ private List parseRanges(Object input) { - if (!(input instanceof List)) { + if (!(input instanceof List intervals)) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Expected List for ranges but got " + input.getClass().getSimpleName() + " = " + input); } - List intervals = (List) input; List ranges = new ArrayList<>(); for (Object obj : intervals) { if (!(obj instanceof Map)) { diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java b/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java index 5b8fec842c9..55c852d098c 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java @@ -61,8 +61,7 @@ public FacetSort(final String sortVariable, final SortDirection sortDirection) { @Override public boolean equals(Object other) { - if (other instanceof FacetSort) { - final FacetSort that = (FacetSort) other; + if (other instanceof FacetSort that) { return this.sortVariable.equals(that.sortVariable) && this.sortDirection.equals(that.sortDirection); } diff --git a/solr/core/src/java/org/apache/solr/search/facet/FieldUtil.java b/solr/core/src/java/org/apache/solr/search/facet/FieldUtil.java index 07abc6c6ab1..cdc72532863 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FieldUtil.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FieldUtil.java @@ -84,9 +84,7 @@ public static boolean isFieldCache(SortedDocValues singleDv) { public static void visitOrds(SortedDocValues singleDv, DocIdSetIterator disi, OrdFunc ordFunc) throws IOException { int doc; - if (singleDv instanceof FieldCacheImpl.SortedDocValuesImpl.Iter) { - FieldCacheImpl.SortedDocValuesImpl.Iter fc = - (FieldCacheImpl.SortedDocValuesImpl.Iter) singleDv; + if (singleDv instanceof FieldCacheImpl.SortedDocValuesImpl.Iter fc) { while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { ordFunc.handleOrd(doc, fc.getOrd(doc)); } @@ -102,9 +100,7 @@ public static void visitOrds(SortedDocValues singleDv, DocIdSetIterator disi, Or } public static OrdValues getOrdValues(SortedDocValues singleDv, DocIdSetIterator disi) { - if (singleDv instanceof FieldCacheImpl.SortedDocValuesImpl.Iter) { - FieldCacheImpl.SortedDocValuesImpl.Iter fc = - (FieldCacheImpl.SortedDocValuesImpl.Iter) singleDv; + if (singleDv instanceof FieldCacheImpl.SortedDocValuesImpl.Iter fc) { return new FCOrdValues(fc, disi); } return new DVOrdValues(singleDv, disi); diff --git a/solr/core/src/java/org/apache/solr/search/facet/PercentileAgg.java b/solr/core/src/java/org/apache/solr/search/facet/PercentileAgg.java index 1a63f9fa7bf..0d0a842f99e 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/PercentileAgg.java +++ b/solr/core/src/java/org/apache/solr/search/facet/PercentileAgg.java @@ -86,8 +86,7 @@ public FacetMerger createFacetMerger(Object prototype) { @Override public boolean equals(Object o) { - if (!(o instanceof PercentileAgg)) return false; - PercentileAgg other = (PercentileAgg) o; + if (!(o instanceof PercentileAgg other)) return false; return this.arg.equals(other.arg) && this.percentiles.equals(other.percentiles); } diff --git a/solr/core/src/java/org/apache/solr/search/facet/RelatednessAgg.java b/solr/core/src/java/org/apache/solr/search/facet/RelatednessAgg.java index daba3e5ec0e..f55690bbd55 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/RelatednessAgg.java +++ b/solr/core/src/java/org/apache/solr/search/facet/RelatednessAgg.java @@ -118,10 +118,9 @@ public String description() { @Override public boolean equals(Object o) { - if (!(o instanceof RelatednessAgg)) { + if (!(o instanceof RelatednessAgg that)) { return false; } - RelatednessAgg that = (RelatednessAgg) o; return Objects.equals(fgQ, that.fgQ) && Objects.equals(bgQ, that.bgQ) && min_pop == that.min_pop; @@ -571,10 +570,9 @@ public int hashCode() { @Override public boolean equals(Object other) { - if (!(other instanceof BucketData)) { + if (!(other instanceof BucketData that)) { return false; } - BucketData that = (BucketData) other; // we will most certainly be compared to other buckets of the same Agg instance, so compare // counts first return this.implied == that.implied diff --git a/solr/core/src/java/org/apache/solr/search/function/DualDoubleFunction.java b/solr/core/src/java/org/apache/solr/search/function/DualDoubleFunction.java index 718e81d9528..1d2371dbd33 100644 --- a/solr/core/src/java/org/apache/solr/search/function/DualDoubleFunction.java +++ b/solr/core/src/java/org/apache/solr/search/function/DualDoubleFunction.java @@ -98,8 +98,7 @@ public int hashCode() { @Override public boolean equals(Object o) { - if (!(o instanceof DualDoubleFunction)) return false; - DualDoubleFunction other = (DualDoubleFunction) o; + if (!(o instanceof DualDoubleFunction other)) return false; return Objects.equals(this.a, other.a) && Objects.equals(this.b, other.b); } } diff --git a/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java b/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java index 7664a822bcb..2d38957912d 100644 --- a/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java +++ b/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java @@ -109,8 +109,7 @@ public Object objectVal(int doc) { @Override public boolean equals(Object o) { - if (!(o instanceof FileFloatSource)) return false; - FileFloatSource other = (FileFloatSource) o; + if (!(o instanceof FileFloatSource other)) return false; return this.field.getName().equals(other.field.getName()) && this.keyField.getName().equals(other.keyField.getName()) && this.defVal == other.defVal @@ -242,8 +241,7 @@ public Entry(FileFloatSource ffs) { @Override public boolean equals(Object o) { - if (!(o instanceof Entry)) return false; - Entry other = (Entry) o; + if (!(o instanceof Entry other)) return false; return ffs.equals(other.ffs); } diff --git a/solr/core/src/java/org/apache/solr/search/function/MultiStringFunction.java b/solr/core/src/java/org/apache/solr/search/function/MultiStringFunction.java index 6379beb004c..19744e9da7d 100644 --- a/solr/core/src/java/org/apache/solr/search/function/MultiStringFunction.java +++ b/solr/core/src/java/org/apache/solr/search/function/MultiStringFunction.java @@ -134,8 +134,7 @@ public void fillValue(int doc) throws IOException { @Override public boolean equals(Object o) { - if (!(o instanceof MultiStringFunction)) return false; - MultiStringFunction other = (MultiStringFunction) o; + if (!(o instanceof MultiStringFunction other)) return false; return Objects.equals(this.name(), other.name()) && Arrays.equals(this.sources, other.sources); } diff --git a/solr/core/src/java/org/apache/solr/search/function/OrdFieldSource.java b/solr/core/src/java/org/apache/solr/search/function/OrdFieldSource.java index bbcb81363b5..2442674e096 100644 --- a/solr/core/src/java/org/apache/solr/search/function/OrdFieldSource.java +++ b/solr/core/src/java/org/apache/solr/search/function/OrdFieldSource.java @@ -73,9 +73,7 @@ public FunctionValues getValues(Map context, LeafReaderContext r final int off = readerContext.docBase; final LeafReader r; Object o = context.get("searcher"); - if (o instanceof SolrIndexSearcher) { - @SuppressWarnings("resource") - final SolrIndexSearcher is = (SolrIndexSearcher) o; + if (o instanceof @SuppressWarnings("resource") SolrIndexSearcher is) { SchemaField sf = is.getSchema().getFieldOrNull(field); if (sf != null && sf.getType().isPointField()) { throw new SolrException( diff --git a/solr/core/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java b/solr/core/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java index 5e725f79dab..45b343e29da 100644 --- a/solr/core/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java +++ b/solr/core/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java @@ -71,9 +71,7 @@ public FunctionValues getValues(Map context, LeafReaderContext r final int off = readerContext.docBase; final LeafReader r; Object o = context.get("searcher"); - if (o instanceof SolrIndexSearcher) { - @SuppressWarnings("resource") - final SolrIndexSearcher is = (SolrIndexSearcher) o; + if (o instanceof @SuppressWarnings("resource") SolrIndexSearcher is) { SchemaField sf = is.getSchema().getFieldOrNull(field); if (sf != null && sf.getType().isPointField()) { throw new SolrException( @@ -122,8 +120,7 @@ public int intVal(int doc) throws IOException { @Override public boolean equals(Object o) { - if (!(o instanceof ReverseOrdFieldSource)) return false; - ReverseOrdFieldSource other = (ReverseOrdFieldSource) o; + if (!(o instanceof ReverseOrdFieldSource other)) return false; return this.field.equals(other.field); } diff --git a/solr/core/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java b/solr/core/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java index 457bc756b32..0deb5744e8d 100644 --- a/solr/core/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java +++ b/solr/core/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java @@ -147,9 +147,8 @@ public void visit(QueryVisitor visitor) { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof ValueSourceRangeFilter)) return false; + if (!(o instanceof ValueSourceRangeFilter other)) return false; - ValueSourceRangeFilter other = (ValueSourceRangeFilter) o; return Objects.equals(this.valueSource, other.valueSource) && this.includeLower == other.includeLower && this.includeUpper == other.includeUpper diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/GeoDistValueSourceParser.java b/solr/core/src/java/org/apache/solr/search/function/distance/GeoDistValueSourceParser.java index e43f668929e..3db627ea7d1 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/GeoDistValueSourceParser.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/GeoDistValueSourceParser.java @@ -227,8 +227,7 @@ private MultiValueSource getMultiValueSource(FunctionQParser fp, String sfield) throws SyntaxError { SchemaField sf = fp.getReq().getSchema().getField(sfield); FieldType type = sf.getType(); - if (type instanceof AbstractSpatialFieldType) { - AbstractSpatialFieldType asft = (AbstractSpatialFieldType) type; + if (type instanceof AbstractSpatialFieldType asft) { return new SpatialStrategyMultiValueSource(asft.getStrategy(sfield), asft.getDistanceUnits()); } ValueSource vs = type.getValueSource(sf, fp); diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/GeohashFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/GeohashFunction.java index 06955ac4a67..730b6657186 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/GeohashFunction.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/GeohashFunction.java @@ -70,9 +70,7 @@ public String toString(int doc) throws IOException { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof GeohashFunction)) return false; - - GeohashFunction that = (GeohashFunction) o; + if (!(o instanceof GeohashFunction that)) return false; if (!lat.equals(that.lat)) return false; if (!lon.equals(that.lon)) return false; diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/GeohashHaversineFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/GeohashHaversineFunction.java index 6ffb4691d8e..7c13ab9d56e 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/GeohashHaversineFunction.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/GeohashHaversineFunction.java @@ -105,8 +105,7 @@ public void createWeight(Map context, IndexSearcher searcher) th @Override public boolean equals(Object o) { - if (!(o instanceof GeohashHaversineFunction)) return false; - GeohashHaversineFunction other = (GeohashHaversineFunction) o; + if (!(o instanceof GeohashHaversineFunction other)) return false; return Objects.equals(this.name(), other.name()) && Objects.equals(geoHash1, other.geoHash1) && Objects.equals(geoHash2, other.geoHash2) diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/HaversineConstFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/HaversineConstFunction.java index 635f8b9a1ec..fb6500f9a4e 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/HaversineConstFunction.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/HaversineConstFunction.java @@ -99,8 +99,7 @@ public void createWeight(Map context, IndexSearcher searcher) th @Override public boolean equals(Object o) { - if (!(o instanceof HaversineConstFunction)) return false; - HaversineConstFunction other = (HaversineConstFunction) o; + if (!(o instanceof HaversineConstFunction other)) return false; return this.latCenter == other.latCenter && this.lonCenter == other.lonCenter && this.p2.equals(other.p2); diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/HaversineFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/HaversineFunction.java index 2a0203536f9..6e6a640efb1 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/HaversineFunction.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/HaversineFunction.java @@ -124,8 +124,7 @@ public void createWeight(Map context, IndexSearcher searcher) th @Override public boolean equals(Object o) { - if (!(o instanceof HaversineFunction)) return false; - HaversineFunction other = (HaversineFunction) o; + if (!(o instanceof HaversineFunction other)) return false; return Objects.equals(this.name(), other.name()) && Objects.equals(p1, other.p1) && Objects.equals(p2, other.p2) diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/SquaredEuclideanFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/SquaredEuclideanFunction.java index 69acf99b62d..42153f5aab7 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/SquaredEuclideanFunction.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/SquaredEuclideanFunction.java @@ -54,11 +54,9 @@ protected double distance(int doc, FunctionValues dv1, FunctionValues dv2) throw @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof SquaredEuclideanFunction)) return false; + if (!(o instanceof SquaredEuclideanFunction that)) return false; if (!super.equals(o)) return false; - SquaredEuclideanFunction that = (SquaredEuclideanFunction) o; - if (!name.equals(that.name)) return false; return true; diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/StringDistanceFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/StringDistanceFunction.java index 9b47d261c2b..b3c418b8d6a 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/StringDistanceFunction.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/StringDistanceFunction.java @@ -86,9 +86,7 @@ public String description() { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof StringDistanceFunction)) return false; - - StringDistanceFunction that = (StringDistanceFunction) o; + if (!(o instanceof StringDistanceFunction that)) return false; if (!dist.equals(that.dist)) return false; if (!str1.equals(that.str1)) return false; diff --git a/solr/core/src/java/org/apache/solr/search/function/distance/VectorDistanceFunction.java b/solr/core/src/java/org/apache/solr/search/function/distance/VectorDistanceFunction.java index d41479559ee..e5c3d8aea22 100644 --- a/solr/core/src/java/org/apache/solr/search/function/distance/VectorDistanceFunction.java +++ b/solr/core/src/java/org/apache/solr/search/function/distance/VectorDistanceFunction.java @@ -184,9 +184,7 @@ public void createWeight(Map context, IndexSearcher searcher) th @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof VectorDistanceFunction)) return false; - - VectorDistanceFunction that = (VectorDistanceFunction) o; + if (!(o instanceof VectorDistanceFunction that)) return false; if (Float.compare(that.power, power) != 0) return false; if (!source1.equals(that.source1)) return false; diff --git a/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/SearchGroupsResultTransformer.java b/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/SearchGroupsResultTransformer.java index 8ca72d10878..b94863407a9 100644 --- a/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/SearchGroupsResultTransformer.java +++ b/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/SearchGroupsResultTransformer.java @@ -57,8 +57,7 @@ public NamedList> transform(List> data) throws IOEx final NamedList> result = new NamedList<>(data.size()); for (Command command : data) { final NamedList commandResult = new NamedList<>(2); - if (command instanceof SearchGroupsFieldCommand) { - SearchGroupsFieldCommand fieldCommand = (SearchGroupsFieldCommand) command; + if (command instanceof SearchGroupsFieldCommand fieldCommand) { final SearchGroupsFieldCommandResult fieldCommandResult = fieldCommand.result(); final Collection> searchGroups = fieldCommandResult.getSearchGroups(); if (searchGroups != null) { diff --git a/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java b/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java index 83255675fdd..5a924151fe5 100644 --- a/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java +++ b/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java @@ -72,12 +72,10 @@ public NamedList> transform(List> data) throws IOEx final IndexSchema schema = rb.req.getSearcher().getSchema(); for (Command command : data) { NamedList commandResult; - if (command instanceof TopGroupsFieldCommand) { - TopGroupsFieldCommand fieldCommand = (TopGroupsFieldCommand) command; + if (command instanceof TopGroupsFieldCommand fieldCommand) { SchemaField groupField = schema.getField(fieldCommand.getKey()); commandResult = serializeTopGroups(fieldCommand.result(), groupField); - } else if (command instanceof QueryCommand) { - QueryCommand queryCommand = (QueryCommand) command; + } else if (command instanceof QueryCommand queryCommand) { commandResult = serializeTopDocs(queryCommand.result()); } else { commandResult = null; @@ -241,11 +239,10 @@ protected NamedList serializeTopGroups(TopGroups data, SchemaF if (!Float.isNaN(searchGroup.scoreDocs[i].score)) { document.add("score", searchGroup.scoreDocs[i].score); } - if (!(searchGroup.scoreDocs[i] instanceof FieldDoc)) { + if (!(searchGroup.scoreDocs[i] instanceof FieldDoc fieldDoc)) { continue; // thus don't add sortValues below } - FieldDoc fieldDoc = (FieldDoc) searchGroup.scoreDocs[i]; Object[] convertedSortValues = new Object[fieldDoc.fields.length]; for (int j = 0; j < fieldDoc.fields.length; j++) { Object sortValue = fieldDoc.fields[j]; @@ -304,11 +301,10 @@ protected NamedList serializeTopDocs(QueryCommandResult result) throws I if (!Float.isNaN(scoreDoc.score)) { document.add("score", scoreDoc.score); } - if (!(scoreDoc instanceof FieldDoc)) { + if (!(scoreDoc instanceof FieldDoc fieldDoc)) { continue; // thus don't add sortValues below } - FieldDoc fieldDoc = (FieldDoc) scoreDoc; Object[] convertedSortValues = new Object[fieldDoc.fields.length]; for (int j = 0; j < fieldDoc.fields.length; j++) { Object sortValue = fieldDoc.fields[j]; diff --git a/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/MainEndResultTransformer.java b/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/MainEndResultTransformer.java index 1e18b40a21e..94c475237bf 100644 --- a/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/MainEndResultTransformer.java +++ b/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/MainEndResultTransformer.java @@ -72,8 +72,7 @@ public void transform( docList.setMaxScore(maxScore); } rb.rsp.addResponse(docList); - } else if (value instanceof QueryCommandResult) { - QueryCommandResult queryCommandResult = (QueryCommandResult) value; + } else if (value instanceof QueryCommandResult queryCommandResult) { SolrDocumentList docList = new SolrDocumentList(); TopDocs topDocs = queryCommandResult.getTopDocs(); diff --git a/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/SimpleEndResultTransformer.java b/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/SimpleEndResultTransformer.java index adfe736dfa8..171ee751f18 100644 --- a/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/SimpleEndResultTransformer.java +++ b/solr/core/src/java/org/apache/solr/search/grouping/endresulttransformer/SimpleEndResultTransformer.java @@ -70,8 +70,7 @@ public void transform( } command.add("doclist", docList); commands.add(entry.getKey(), command); - } else if (value instanceof QueryCommandResult) { - QueryCommandResult queryCommandResult = (QueryCommandResult) value; + } else if (value instanceof QueryCommandResult queryCommandResult) { NamedList command = new SimpleOrderedMap<>(); command.add("matches", queryCommandResult.getMatches()); diff --git a/solr/core/src/java/org/apache/solr/search/join/ChildFieldValueSourceParser.java b/solr/core/src/java/org/apache/solr/search/join/ChildFieldValueSourceParser.java index 24d252041ba..720accd090c 100644 --- a/solr/core/src/java/org/apache/solr/search/join/ChildFieldValueSourceParser.java +++ b/solr/core/src/java/org/apache/solr/search/join/ChildFieldValueSourceParser.java @@ -100,8 +100,7 @@ public int hashCode() { @Override public boolean equals(Object obj) { if (this == obj) return true; - if (!(obj instanceof BlockJoinSortFieldValueSource)) return false; - BlockJoinSortFieldValueSource other = (BlockJoinSortFieldValueSource) obj; + if (!(obj instanceof BlockJoinSortFieldValueSource other)) return false; return Objects.equals(childField, other.childField) && Objects.equals(childFilter, other.childFilter) && Objects.equals(parentFilter, other.parentFilter); diff --git a/solr/core/src/java/org/apache/solr/search/join/FiltersQParser.java b/solr/core/src/java/org/apache/solr/search/join/FiltersQParser.java index 0766a8f46d5..05c705aa1ce 100644 --- a/solr/core/src/java/org/apache/solr/search/join/FiltersQParser.java +++ b/solr/core/src/java/org/apache/solr/search/join/FiltersQParser.java @@ -133,8 +133,7 @@ private Collection excludeSet(Map tagMap, Set tagsToExclu // tagMap has entries of List>, but subject to change in the future if (!(olst instanceof Collection)) continue; for (Object o : (Collection) olst) { - if (!(o instanceof QParser)) continue; - QParser qp = (QParser) o; + if (!(o instanceof QParser qp)) continue; excludeSet.put(qp, Boolean.TRUE); } } diff --git a/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java index cc227b086b4..d61903440af 100644 --- a/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/join/ScoreJoinQParserPlugin.java @@ -154,8 +154,7 @@ public int hashCode() { public boolean equals(Object obj) { if (this == obj) return true; if (!super.equals(obj)) return false; - if (!(obj instanceof OtherCoreJoinQuery)) return false; - OtherCoreJoinQuery other = (OtherCoreJoinQuery) obj; + if (!(obj instanceof OtherCoreJoinQuery other)) return false; return (fromCoreOpenTime == other.fromCoreOpenTime) && Objects.equals(fromIndex, other.fromIndex); } diff --git a/solr/core/src/java/org/apache/solr/search/mlt/AbstractMLTQParser.java b/solr/core/src/java/org/apache/solr/search/mlt/AbstractMLTQParser.java index 356d7583f98..19aded8a79c 100644 --- a/solr/core/src/java/org/apache/solr/search/mlt/AbstractMLTQParser.java +++ b/solr/core/src/java/org/apache/solr/search/mlt/AbstractMLTQParser.java @@ -138,8 +138,7 @@ protected BooleanQuery parseMLTQuery(Supplier fieldsFallback, MLTInvok for (BooleanClause clause : rawMLTQuery) { Query q = clause.getQuery(); float originalBoost = 1f; - if (q instanceof BoostQuery) { - BoostQuery bq = (BoostQuery) q; + if (q instanceof BoostQuery bq) { q = bq.getQuery(); originalBoost = bq.getBoost(); } diff --git a/solr/core/src/java/org/apache/solr/search/neural/KnnQParser.java b/solr/core/src/java/org/apache/solr/search/neural/KnnQParser.java index 166dada5b7f..b6d9f2541cd 100644 --- a/solr/core/src/java/org/apache/solr/search/neural/KnnQParser.java +++ b/solr/core/src/java/org/apache/solr/search/neural/KnnQParser.java @@ -26,8 +26,8 @@ public class KnnQParser extends AbstractVectorQParserBase { // retrieve the top K results based on the distance similarity function - static final String TOP_K = "topK"; - static final int DEFAULT_TOP_K = 10; + protected static final String TOP_K = "topK"; + protected static final int DEFAULT_TOP_K = 10; public KnnQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { super(qstr, localParams, params, req); diff --git a/solr/core/src/java/org/apache/solr/security/AllowListUrlChecker.java b/solr/core/src/java/org/apache/solr/security/AllowListUrlChecker.java index 9bcede9b060..9fbffc4cdfb 100644 --- a/solr/core/src/java/org/apache/solr/security/AllowListUrlChecker.java +++ b/solr/core/src/java/org/apache/solr/security/AllowListUrlChecker.java @@ -27,6 +27,7 @@ import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.core.NodeConfig; @@ -85,6 +86,9 @@ public String toString() { /** Allow list of hosts. Elements in the list will be host:port (no protocol or context). */ private final Set hostAllowList; + private volatile Set liveHostUrlsCache; + private volatile Set liveNodesCache; + /** * @param urlAllowList List of allowed URLs. URLs must be well-formed, missing protocol is * tolerated. An empty list means there is no explicit allow-list of URLs, in this case no URL @@ -136,11 +140,10 @@ public void checkAllowList(List urls) throws MalformedURLException { */ public void checkAllowList(List urls, ClusterState clusterState) throws MalformedURLException { - Set clusterHostAllowList = - clusterState == null ? Collections.emptySet() : clusterState.getHostAllowList(); + Set liveHostUrls = getLiveHostUrls(clusterState); for (String url : urls) { String hostPort = parseHostPort(url); - if (clusterHostAllowList.stream().noneMatch(hostPort::equalsIgnoreCase) + if (liveHostUrls.stream().noneMatch(hostPort::equalsIgnoreCase) && hostAllowList.stream().noneMatch(hostPort::equalsIgnoreCase)) { throw new SolrException( SolrException.ErrorCode.FORBIDDEN, @@ -154,6 +157,33 @@ public void checkAllowList(List urls, ClusterState clusterState) } } + /** + * Gets the set of live hosts urls (host:port) built from the set of live nodes. The set is cached + * to be reused until the live nodes change. + */ + private Set getLiveHostUrls(ClusterState clusterState) { + if (clusterState == null) { + return Set.of(); + } + if (liveHostUrlsCache == null || clusterState.getLiveNodes() != liveNodesCache) { + synchronized (this) { + Set liveNodes = clusterState.getLiveNodes(); + if (liveHostUrlsCache == null || liveNodes != liveNodesCache) { + liveHostUrlsCache = buildLiveHostUrls(liveNodes); + liveNodesCache = liveNodes; + } + } + } + return liveHostUrlsCache; + } + + @VisibleForTesting + Set buildLiveHostUrls(Set liveNodes) { + return liveNodes.stream() + .map((liveNode) -> liveNode.substring(0, liveNode.indexOf('_'))) + .collect(Collectors.toSet()); + } + /** Whether this checker has been created with a non-empty allow-list of URLs. */ public boolean hasExplicitAllowList() { return !hostAllowList.isEmpty(); diff --git a/solr/core/src/java/org/apache/solr/security/AuthorizationContext.java b/solr/core/src/java/org/apache/solr/security/AuthorizationContext.java index c2fd9215e33..715294720c2 100644 --- a/solr/core/src/java/org/apache/solr/security/AuthorizationContext.java +++ b/solr/core/src/java/org/apache/solr/security/AuthorizationContext.java @@ -42,10 +42,9 @@ public String toString() { /** * This method returns the {@link Principal} corresponding to the authenticated user for the * current request. The value returned by {@link Principal#getName()} depends on the - * authentication mechanism used (e.g. for user "foo" with BASIC authentication the result would - * be "foo". On the other hand with KERBEROS it would be foo@REALMNAME). The {@link - * #getUserName()} method may be preferred to extract the identity of the authenticated user - * instead of this method. + * authentication mechanism used. For example for the user "foo" with BASIC authentication the + * result would be "foo". The {@link #getUserName()} method may be preferred to extract the + * identity of the authenticated user instead of this method. * * @return user principal in case of an authenticated request null in case of unauthenticated * request @@ -56,7 +55,7 @@ public String toString() { * This method returns the name of the authenticated user for the current request. The return * value of this method is agnostic of the underlying authentication mechanism used. * - * @return user name in case of an authenticated user null in case of unauthenticated request + * @return username in case of an authenticated user null in case of unauthenticated request */ public abstract String getUserName(); diff --git a/solr/core/src/java/org/apache/solr/security/BasicAuthPlugin.java b/solr/core/src/java/org/apache/solr/security/BasicAuthPlugin.java index cadc4fed946..07663c92f42 100644 --- a/solr/core/src/java/org/apache/solr/security/BasicAuthPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/BasicAuthPlugin.java @@ -102,8 +102,7 @@ public Map edit(Map latestConf, List getUserFromJettyRequest(Request request) { (String) request.getAttributes().get(CACHED_REQUEST_USER_KEY)); } }; - client.addListenerFactory(() -> listener); + if (client != null) { + client.addListenerFactory(() -> listener); + } + if (builder != null) { + builder.addListenerFactory(() -> listener); + } } @Override diff --git a/solr/core/src/java/org/apache/solr/security/Permission.java b/solr/core/src/java/org/apache/solr/security/Permission.java index a93020b7bf6..a344b3f2ce1 100644 --- a/solr/core/src/java/org/apache/solr/security/Permission.java +++ b/solr/core/src/java/org/apache/solr/security/Permission.java @@ -148,8 +148,7 @@ static Set readValueAsSet(Map m, String key) { } return null; } - if (val instanceof Collection) { - Collection list = (Collection) val; + if (val instanceof Collection list) { for (Object o : list) result.add(String.valueOf(o)); } else if (val instanceof String) { result.add((String) val); diff --git a/solr/core/src/java/org/apache/solr/security/RuleBasedAuthorizationPlugin.java b/solr/core/src/java/org/apache/solr/security/RuleBasedAuthorizationPlugin.java index 9788d8bef9e..7611b5559bb 100644 --- a/solr/core/src/java/org/apache/solr/security/RuleBasedAuthorizationPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/RuleBasedAuthorizationPlugin.java @@ -29,7 +29,6 @@ */ public class RuleBasedAuthorizationPlugin extends RuleBasedAuthorizationPluginBase { private final Map> usersVsRoles = new HashMap<>(); - private boolean useShortName; @Override public void init(Map initInfo) { @@ -41,17 +40,11 @@ public void init(Map initInfo) { String roleName = e.getKey(); usersVsRoles.put(roleName, Permission.readValueAsSet(map, roleName)); } - useShortName = - Boolean.parseBoolean(initInfo.getOrDefault("useShortName", Boolean.FALSE).toString()); } @Override public Set getUserRoles(AuthorizationContext context) { - if (useShortName) { - return usersVsRoles.get(context.getUserName()); - } else { - return getUserRoles(context.getUserPrincipal()); - } + return getUserRoles(context.getUserPrincipal()); } /** diff --git a/solr/core/src/java/org/apache/solr/security/RuleBasedAuthorizationPluginBase.java b/solr/core/src/java/org/apache/solr/security/RuleBasedAuthorizationPluginBase.java index 7c37e034c73..08321f1bf17 100644 --- a/solr/core/src/java/org/apache/solr/security/RuleBasedAuthorizationPluginBase.java +++ b/solr/core/src/java/org/apache/solr/security/RuleBasedAuthorizationPluginBase.java @@ -216,7 +216,7 @@ private boolean predefinedPermissionAppliesToRequest( if (predefinedPermission.wellknownName == PermissionNameProvider.Name.ALL) { log.trace("'ALL' perm applies to all requests; perm applies."); return true; // 'ALL' applies to everything! - } else if (!(context.getHandler() instanceof PermissionNameProvider)) { + } else if (!(context.getHandler() instanceof PermissionNameProvider handler)) { // TODO: Is this code path needed anymore, now that all handlers implement the interface? // context.getHandler returns Object and is not documented if (log.isTraceEnabled()) { @@ -227,7 +227,6 @@ private boolean predefinedPermissionAppliesToRequest( // We're not 'ALL', and the handler isn't associated with any other predefined permissions return false; } else { - PermissionNameProvider handler = (PermissionNameProvider) context.getHandler(); PermissionNameProvider.Name permissionName = handler.getPermissionName(context); boolean applies = diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index 99291fc6ad0..1e6c9f42e29 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -278,7 +278,7 @@ protected void init() throws Exception { } else { // if we couldn't find it locally, look on other nodes if (idx > 0) { - extractRemotePath(collectionName, origCorename); + extractRemotePath(collectionName); if (action == REMOTEQUERY) { path = path.substring(idx); return; @@ -461,10 +461,10 @@ protected void extractHandlerFromURLPath(SolrRequestParsers parser) throws Excep } } - protected void extractRemotePath(String collectionName, String origCorename) + protected void extractRemotePath(String collectionName) throws KeeperException, InterruptedException, SolrException { assert core == null; - coreUrl = getRemoteCoreUrl(collectionName, origCorename); + coreUrl = getRemoteCoreUrl(collectionName); // don't proxy for internal update requests invalidStates = checkStateVersionsAreValid(queryParams.get(CloudSolrClient.STATE_VERSION)); if (coreUrl != null @@ -1089,46 +1089,16 @@ private SolrCore checkProps(ZkNodeProps zkProps) { return core; } - private void getSlicesForCollections( - ClusterState clusterState, Collection slices, boolean activeSlices) { - if (activeSlices) { - for (Map.Entry entry : clusterState.getCollectionsMap().entrySet()) { - final Slice[] activeCollectionSlices = entry.getValue().getActiveSlicesArr(); - if (activeCollectionSlices != null) { - Collections.addAll(slices, activeCollectionSlices); - } - } - } else { - for (Map.Entry entry : clusterState.getCollectionsMap().entrySet()) { - final Collection collectionSlices = entry.getValue().getSlices(); - if (collectionSlices != null) { - slices.addAll(collectionSlices); - } - } - } - } - - protected String getRemoteCoreUrl(String collectionName, String origCorename) - throws SolrException { + protected String getRemoteCoreUrl(String collectionName) throws SolrException { ClusterState clusterState = cores.getZkController().getClusterState(); final DocCollection docCollection = clusterState.getCollectionOrNull(collectionName); - Slice[] slices = (docCollection != null) ? docCollection.getActiveSlicesArr() : null; - List activeSlices = new ArrayList<>(); - boolean byCoreName = false; + if (docCollection == null) { + return null; + } + Collection activeSlices = docCollection.getActiveSlices(); int totalReplicas = 0; - if (slices == null) { - byCoreName = true; - activeSlices = new ArrayList<>(); - getSlicesForCollections(clusterState, activeSlices, true); - if (activeSlices.isEmpty()) { - getSlicesForCollections(clusterState, activeSlices, false); - } - } else { - Collections.addAll(activeSlices, slices); - } - for (Slice s : activeSlices) { totalReplicas += s.getReplicas().size(); } @@ -1151,41 +1121,30 @@ protected String getRemoteCoreUrl(String collectionName, String origCorename) "No active replicas found for collection: " + collectionName); } - String coreUrl = - getCoreUrl(collectionName, origCorename, clusterState, activeSlices, byCoreName, true); + String coreUrl = getCoreUrl(activeSlices, true, clusterState.getLiveNodes()); if (coreUrl == null) { - coreUrl = - getCoreUrl(collectionName, origCorename, clusterState, activeSlices, byCoreName, false); + coreUrl = getCoreUrl(activeSlices, false, clusterState.getLiveNodes()); } return coreUrl; } private String getCoreUrl( - String collectionName, - String origCorename, - ClusterState clusterState, - List slices, - boolean byCoreName, - boolean activeReplicas) { - String coreUrl; - Set liveNodes = clusterState.getLiveNodes(); - Collections.shuffle(slices, Utils.RANDOM); + Collection slices, boolean activeReplicas, Set liveNodes) { + + Iterator shuffledSlices = new RandomIterator<>(Utils.RANDOM, slices); + while (shuffledSlices.hasNext()) { + Slice slice = shuffledSlices.next(); - for (Slice slice : slices) { - List randomizedReplicas = new ArrayList<>(slice.getReplicas()); - Collections.shuffle(randomizedReplicas, Utils.RANDOM); + Iterator shuffledReplicas = new RandomIterator<>(Utils.RANDOM, slice.getReplicas()); + while (shuffledReplicas.hasNext()) { + Replica replica = shuffledReplicas.next(); - for (Replica replica : randomizedReplicas) { if (!activeReplicas || (liveNodes.contains(replica.getNodeName()) && replica.getState() == Replica.State.ACTIVE)) { - if (byCoreName && !Objects.equals(origCorename, replica.getStr(CORE_NAME_PROP))) { - // if it's by core name, make sure they match - continue; - } if (Objects.equals(replica.getBaseUrl(), cores.getZkController().getBaseUrl())) { // don't count a local core continue; diff --git a/solr/core/src/java/org/apache/solr/servlet/ResponseUtils.java b/solr/core/src/java/org/apache/solr/servlet/ResponseUtils.java index 1f7427f669f..f28b1c2cfea 100644 --- a/solr/core/src/java/org/apache/solr/servlet/ResponseUtils.java +++ b/solr/core/src/java/org/apache/solr/servlet/ResponseUtils.java @@ -68,8 +68,7 @@ public static int getErrorInfo(Throwable ex, NamedList info, Logger log) public static int getErrorInfo( Throwable ex, NamedList info, Logger log, boolean hideTrace) { int code = 500; - if (ex instanceof SolrException) { - SolrException solrExc = (SolrException) ex; + if (ex instanceof SolrException solrExc) { code = solrExc.code(); NamedList errorMetadata = solrExc.getMetadata(); if (errorMetadata == null) { @@ -79,8 +78,7 @@ public static int getErrorInfo( errorMetadata.add( ErrorInfo.ROOT_ERROR_CLASS, SolrException.getRootCause(ex).getClass().getName()); info.add("metadata", errorMetadata); - if (ex instanceof ApiBag.ExceptionWithErrObject) { - ApiBag.ExceptionWithErrObject exception = (ApiBag.ExceptionWithErrObject) ex; + if (ex instanceof ApiBag.ExceptionWithErrObject exception) { info.add("details", exception.getErrs()); } } @@ -142,14 +140,12 @@ public static ErrorInfo getTypedErrorInfo(Throwable ex, Logger log) { public static ErrorInfo getTypedErrorInfo(Throwable ex, Logger log, boolean hideTrace) { final ErrorInfo errorInfo = new ErrorInfo(); int code = 500; - if (ex instanceof SolrException) { - SolrException solrExc = (SolrException) ex; + if (ex instanceof SolrException solrExc) { code = solrExc.code(); errorInfo.metadata = new ErrorInfo.ErrorMetadata(); errorInfo.metadata.errorClass = ex.getClass().getName(); errorInfo.metadata.rootErrorClass = SolrException.getRootCause(ex).getClass().getName(); - if (ex instanceof ApiBag.ExceptionWithErrObject) { - ApiBag.ExceptionWithErrObject exception = (ApiBag.ExceptionWithErrObject) ex; + if (ex instanceof ApiBag.ExceptionWithErrObject exception) { errorInfo.details = exception.getErrs(); } } diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java index 189ebeec134..392fe02ab13 100644 --- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java +++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java @@ -325,7 +325,12 @@ private void authenticateRequest( // trivial impl here to keep existing code happy while making the flow clearer. Chain will // be called after this method completes. Eventually auth all moves to its own filter // (hopefully). Most auth plugins simply return true after calling this anyway, so they - // obviously don't care. Kerberos plugins seem to mostly use it to satisfy the api of a + // obviously don't care. + // + // The Hadoop Auth Plugin was removed in SOLR-17540, however leaving the below reference + // for future readers, as there may be an option to simplify this logic. + // + // Kerberos plugins seem to mostly use it to satisfy the api of a // wrapped instance of javax.servlet.Filter and neither of those seem to be doing anything // fancy with the filter chain, so this would seem to be a hack brought on by the fact that // our auth code has been forced to be code within dispatch filter, rather than being a diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java index ee459c29231..8902acf51a3 100644 --- a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java +++ b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java @@ -19,7 +19,6 @@ import static org.apache.solr.common.params.CommonParams.PATH; import java.io.ByteArrayOutputStream; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.PushbackInputStream; @@ -31,6 +30,7 @@ import java.nio.charset.CharsetDecoder; import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.security.Principal; import java.util.ArrayList; import java.util.Arrays; @@ -236,7 +236,7 @@ private SolrQueryRequest buildRequestFrom( "Remote Streaming is disabled. See https://solr.apache.org/guide/solr/latest/configuration-guide/requestdispatcher.html for help"); } for (final String file : strs) { - ContentStreamBase stream = new ContentStreamBase.FileStream(new File(file)); + ContentStreamBase stream = new ContentStreamBase.FileStream(Path.of(file)); if (contentType != null) { stream.setContentType(contentType); } diff --git a/solr/core/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java b/solr/core/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java index a54ea8ab9e1..1c0e7006185 100644 --- a/solr/core/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java +++ b/solr/core/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java @@ -83,7 +83,7 @@ public String init(NamedList config, SolrCore core) { indexDir = (String) config.get(INDEX_DIR); // If indexDir is relative then create index inside core.getDataDir() if (indexDir != null) { - if (!new File(indexDir).isAbsolute()) { + if (!Path.of(indexDir).isAbsolute()) { indexDir = core.getDataDir() + File.separator + indexDir; } } diff --git a/solr/core/src/java/org/apache/solr/spelling/PossibilityIterator.java b/solr/core/src/java/org/apache/solr/spelling/PossibilityIterator.java index 8f676fa06ea..821df88c4c9 100644 --- a/solr/core/src/java/org/apache/solr/spelling/PossibilityIterator.java +++ b/solr/core/src/java/org/apache/solr/spelling/PossibilityIterator.java @@ -363,8 +363,7 @@ public int hashCode() { public boolean equals(Object obj) { if (this == obj) return true; if (obj == null) return false; - if (!(obj instanceof RankedSpellPossibility)) return false; - RankedSpellPossibility other = (RankedSpellPossibility) obj; + if (!(obj instanceof RankedSpellPossibility other)) return false; return Objects.equals(corrections, other.corrections); } diff --git a/solr/core/src/java/org/apache/solr/spelling/ResultEntry.java b/solr/core/src/java/org/apache/solr/spelling/ResultEntry.java index 67dd64f5f2d..cadb3c76d6e 100644 --- a/solr/core/src/java/org/apache/solr/spelling/ResultEntry.java +++ b/solr/core/src/java/org/apache/solr/spelling/ResultEntry.java @@ -42,8 +42,7 @@ public int hashCode() { @Override public boolean equals(Object obj) { if (this == obj) return true; - if (!(obj instanceof ResultEntry)) return false; - ResultEntry other = (ResultEntry) obj; + if (!(obj instanceof ResultEntry other)) return false; return freq == other.freq && Objects.equals(suggestion, other.suggestion) && Objects.equals(token, other.token); diff --git a/solr/core/src/java/org/apache/solr/spelling/Token.java b/solr/core/src/java/org/apache/solr/spelling/Token.java index c625a9fbdc9..38441722a29 100644 --- a/solr/core/src/java/org/apache/solr/spelling/Token.java +++ b/solr/core/src/java/org/apache/solr/spelling/Token.java @@ -132,9 +132,8 @@ public void clear() { @Override public boolean equals(Object obj) { if (obj == this) return true; - if (!(obj instanceof Token)) return false; + if (!(obj instanceof Token other)) return false; - final Token other = (Token) obj; return (flags == other.flags && (Objects.equals(payload, other.payload)) && super.equals(obj)); } diff --git a/solr/core/src/java/org/apache/solr/spelling/suggest/fst/AnalyzingInfixLookupFactory.java b/solr/core/src/java/org/apache/solr/spelling/suggest/fst/AnalyzingInfixLookupFactory.java index ead7cea97f7..682b9a81a29 100644 --- a/solr/core/src/java/org/apache/solr/spelling/suggest/fst/AnalyzingInfixLookupFactory.java +++ b/solr/core/src/java/org/apache/solr/spelling/suggest/fst/AnalyzingInfixLookupFactory.java @@ -18,6 +18,7 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -87,7 +88,7 @@ public Lookup create(NamedList params, SolrCore core) { String indexPath = params.get(INDEX_PATH) != null ? params.get(INDEX_PATH).toString() : DEFAULT_INDEX_PATH; - if (new File(indexPath).isAbsolute() == false) { + if (!Path.of(indexPath).isAbsolute()) { indexPath = core.getDataDir() + File.separator + indexPath; } @@ -108,7 +109,7 @@ public Lookup create(NamedList params, SolrCore core) { try { return new AnalyzingInfixSuggester( - FSDirectory.open(new File(indexPath).toPath()), + FSDirectory.open(Path.of(indexPath)), indexAnalyzer, queryAnalyzer, minPrefixChars, diff --git a/solr/core/src/java/org/apache/solr/spelling/suggest/fst/BlendedInfixLookupFactory.java b/solr/core/src/java/org/apache/solr/spelling/suggest/fst/BlendedInfixLookupFactory.java index 7b11e8ed594..09a85917c94 100644 --- a/solr/core/src/java/org/apache/solr/spelling/suggest/fst/BlendedInfixLookupFactory.java +++ b/solr/core/src/java/org/apache/solr/spelling/suggest/fst/BlendedInfixLookupFactory.java @@ -18,6 +18,7 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -78,7 +79,7 @@ public Lookup create(NamedList params, SolrCore core) { String indexPath = params.get(INDEX_PATH) != null ? params.get(INDEX_PATH).toString() : DEFAULT_INDEX_PATH; - if (new File(indexPath).isAbsolute() == false) { + if (!Path.of(indexPath).isAbsolute()) { indexPath = core.getDataDir() + File.separator + indexPath; } @@ -109,7 +110,7 @@ public Lookup create(NamedList params, SolrCore core) { try { return new BlendedInfixSuggester( - FSDirectory.open(new File(indexPath).toPath()), + FSDirectory.open(Path.of(indexPath)), indexAnalyzer, queryAnalyzer, minPrefixChars, diff --git a/solr/core/src/java/org/apache/solr/uninverting/FieldCacheImpl.java b/solr/core/src/java/org/apache/solr/uninverting/FieldCacheImpl.java index fcaef9b604f..6f23c14a6d7 100644 --- a/solr/core/src/java/org/apache/solr/uninverting/FieldCacheImpl.java +++ b/solr/core/src/java/org/apache/solr/uninverting/FieldCacheImpl.java @@ -225,8 +225,7 @@ static class CacheKey { /** Two of these are equal iff they reference the same field and type. */ @Override public boolean equals(Object o) { - if (o instanceof CacheKey) { - CacheKey other = (CacheKey) o; + if (o instanceof CacheKey other) { if (other.field.equals(field)) { if (other.custom == null) { if (custom == null) return true; diff --git a/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java b/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java index bb18b770e5d..6a749af2ac9 100644 --- a/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java +++ b/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java @@ -21,9 +21,21 @@ /** A commit index command encapsulated in an object. */ public class CommitUpdateCommand extends UpdateCommand { + + // Behavior about opening a searcher after a hard commit. + // - Open a standard searcher and wait for it. (default) + // openSearcher == true && waitSearcher == true && closeSearcher == false + // - Open a standard searcher, but do not wait for it. + // openSearcher == true && waitSearcher == false && closeSearcher == false + // - Open a real-time searcher quicker without auto-warming. + // openSearcher == false && closeSearcher == false + // - Do not open any searcher and prevent further updates (e.g. when the core is closing). + // closeSearcher == true + public boolean optimize; public boolean openSearcher = true; // open a new searcher as part of a hard commit public boolean waitSearcher = true; + private boolean closeSearcher = false; public boolean expungeDeletes = false; public boolean softCommit = false; public boolean prepareCommit = false; @@ -47,6 +59,26 @@ public CommitUpdateCommand(SolrQueryRequest req, boolean optimize) { this.optimize = optimize; } + /** + * Creates a {@link CommitUpdateCommand} to commit before closing the core and also prevent any + * new update by setting the core in read-only mode. Does not open a new searcher. + */ + public static CommitUpdateCommand closeOnCommit(SolrQueryRequest req, boolean optimize) { + CommitUpdateCommand cmd = new CommitUpdateCommand(req, optimize); + cmd.openSearcher = false; + cmd.waitSearcher = false; + cmd.closeSearcher = true; + return cmd; + } + + /** + * Indicates whether this command is a commit before the core is closed. Any new updates must be + * prevented after the commit. + */ + public boolean isClosingOnCommit() { + return closeSearcher; + } + @Override public String name() { return "commit"; diff --git a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java index d730cc137fb..2704ef4e027 100644 --- a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java +++ b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java @@ -738,6 +738,9 @@ public void commit(CommitUpdateCommand cmd) throws IOException { RefCounted iw = solrCoreState.getIndexWriter(core); try { + if (cmd.isClosingOnCommit()) { + core.readOnly = true; + } IndexWriter writer = iw.get(); if (cmd.optimize) { writer.forceMerge(cmd.maxOptimizeSegments); @@ -786,7 +789,7 @@ public void commit(CommitUpdateCommand cmd) throws IOException { if (ulog != null) ulog.preSoftCommit(cmd); if (cmd.openSearcher) { core.getSearcher(true, false, waitSearcher); - } else { + } else if (!cmd.isClosingOnCommit()) { // force open a new realtime searcher so realtime-get and versioning code can see the // latest RefCounted searchHolder = core.openNewSearcher(true, true); @@ -971,8 +974,10 @@ public void closeWriter(IndexWriter writer) throws IOException { // todo: refactor this shared code (or figure out why a real CommitUpdateCommand can't // be used) - SolrIndexWriter.setCommitData(writer, cmd.getVersion(), null); - writer.commit(); + if (shouldCommit(cmd, writer)) { + SolrIndexWriter.setCommitData(writer, cmd.getVersion(), cmd.commitData); + writer.commit(); + } synchronized (solrCoreState.getUpdateLock()) { ulog.postCommit(cmd); diff --git a/solr/core/src/java/org/apache/solr/update/PeerSync.java b/solr/core/src/java/org/apache/solr/update/PeerSync.java index 7636bf54452..1de7cbe3d6a 100644 --- a/solr/core/src/java/org/apache/solr/update/PeerSync.java +++ b/solr/core/src/java/org/apache/solr/update/PeerSync.java @@ -603,11 +603,8 @@ static class Updater { // comparator that sorts update records by absolute value of version, putting lowest first private static final Comparator updateRecordComparator = (o1, o2) -> { - if (!(o1 instanceof List)) return 1; - if (!(o2 instanceof List)) return -1; - - List lst1 = (List) o1; - List lst2 = (List) o2; + if (!(o1 instanceof List lst1)) return 1; + if (!(o2 instanceof List lst2)) return -1; long l1 = Math.abs((Long) lst1.get(1)); long l2 = Math.abs((Long) lst2.get(1)); diff --git a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java index 7f1998d6c82..7683250226f 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java +++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java @@ -617,8 +617,7 @@ public int hashCode() { @Override public boolean equals(Object obj) { if (this == obj) return true; - if (!(obj instanceof StdNode)) return false; - StdNode other = (StdNode) obj; + if (!(obj instanceof StdNode other)) return false; return (this.retry == other.retry) && (this.maxRetries == other.maxRetries) && Objects.equals(this.nodeProps.getBaseUrl(), other.nodeProps.getBaseUrl()) @@ -700,8 +699,7 @@ public int hashCode() { public boolean equals(Object obj) { if (this == obj) return true; if (!super.equals(obj)) return false; - if (!(obj instanceof ForwardNode)) return false; - ForwardNode other = (ForwardNode) obj; + if (!(obj instanceof ForwardNode other)) return false; return Objects.equals(nodeProps.getCoreUrl(), other.nodeProps.getCoreUrl()); } } diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java b/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java index fa12d7b51d4..a7267fb298d 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java +++ b/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java @@ -633,10 +633,9 @@ public boolean equals(Object obj) { if (this == obj) { return true; } - if (!(obj instanceof SplittingQuery)) { + if (!(obj instanceof SplittingQuery q)) { return false; } - SplittingQuery q = (SplittingQuery) obj; return partition == q.partition; } diff --git a/solr/core/src/java/org/apache/solr/update/TransactionLog.java b/solr/core/src/java/org/apache/solr/update/TransactionLog.java index f8c0d48731d..07b1d0b94fa 100644 --- a/solr/core/src/java/org/apache/solr/update/TransactionLog.java +++ b/solr/core/src/java/org/apache/solr/update/TransactionLog.java @@ -93,8 +93,7 @@ public class TransactionLog implements Closeable { new JavaBinCodec.ObjectResolver() { @Override public Object resolve(Object o, JavaBinCodec codec) throws IOException { - if (o instanceof BytesRef) { - BytesRef br = (BytesRef) o; + if (o instanceof BytesRef br) { codec.writeByteArray(br.bytes, br.offset, br.length); return null; } @@ -161,8 +160,7 @@ protected Object readObject(DataInputInputStream dis) throws IOException { @Override public boolean writePrimitive(Object val) throws IOException { - if (val instanceof java.util.UUID) { - java.util.UUID uuid = (java.util.UUID) val; + if (val instanceof java.util.UUID uuid) { daos.writeByte(UUID); daos.writeLong(uuid.getMostSignificantBits()); daos.writeLong(uuid.getLeastSignificantBits()); diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLog.java b/solr/core/src/java/org/apache/solr/update/UpdateLog.java index d1c430bceee..837f031ccec 100644 --- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java +++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java @@ -24,8 +24,6 @@ import com.codahale.metrics.Gauge; import com.codahale.metrics.Meter; import java.io.Closeable; -import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.UncheckedIOException; import java.lang.invoke.MethodHandles; @@ -391,7 +389,8 @@ public void init(PluginInfo info) { } int timeoutMs = objToInt( - info.initArgs.get("docLockTimeoutMs", info.initArgs.get("versionBucketLockTimeoutMs")), + info.initArgs.getOrDefault( + "docLockTimeoutMs", info.initArgs.get("versionBucketLockTimeoutMs")), EnvUtils.getPropertyAsLong("solr.update.docLockTimeoutMs", 0L).intValue()); updateLocks = new UpdateLocks(timeoutMs); @@ -580,7 +579,7 @@ protected void initTlogDir(SolrCore core) { } catch (IOException e) { throw new SolrException(ErrorCode.SERVER_ERROR, "Could not set up tlogs", e); } - tlogFiles = getLogList(tlogDir.toFile()); + tlogFiles = getLogList(tlogDir); id = getLastLogId() + 1; // add 1 since we will create a new log for the next update if (debug) { @@ -731,14 +730,17 @@ private boolean updateFromOldTlogs(UpdateCommand cmd) { return (cmd.getFlags() & UpdateCommand.REPLAY) != 0 && state == State.REPLAYING; } - public String[] getLogList(File directory) { + public String[] getLogList(Path directory) { final String prefix = TLOG_NAME + '.'; - String[] names = directory.list((dir, name) -> name.startsWith(prefix)); - if (names == null) { - throw new RuntimeException(new FileNotFoundException(directory.getAbsolutePath())); + try (Stream files = Files.list(directory)) { + return files + .map((file) -> file.getFileName().toString()) + .filter((name) -> name.startsWith(prefix)) + .sorted() + .toArray(String[]::new); + } catch (IOException e) { + throw new RuntimeException(e); } - Arrays.sort(names); - return names; } public long getLastLogId() { @@ -1215,8 +1217,7 @@ private synchronized List getEntryFromTLog( lookupVersion); } - if (obj != null && obj instanceof List) { - List tmpEntry = (List) obj; + if (obj != null && obj instanceof List tmpEntry) { if (tmpEntry.size() >= 2 && // why not Objects.equals(lookupVersion, tmpEntry.get())? diff --git a/solr/core/src/java/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactory.java index a56361a4652..daea8ce83a4 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactory.java @@ -191,11 +191,10 @@ private static List parseTypeMappings(NamedList args) { throw new SolrException( SERVER_ERROR, "'" + TYPE_MAPPING_PARAM + "' init param cannot be null"); } - if (!(typeMappingObj instanceof NamedList)) { + if (!(typeMappingObj instanceof NamedList typeMappingNamedList)) { throw new SolrException( SERVER_ERROR, "'" + TYPE_MAPPING_PARAM + "' init param must be a "); } - NamedList typeMappingNamedList = (NamedList) typeMappingObj; Object fieldTypeObj = typeMappingNamedList.remove(FIELD_TYPE_PARAM); if (null == fieldTypeObj) { @@ -256,11 +255,10 @@ private static List parseTypeMappings(NamedList args) { Collection copyFieldDefs = new ArrayList<>(); while (typeMappingNamedList.get(COPY_FIELD_PARAM) != null) { Object copyFieldObj = typeMappingNamedList.remove(COPY_FIELD_PARAM); - if (!(copyFieldObj instanceof NamedList)) { + if (!(copyFieldObj instanceof NamedList copyFieldNamedList)) { throw new SolrException( SERVER_ERROR, "'" + COPY_FIELD_PARAM + "' init param must be a "); } - NamedList copyFieldNamedList = (NamedList) copyFieldObj; // dest Object destObj = copyFieldNamedList.remove(DEST_PARAM); if (null == destObj) { diff --git a/solr/core/src/java/org/apache/solr/update/processor/AtomicUpdateDocumentMerger.java b/solr/core/src/java/org/apache/solr/update/processor/AtomicUpdateDocumentMerger.java index b3544170c68..dbfd33959bd 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/AtomicUpdateDocumentMerger.java +++ b/solr/core/src/java/org/apache/solr/update/processor/AtomicUpdateDocumentMerger.java @@ -587,11 +587,10 @@ protected void doInc(SolrInputDocument toDoc, SolrInputField sif, Object fieldVa // behavior similar to doAdd/doSet Object resObj = getNativeFieldValue(sf.getName(), fieldVal); - if (!(resObj instanceof Number)) { + if (!(resObj instanceof Number result)) { throw new SolrException( ErrorCode.BAD_REQUEST, "Invalid input '" + resObj + "' for field " + sf.getName()); } - Number result = (Number) resObj; if (oldVal instanceof Long) { result = ((Long) oldVal).longValue() + result.longValue(); } else if (oldVal instanceof Float) { @@ -691,10 +690,9 @@ private Object getNativeFieldValue(String fieldName, Object val) { } private static boolean isChildDoc(Object obj) { - if (!(obj instanceof Collection)) { + if (!(obj instanceof Collection objValues)) { return obj instanceof SolrDocumentBase; } - Collection objValues = (Collection) obj; if (objValues.size() == 0) { return false; } diff --git a/solr/core/src/java/org/apache/solr/update/processor/ClassificationUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/ClassificationUpdateProcessorFactory.java index 1d0e57bc3e9..7f599682eaf 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/ClassificationUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/ClassificationUpdateProcessorFactory.java @@ -92,8 +92,7 @@ public void init(final NamedList args) { String algorithmString = params.get(ALGORITHM_PARAM); Algorithm classificationAlgorithm; try { - if (algorithmString == null - || Algorithm.valueOf(algorithmString.toUpperCase(Locale.ROOT)) == null) { + if (algorithmString == null) { classificationAlgorithm = DEFAULT_ALGORITHM; } else { classificationAlgorithm = Algorithm.valueOf(algorithmString.toUpperCase(Locale.ROOT)); diff --git a/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java index 8f1786a4098..4240454b84d 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java @@ -349,11 +349,10 @@ private void initSourceSelectorSyntax(NamedList args) { throw new SolrException( SERVER_ERROR, "Init param '" + SOURCE_PARAM + "' child 'exclude' can not be null"); } - if (!(excObj instanceof NamedList)) { + if (!(excObj instanceof NamedList exc)) { throw new SolrException( SERVER_ERROR, "Init param '" + SOURCE_PARAM + "' child 'exclude' must be "); } - NamedList exc = (NamedList) excObj; srcExclusions.add(parseSelectorParams(exc)); if (0 < exc.size()) { throw new SolrException( @@ -400,8 +399,7 @@ private void initSourceSelectorSyntax(NamedList args) { + "for CloneFieldUpdateProcessorFactory for further details."); } - if (d instanceof NamedList) { - NamedList destList = (NamedList) d; + if (d instanceof NamedList destList) { Object patt = destList.remove(PATTERN_PARAM); Object replacement = destList.remove(REPLACEMENT_PARAM); diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java index 8e3153046ad..2edbeb6c345 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java @@ -1229,8 +1229,7 @@ public DistributedUpdatesAsyncException(List errors) { // create a merged copy of the metadata from all wrapped exceptions NamedList metadata = new NamedList<>(); for (SolrError error : errors) { - if (error.e instanceof SolrException) { - SolrException e = (SolrException) error.e; + if (error.e instanceof SolrException e) { NamedList eMeta = e.getMetadata(); if (null != eMeta) { metadata.addAll(eMeta); diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java index 8c8c3ce6a08..f744be22755 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java @@ -680,11 +680,9 @@ private String getLeaderUrlZk(String id) { @Override void setupRequest(UpdateCommand cmd) { zkCheck(cmd); - if (cmd instanceof AddUpdateCommand) { - AddUpdateCommand acmd = (AddUpdateCommand) cmd; + if (cmd instanceof AddUpdateCommand acmd) { nodes = setupRequest(acmd.getIndexedIdStr(), acmd.getSolrInputDocument(), null, cmd); - } else if (cmd instanceof DeleteUpdateCommand) { - DeleteUpdateCommand dcmd = (DeleteUpdateCommand) cmd; + } else if (cmd instanceof DeleteUpdateCommand dcmd) { nodes = setupRequest( dcmd.getId(), @@ -975,8 +973,7 @@ protected List getNodesByRoutingRules( ClusterState cstate, DocCollection coll, String id, SolrInputDocument doc) { DocRouter router = coll.getRouter(); List nodes = null; - if (router instanceof CompositeIdRouter) { - CompositeIdRouter compositeIdRouter = (CompositeIdRouter) router; + if (router instanceof CompositeIdRouter compositeIdRouter) { String myShardId = cloudDesc.getShardId(); Slice slice = coll.getSlice(myShardId); Map routingRules = slice.getRoutingRules(); @@ -1248,8 +1245,7 @@ protected void doDistribFinish() { String collection = null; String shardId = null; - if (error.req.node instanceof SolrCmdDistributor.StdNode) { - SolrCmdDistributor.StdNode stdNode = (SolrCmdDistributor.StdNode) error.req.node; + if (error.req.node instanceof SolrCmdDistributor.StdNode stdNode) { collection = stdNode.getCollection(); shardId = stdNode.getShardId(); diff --git a/solr/core/src/java/org/apache/solr/update/processor/FieldMutatingUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/FieldMutatingUpdateProcessorFactory.java index 1927cd0b8c9..7571d918a14 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/FieldMutatingUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/FieldMutatingUpdateProcessorFactory.java @@ -165,11 +165,10 @@ public static Collection parseSelectorExclusionParams(NamedList< throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "'exclude' init param can not be null"); } - if (!(excObj instanceof NamedList)) { + if (!(excObj instanceof NamedList exc)) { throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "'exclude' init param must be "); } - NamedList exc = (NamedList) excObj; exclusions.add(parseSelectorParams(exc)); if (0 < exc.size()) { throw new SolrException( diff --git a/solr/core/src/java/org/apache/solr/update/processor/HTMLStripFieldUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/HTMLStripFieldUpdateProcessorFactory.java index b16466f42a5..a710ccdfe27 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/HTMLStripFieldUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/HTMLStripFieldUpdateProcessorFactory.java @@ -59,8 +59,7 @@ public UpdateRequestProcessor getInstance( getSelector(), next, src -> { - if (src instanceof CharSequence) { - CharSequence s = (CharSequence) src; + if (src instanceof CharSequence s) { StringWriter result = new StringWriter(s.length()); try (Reader in = new HTMLStripCharFilter(new StringReader(s.toString()))) { in.transferTo(result); diff --git a/solr/core/src/java/org/apache/solr/update/processor/LastFieldValueUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/LastFieldValueUpdateProcessorFactory.java index 80325f2483e..8be53308a3a 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/LastFieldValueUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/LastFieldValueUpdateProcessorFactory.java @@ -53,9 +53,8 @@ public Collection pickSubset(Collection values) { T result = null; - if (values instanceof List) { + if (values instanceof List l) { // optimize index lookup - List l = (List) values; result = l.get(l.size() - 1); } else if (values instanceof SortedSet) { // optimize tail lookup diff --git a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java index 243890b80fb..96a93b7b45e 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java @@ -85,7 +85,7 @@ private boolean processDocChildren(SolrInputDocument doc, String fullPath) { int childNum = 0; boolean isSingleVal = !(field.getValue() instanceof Collection); for (Object val : field) { - if (!(val instanceof SolrInputDocument)) { + if (!(val instanceof SolrInputDocument cDoc)) { // either all collection items are child docs or none are. break; } @@ -101,7 +101,6 @@ private boolean processDocChildren(SolrInputDocument doc, String fullPath) { + "' , which is reserved for the nested URP"); } final String sChildNum = isSingleVal ? SINGULAR_VALUE_CHAR : String.valueOf(childNum); - SolrInputDocument cDoc = (SolrInputDocument) val; if (!cDoc.containsKey(uniqueKeyFieldName)) { String parentDocId = doc.getField(uniqueKeyFieldName).getFirstValue().toString(); cDoc.setField( diff --git a/solr/core/src/java/org/apache/solr/update/processor/RegexReplaceProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/RegexReplaceProcessorFactory.java index 2c9a02dd652..843727bada4 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/RegexReplaceProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/RegexReplaceProcessorFactory.java @@ -120,8 +120,7 @@ public UpdateRequestProcessor getInstance( getSelector(), next, src -> { - if (src instanceof CharSequence) { - CharSequence txt = (CharSequence) src; + if (src instanceof CharSequence txt) { return pattern.matcher(txt).replaceAll(replacement); } return src; diff --git a/solr/core/src/java/org/apache/solr/update/processor/TolerantUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/TolerantUpdateProcessor.java index 0e2e1a6ca8a..36bfa0fd47e 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/TolerantUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/TolerantUpdateProcessor.java @@ -238,11 +238,10 @@ public void finish() throws IOException { // // instead we trust the metadata that the TolerantUpdateProcessor running on the remote node // added to the exception when it failed. - if (!(error.e instanceof SolrException)) { + if (!(error.e instanceof SolrException remoteErr)) { log.error("async update exception is not SolrException, no metadata to process", error.e); continue; } - SolrException remoteErr = (SolrException) error.e; NamedList remoteErrMetadata = remoteErr.getMetadata(); if (null == remoteErrMetadata) { diff --git a/solr/core/src/java/org/apache/solr/update/processor/TruncateFieldUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/TruncateFieldUpdateProcessorFactory.java index 66127f885f7..717f0a54ff7 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/TruncateFieldUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/TruncateFieldUpdateProcessorFactory.java @@ -89,8 +89,7 @@ public UpdateRequestProcessor getInstance( getSelector(), next, src -> { - if (src instanceof CharSequence) { - CharSequence s = (CharSequence) src; + if (src instanceof CharSequence s) { if (maxLength < s.length()) { return s.subSequence(0, maxLength); } diff --git a/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java b/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java index 1f5476632f8..f7f5cba5852 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java +++ b/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessorChain.java @@ -348,8 +348,7 @@ public int hashCode() { @Override public boolean equals(Object obj) { - if (!(obj instanceof ProcessorInfo)) return false; - ProcessorInfo that = (ProcessorInfo) obj; + if (!(obj instanceof ProcessorInfo that)) return false; return Objects.equals(this.processor, that.processor) && Objects.equals(this.postProcessor, that.postProcessor); diff --git a/solr/core/src/java/org/apache/solr/util/DOMConfigNode.java b/solr/core/src/java/org/apache/solr/util/DOMConfigNode.java index f7a47b5cc93..d35cd4bccc5 100644 --- a/solr/core/src/java/org/apache/solr/util/DOMConfigNode.java +++ b/solr/core/src/java/org/apache/solr/util/DOMConfigNode.java @@ -18,14 +18,11 @@ package org.apache.solr.util; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.Function; -import org.apache.solr.cluster.api.SimpleMap; import org.apache.solr.common.ConfigNode; import org.apache.solr.common.util.DOMUtil; -import org.apache.solr.common.util.WrappedSimpleMap; import org.w3c.dom.Node; import org.w3c.dom.NodeList; @@ -33,7 +30,7 @@ public class DOMConfigNode implements ConfigNode { private final Node node; - SimpleMap attrs; + Map attrs; @Override public String name() { @@ -50,10 +47,10 @@ public DOMConfigNode(Node node) { } @Override - public SimpleMap attributes() { + public Map attributes() { if (attrs != null) return attrs; Map attrs = DOMUtil.toMap(node.getAttributes()); - return this.attrs = attrs.size() == 0 ? EMPTY : new WrappedSimpleMap<>(attrs); + return this.attrs = attrs.isEmpty() ? Map.of() : attrs; } @Override @@ -85,6 +82,4 @@ public void forEachChild(Function fun) { if (Boolean.FALSE.equals(toContinue)) break; } } - - private static final SimpleMap EMPTY = new WrappedSimpleMap<>(Collections.emptyMap()); } diff --git a/solr/core/src/java/org/apache/solr/util/DataConfigNode.java b/solr/core/src/java/org/apache/solr/util/DataConfigNode.java index e8a00075031..4711e644cb4 100644 --- a/solr/core/src/java/org/apache/solr/util/DataConfigNode.java +++ b/solr/core/src/java/org/apache/solr/util/DataConfigNode.java @@ -17,8 +17,10 @@ package org.apache.solr.util; +import java.util.AbstractMap; +import java.util.AbstractSet; import java.util.ArrayList; -import java.util.Collections; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -26,22 +28,20 @@ import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.Predicate; -import org.apache.solr.cluster.api.SimpleMap; import org.apache.solr.common.ConfigNode; import org.apache.solr.common.util.PropertiesUtil; -import org.apache.solr.common.util.WrappedSimpleMap; /** ConfigNode impl that copies and maintains data internally from DOM */ public class DataConfigNode implements ConfigNode { public final String name; - public final SimpleMap attributes; - public final SimpleMap> kids; + public final Map attributes; + public final Map> kids; public final String textData; public DataConfigNode(ConfigNode root) { Map> kids = new LinkedHashMap<>(); name = root.name(); - attributes = wrap(root.attributes()); + attributes = wrapSubstituting(root.attributes()); textData = root.txt(); root.forEachChild( it -> { @@ -54,31 +54,17 @@ public DataConfigNode(ConfigNode root) { e.setValue(List.copyOf(e.getValue())); } } - this.kids = kids.isEmpty() ? EMPTY : new WrappedSimpleMap<>(Map.copyOf(kids)); + this.kids = Map.copyOf(kids); } - public String subtituteVal(String s) { + private static String substituteVal(String s) { return PropertiesUtil.substitute(s, SUBSTITUTES.get()); } - private SimpleMap wrap(SimpleMap delegate) { + /** provides a substitute view, and read-only */ + private static Map wrapSubstituting(Map delegate) { if (delegate.size() == 0) return delegate; // avoid unnecessary object creation - return new SimpleMap<>() { - @Override - public String get(String key) { - return subtituteVal(delegate.get(key)); - } - - @Override - public void forEachEntry(BiConsumer fun) { - delegate.forEachEntry((k, v) -> fun.accept(k, subtituteVal(v))); - } - - @Override - public int size() { - return delegate.size(); - } - }; + return new SubstitutingMap(delegate); } @Override @@ -88,11 +74,11 @@ public String name() { @Override public String txt() { - return subtituteVal(textData); + return substituteVal(textData); } @Override - public SimpleMap attributes() { + public Map attributes() { return attributes; } @@ -104,7 +90,7 @@ public ConfigNode child(String name) { @Override public List getAll(String name) { - return kids.get(name, Collections.emptyList()); + return kids.getOrDefault(name, List.of()); } @Override @@ -126,7 +112,7 @@ public List getAll(Predicate test, Set matchName @Override public void forEachChild(Function fun) { - kids.forEachEntry( + kids.forEach( (s, configNodes) -> { if (configNodes != null) { configNodes.forEach(fun::apply); @@ -134,6 +120,74 @@ public void forEachChild(Function fun) { }); } - public static final SimpleMap> EMPTY = - new WrappedSimpleMap<>(Collections.emptyMap()); + private static class SubstitutingMap extends AbstractMap { + + private final Map delegate; + + SubstitutingMap(Map delegate) { + this.delegate = delegate; + } + + @Override + public String get(Object key) { + return substituteVal(delegate.get(key)); + } + + @Override + public int size() { + return delegate.size(); + } + + @Override + public Set keySet() { + return delegate.keySet(); + } + + @Override + public void forEach(BiConsumer action) { + delegate.forEach((k, v) -> action.accept(k, substituteVal(v))); + } + + @Override + public Set> entrySet() { + return new AbstractSet<>() { + @Override + public Iterator> iterator() { + // using delegate, return an iterator using Streams + return delegate.entrySet().stream() + .map(entry -> (Entry) new SubstitutingEntry(entry)) + .iterator(); + } + + @Override + public int size() { + return delegate.size(); + } + }; + } + + private static class SubstitutingEntry implements Entry { + + private final Entry delegateEntry; + + SubstitutingEntry(Entry delegateEntry) { + this.delegateEntry = delegateEntry; + } + + @Override + public String getKey() { + return delegateEntry.getKey(); + } + + @Override + public String getValue() { + return substituteVal(delegateEntry.getValue()); + } + + @Override + public String setValue(String value) { + throw new UnsupportedOperationException(); + } + } + } } diff --git a/solr/core/src/java/org/apache/solr/util/FileUtils.java b/solr/core/src/java/org/apache/solr/util/FileUtils.java index f90f8a658aa..0b221aae025 100644 --- a/solr/core/src/java/org/apache/solr/util/FileUtils.java +++ b/solr/core/src/java/org/apache/solr/util/FileUtils.java @@ -16,13 +16,7 @@ */ package org.apache.solr.util; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; import org.apache.commons.io.FileExistsException; @@ -30,60 +24,8 @@ /** */ public class FileUtils { - /** - * Resolves a path relative a base directory. - * - *

    This method does what "new File(base,path)" Should do, if it wasn't completely lame: - * If path is absolute, then a File for that path is returned; if it's not absolute, then a File - * is returned using "path" as a child of "base") - */ - public static File resolvePath(File base, String path) { - File r = new File(path); - return r.isAbsolute() ? r : new File(base, path); - } - - public static void copyFile(File src, File destination) throws IOException { - try (FileChannel in = new FileInputStream(src).getChannel(); - FileChannel out = new FileOutputStream(destination).getChannel()) { - in.transferTo(0, in.size(), out); - } - } - - /** - * Copied from Lucene's FSDirectory.fsync(String) - * - * @param fullFile the File to be synced to disk - * @throws IOException if the file could not be synced - */ - public static void sync(File fullFile) throws IOException { - if (fullFile == null || !fullFile.exists()) - throw new FileNotFoundException("File does not exist " + fullFile); - - boolean success = false; - int retryCount = 0; - IOException exc = null; - while (!success && retryCount < 5) { - retryCount++; - try (RandomAccessFile file = new RandomAccessFile(fullFile, "rw")) { - file.getFD().sync(); - success = true; - } catch (IOException ioe) { - if (exc == null) exc = ioe; - try { - // Pause 5 msec - Thread.sleep(5); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - } - } - } - if (!success) - // Throw original exception - throw exc; - } - public static boolean fileExists(String filePathString) { - return new File(filePathString).exists(); + return Files.exists(Path.of(filePathString)); } // Files.createDirectories has odd behavior if the path is a symlink and it already exists @@ -103,4 +45,28 @@ public static Path createDirectories(Path path) throws IOException { } return Files.createDirectories(path); } + + /** + * Checks whether a child path falls under a particular parent + * + *

    Useful for validating user-provided relative paths, which generally aren't expected to + * "escape" a given parent/root directory. Parent and child paths are "normalized" by {@link + * Path#normalize()}. This removes explicit backtracking (e.g. "../") though it will not resolve + * symlinks if any are present in the provided Paths, so some forms of parent "escape" remain + * undetected. Paths needn't exist as a file or directory for comparison purposes. + * + *

    Note, this method does not consult the file system + * + * @param parent the path of a 'parent' node. Path must be syntactically valid but needn't exist. + * @param potentialChild the path of a potential child. Typically obtained via: + * parent.resolve(relativeChildPath). Path must be syntactically valid but needn't exist. + * @return true if 'potentialChild' nests under the provided 'parent', false otherwise. + */ + public static boolean isPathAChildOfParent(Path parent, Path potentialChild) { + final var normalizedParent = parent.toAbsolutePath().normalize(); + final var normalizedChild = potentialChild.toAbsolutePath().normalize(); + + return normalizedChild.startsWith(normalizedParent) + && !normalizedChild.equals(normalizedParent); + } } diff --git a/solr/core/src/java/org/apache/solr/util/PayloadUtils.java b/solr/core/src/java/org/apache/solr/util/PayloadUtils.java index ed667de1e40..cda02b4d75c 100644 --- a/solr/core/src/java/org/apache/solr/util/PayloadUtils.java +++ b/solr/core/src/java/org/apache/solr/util/PayloadUtils.java @@ -50,10 +50,9 @@ public static String getPayloadEncoder(FieldType fieldType) { // component that encodes payloads as floats String encoder = null; Analyzer a = fieldType.getIndexAnalyzer(); - if (a instanceof TokenizerChain) { + if (a instanceof TokenizerChain tc) { // examine the indexing analysis chain for DelimitedPayloadTokenFilterFactory or // NumericPayloadTokenFilterFactory - TokenizerChain tc = (TokenizerChain) a; TokenFilterFactory[] factories = tc.getTokenFilterFactories(); for (TokenFilterFactory factory : factories) { if (factory instanceof DelimitedPayloadTokenFilterFactory) { diff --git a/solr/core/src/java/org/apache/solr/util/SolrJacksonAnnotationInspector.java b/solr/core/src/java/org/apache/solr/util/SolrJacksonAnnotationInspector.java index 97af568274c..e7ad4723c79 100644 --- a/solr/core/src/java/org/apache/solr/util/SolrJacksonAnnotationInspector.java +++ b/solr/core/src/java/org/apache/solr/util/SolrJacksonAnnotationInspector.java @@ -52,8 +52,7 @@ public Version version() { @Override public PropertyName findNameForSerialization(Annotated a) { - if (a instanceof AnnotatedMethod) { - AnnotatedMethod am = (AnnotatedMethod) a; + if (a instanceof AnnotatedMethod am) { JsonProperty prop = am.getAnnotation(JsonProperty.class); if (prop == null) return null; if (prop.value().isEmpty()) { @@ -62,8 +61,7 @@ public PropertyName findNameForSerialization(Annotated a) { return new PropertyName(prop.value()); } } - if (a instanceof AnnotatedField) { - AnnotatedField af = (AnnotatedField) a; + if (a instanceof AnnotatedField af) { JsonProperty prop = af.getAnnotation(JsonProperty.class); if (prop == null) return null; return prop.value().isEmpty() diff --git a/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java b/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java index 44ade037f0e..d820d699985 100644 --- a/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java +++ b/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java @@ -712,8 +712,7 @@ private static void flattenBooleanQuery( Query cq = clause.getQuery(); float boost = fromBoost; - while (cq instanceof BoostQuery) { - BoostQuery bq = (BoostQuery) cq; + while (cq instanceof BoostQuery bq) { cq = bq.getQuery(); boost *= bq.getBoost(); } diff --git a/solr/core/src/java/org/apache/solr/util/VersionedFile.java b/solr/core/src/java/org/apache/solr/util/VersionedFile.java index a852f1fae26..12ffdb63b53 100644 --- a/solr/core/src/java/org/apache/solr/util/VersionedFile.java +++ b/solr/core/src/java/org/apache/solr/util/VersionedFile.java @@ -41,6 +41,7 @@ public class VersionedFile { */ public static InputStream getLatestFile(String dirName, String fileName) throws FileNotFoundException { + // TODO SOLR-8282 move to PATH Collection oldFiles = null; final String prefix = fileName + '.'; File f = new File(dirName, fileName); diff --git a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CPUCircuitBreaker.java b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CPUCircuitBreaker.java index 435ca03ff86..cf226f1e6c0 100644 --- a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CPUCircuitBreaker.java +++ b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CPUCircuitBreaker.java @@ -118,9 +118,7 @@ protected double calculateLiveCPUUsage() { return -1.0; } - if (metric instanceof Gauge) { - @SuppressWarnings({"rawtypes"}) - Gauge gauge = (Gauge) metric; + if (metric instanceof Gauge gauge) { // unwrap if needed if (gauge instanceof SolrMetricManager.GaugeWrapper) { gauge = ((SolrMetricManager.GaugeWrapper) gauge).getGauge(); diff --git a/solr/core/src/java/org/apache/solr/util/stats/InstrumentedHttpRequestExecutor.java b/solr/core/src/java/org/apache/solr/util/stats/InstrumentedHttpRequestExecutor.java index b50751a1ce0..e4be2d9b2fa 100644 --- a/solr/core/src/java/org/apache/solr/util/stats/InstrumentedHttpRequestExecutor.java +++ b/solr/core/src/java/org/apache/solr/util/stats/InstrumentedHttpRequestExecutor.java @@ -49,8 +49,7 @@ public class InstrumentedHttpRequestExecutor extends HttpRequestExecutor try { final RequestLine requestLine = request.getRequestLine(); String schemeHostPort = null; - if (request instanceof HttpRequestWrapper) { - HttpRequestWrapper wrapper = (HttpRequestWrapper) request; + if (request instanceof HttpRequestWrapper wrapper) { if (wrapper.getTarget() != null) { schemeHostPort = wrapper.getTarget().getSchemeName() @@ -80,8 +79,7 @@ public class InstrumentedHttpRequestExecutor extends HttpRequestExecutor try { final RequestLine requestLine = request.getRequestLine(); String schemeHostPort = null; - if (request instanceof HttpRequestWrapper) { - HttpRequestWrapper wrapper = (HttpRequestWrapper) request; + if (request instanceof HttpRequestWrapper wrapper) { if (wrapper.getTarget() != null) { schemeHostPort = wrapper.getTarget().getSchemeName() diff --git a/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java b/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java index 71f240ac0a8..9cff065c06f 100644 --- a/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java +++ b/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java @@ -192,8 +192,7 @@ static void toSolrInputDocument(String prefix, SolrInputDocument doc, Object o) doc.addField(key, v); } }; - if (o instanceof MapWriter) { - MapWriter writer = (MapWriter) o; + if (o instanceof MapWriter writer) { writer._forEachEntry(consumer); } else if (o instanceof Map) { @SuppressWarnings({"unchecked"}) @@ -201,8 +200,7 @@ static void toSolrInputDocument(String prefix, SolrInputDocument doc, Object o) for (Map.Entry entry : map.entrySet()) { consumer.accept(entry.getKey(), entry.getValue()); } - } else if (o instanceof IteratorWriter) { - IteratorWriter writer = (IteratorWriter) o; + } else if (o instanceof IteratorWriter writer) { final String name = prefix != null ? prefix : "value"; try { writer.writeIter( @@ -348,11 +346,9 @@ public static void convertMetric( boolean simple, String separator, BiConsumer consumer) { - if (metric instanceof Counter) { - Counter counter = (Counter) metric; + if (metric instanceof Counter counter) { convertCounter(n, counter, propertyFilter, compact, consumer); - } else if (metric instanceof Gauge) { - Gauge gauge = (Gauge) metric; + } else if (metric instanceof Gauge gauge) { // unwrap if needed if (gauge instanceof SolrMetricManager.GaugeWrapper) { gauge = ((SolrMetricManager.GaugeWrapper) gauge).getGauge(); @@ -372,11 +368,9 @@ public static void convertMetric( throw ie; } } - } else if (metric instanceof Meter) { - Meter meter = (Meter) metric; + } else if (metric instanceof Meter meter) { convertMeter(n, meter, propertyFilter, simple, separator, consumer); - } else if (metric instanceof Timer) { - Timer timer = (Timer) metric; + } else if (metric instanceof Timer timer) { convertTimer(n, timer, propertyFilter, skipHistograms, simple, separator, consumer); } else if (metric instanceof Histogram) { if (!skipHistograms) { diff --git a/solr/core/src/java/org/apache/solr/util/vector/DenseVectorParser.java b/solr/core/src/java/org/apache/solr/util/vector/DenseVectorParser.java index 29001b3d789..14c4b4ff653 100644 --- a/solr/core/src/java/org/apache/solr/util/vector/DenseVectorParser.java +++ b/solr/core/src/java/org/apache/solr/util/vector/DenseVectorParser.java @@ -51,11 +51,10 @@ protected void parseVector() { } protected void parseIndexVector() { - if (!(inputValue instanceof List)) { + if (!(inputValue instanceof List inputVector)) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "incorrect vector format. " + errorMessage()); } - List inputVector = (List) inputValue; checkVectorDimension(inputVector.size()); if (inputVector.get(0) instanceof CharSequence) { for (int i = 0; i < dimension; i++) { diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-replication-legacy.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-replication-legacy.xml deleted file mode 100644 index ddd116be38a..00000000000 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-replication-legacy.xml +++ /dev/null @@ -1,62 +0,0 @@ - - - - - - ${tests.luceneMatchVersion:LATEST} - - - ${solr.data.dir:} - - - - - - - - true - - - - - - - - - - - - - - commit - - - http://127.0.0.1:TEST_PORT/solr/collection1 - 00:00:01 - COMPRESSION - - - - - - - max-age=30, public - - - - diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-test-misc.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-test-misc.xml index 1020db8319e..cbfe9752574 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-test-misc.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-test-misc.xml @@ -28,12 +28,6 @@ - - - - - - prefix-${solr.test.sys.prop2}-suffix diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml index 2c06096732c..856079f329e 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml @@ -59,7 +59,7 @@ - + ${solr.autoCommit.maxTime:-1} diff --git a/solr/core/src/test-files/solr/configsets/upload/with-lib-directive/solrconfig.xml b/solr/core/src/test-files/solr/configsets/upload/with-lib-directive/solrconfig.xml deleted file mode 100644 index 315bfffbf6c..00000000000 --- a/solr/core/src/test-files/solr/configsets/upload/with-lib-directive/solrconfig.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - - - - - - - ${solr.data.dir:} - - - - ${tests.luceneMatchVersion:LATEST} - - - - - - explicit - true - text - - - - - - - diff --git a/solr/core/src/test/org/apache/solr/SolrTestCaseJ4Test.java b/solr/core/src/test/org/apache/solr/SolrTestCaseJ4Test.java index 976eea1f0db..74d53019dbe 100644 --- a/solr/core/src/test/org/apache/solr/SolrTestCaseJ4Test.java +++ b/solr/core/src/test/org/apache/solr/SolrTestCaseJ4Test.java @@ -48,7 +48,7 @@ public static void beforeClass() throws Exception { FileUtils.touch(new File(tmpSolrHome, "core0/core.properties")); FileUtils.touch(new File(tmpSolrHome, "core1/core.properties")); - Files.copy(getFile("solr/solr.xml").toPath(), Path.of(tmpSolrHome, "solr.xml")); + Files.copy(getFile("solr/solr.xml"), Path.of(tmpSolrHome, "solr.xml")); initCore("solrconfig-minimal.xml", "schema-tiny.xml", tmpSolrHome, "core1"); } diff --git a/solr/core/src/test/org/apache/solr/TestDistributedSearch.java b/solr/core/src/test/org/apache/solr/TestDistributedSearch.java index 375f0cc479f..5100ea7c38f 100644 --- a/solr/core/src/test/org/apache/solr/TestDistributedSearch.java +++ b/solr/core/src/test/org/apache/solr/TestDistributedSearch.java @@ -1139,8 +1139,7 @@ public void test() throws Exception { "stat should be a Number: " + s + " -> " + svals.get(s).getClass(), svals.get(s) instanceof Number); // some loose assertions since we're iterating over various stats - if (svals.get(s) instanceof Double) { - Double val = (Double) svals.get(s); + if (svals.get(s) instanceof Double val) { assertFalse("stat shouldn't be NaN: " + s, val.isNaN()); assertFalse("stat shouldn't be Inf: " + s, val.isInfinite()); assertNotEquals("stat shouldn't be 0: " + s, 0.0D, val, 0.0); diff --git a/solr/core/src/test/org/apache/solr/TestTrie.java b/solr/core/src/test/org/apache/solr/TestTrie.java index ad1bd21f15a..ff2777960d1 100644 --- a/solr/core/src/test/org/apache/solr/TestTrie.java +++ b/solr/core/src/test/org/apache/solr/TestTrie.java @@ -444,8 +444,7 @@ public void testTrieFacet_PrecisionStep() throws Exception { private void checkPrecisionSteps(String fieldType) { FieldType type = h.getCore().getLatestSchema().getFieldType(fieldType); - if (type instanceof TrieField) { - TrieField field = (TrieField) type; + if (type instanceof TrieField field) { assertTrue(field.getPrecisionStep() > 0 && field.getPrecisionStep() < 64); } } diff --git a/solr/core/src/test/org/apache/solr/cli/CLIUtilsTest.java b/solr/core/src/test/org/apache/solr/cli/CLIUtilsTest.java new file mode 100644 index 00000000000..928266de7c0 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cli/CLIUtilsTest.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cli; + +import java.net.SocketException; +import java.net.URISyntaxException; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.SolrException; +import org.junit.Test; + +public class CLIUtilsTest extends SolrCloudTestCase { + + @Test + public void testDefaultSolrUrlWithNoProperties() { + System.clearProperty("solr.url.scheme"); + System.clearProperty("solr.tool.host"); + System.clearProperty("jetty.port"); + assertEquals( + "Default Solr URL should match with no properties set.", + "http://localhost:8983", + CLIUtils.getDefaultSolrUrl()); + } + + @Test + public void testDefaultSolrUrlWithProperties() { + System.setProperty("solr.url.scheme", "https"); + System.setProperty("solr.tool.host", "other.local"); + System.setProperty("jetty.port", "1234"); + assertEquals( + "Default Solr URL should match with custom properties set.", + "https://other.local:1234", + CLIUtils.getDefaultSolrUrl()); + } + + @Test + public void testCommunicationErrors() { + // communication errors + Exception serverException = new Exception(new SolrServerException("")); + assertTrue( + "SolrServerException should be communication error", + CLIUtils.checkCommunicationError(serverException)); + + Exception socketException = new RuntimeException(new Exception(new SocketException())); + assertTrue( + "SocketException should be communication error", + CLIUtils.checkCommunicationError(socketException)); + + // TODO See if this should be a communication error or not + // Exception parentException = new SolrServerException(new Exception()); + // assertTrue( + // "SolrServerException with different root cause should be communication error", + // CLIUtils.checkCommunicationError(parentException)); + + Exception rootException = new SolrServerException(""); + assertTrue( + "SolrServerException with no cause should be communication error", + CLIUtils.checkCommunicationError(rootException)); + + // non-communication errors + Exception exception1 = new NullPointerException(); + assertFalse( + "NullPointerException should not be communication error", + CLIUtils.checkCommunicationError(exception1)); + + Exception exception2 = new RuntimeException(new Exception()); + assertFalse( + "Exception should not be communication error", + CLIUtils.checkCommunicationError(exception2)); + } + + @Test + public void testCodeForAuthError() throws SolrException { + // auth errors + assertThrows( + "Forbidden (403) should throw SolrException", + SolrException.class, + () -> CLIUtils.checkCodeForAuthError(SolrException.ErrorCode.FORBIDDEN.code)); + assertThrows( + "Unauthorized (401) should throw SolrException", + SolrException.class, + () -> CLIUtils.checkCodeForAuthError(SolrException.ErrorCode.UNAUTHORIZED.code)); + + // non auth errors + CLIUtils.checkCodeForAuthError(SolrException.ErrorCode.BAD_REQUEST.code); + CLIUtils.checkCodeForAuthError(SolrException.ErrorCode.CONFLICT.code); + CLIUtils.checkCodeForAuthError(SolrException.ErrorCode.SERVER_ERROR.code); + CLIUtils.checkCodeForAuthError(0); // Unknown + CLIUtils.checkCodeForAuthError(200); // HTTP OK + } + + @Test + public void testResolveSolrUrl() { + assertEquals(CLIUtils.normalizeSolrUrl("http://localhost:8983/solr"), "http://localhost:8983"); + assertEquals(CLIUtils.normalizeSolrUrl("http://localhost:8983/solr/"), "http://localhost:8983"); + assertEquals(CLIUtils.normalizeSolrUrl("http://localhost:8983/"), "http://localhost:8983"); + assertEquals(CLIUtils.normalizeSolrUrl("http://localhost:8983"), "http://localhost:8983"); + assertEquals( + CLIUtils.normalizeSolrUrl("http://localhost:8983/solr/", false), "http://localhost:8983"); + } + + @Test + public void testPortExtraction() throws URISyntaxException { + assertEquals( + "Should extract explicit port from valid URL", + 8983, + CLIUtils.portFromUrl("http://localhost:8983")); + + assertEquals( + "Should extract explicit port from valid URL with trailing slash", + 1234, + CLIUtils.portFromUrl("http://localhost:1234/")); + + assertEquals( + "Should extract implicit HTTP port (80)", 80, CLIUtils.portFromUrl("http://localhost")); + + assertEquals( + "Should extract implicit HTTPS port (443)", 443, CLIUtils.portFromUrl("https://localhost")); + + // TODO See if we could be more lenient and fallback to defaults instead. + assertThrows( + "Should throw NullpointerException if no scheme provided", + NullPointerException.class, + () -> CLIUtils.portFromUrl("localhost")); + + // Note that a bunch of invalid URIs like "http::example.com", "http:/example.com" and + // "//example.com" are not throwing URISyntaxException. This however is an issue of + // java.lang.URI, which is very lenient. + } +} diff --git a/solr/core/src/test/org/apache/solr/cli/DeleteToolTest.java b/solr/core/src/test/org/apache/solr/cli/DeleteToolTest.java index 8f66d104d4e..45c48697663 100644 --- a/solr/core/src/test/org/apache/solr/cli/DeleteToolTest.java +++ b/solr/core/src/test/org/apache/solr/cli/DeleteToolTest.java @@ -60,8 +60,6 @@ public void testDeleteCollectionWithBasicAuth() throws Exception { "delete", "-c", "testDeleteCollectionWithBasicAuth", - "--delete-config", - "false", "-z", cluster.getZkClient().getZkServerAddress(), "--credentials", diff --git a/solr/core/src/test/org/apache/solr/cli/PostToolTest.java b/solr/core/src/test/org/apache/solr/cli/PostToolTest.java index ad37f73c318..9eb3e783a2d 100644 --- a/solr/core/src/test/org/apache/solr/cli/PostToolTest.java +++ b/solr/core/src/test/org/apache/solr/cli/PostToolTest.java @@ -76,6 +76,7 @@ public void testBasicRun() throws Exception { withBasicAuth(CollectionAdminRequest.createCollection(collection, "conf1", 1, 1, 0, 0)) .processAndWait(cluster.getSolrClient(), 10); + waitForState("creating", collection, activeClusterShape(1, 1)); File jsonDoc = File.createTempFile("temp", ".json"); @@ -281,20 +282,21 @@ public void testAppendUrlPath() { @Test public void testGuessType() { File f = new File("foo.doc"); - assertEquals("application/msword", PostTool.guessType(f)); + assertEquals("application/msword", PostTool.guessType(f.toPath())); f = new File("foobar"); - assertEquals("application/octet-stream", PostTool.guessType(f)); + assertEquals("application/octet-stream", PostTool.guessType(f.toPath())); f = new File("foo.json"); - assertEquals("application/json", PostTool.guessType(f)); + assertEquals("application/json", PostTool.guessType(f.toPath())); } @Test - public void testDoFilesMode() { + public void testDoFilesMode() throws IOException { PostTool postTool = new PostTool(); postTool.recursive = 0; postTool.dryRun = true; postTool.solrUpdateUrl = URI.create("http://localhost:8983/solr/fake/update"); - File dir = getFile("exampledocs"); + // TODO SOLR-8282 move to PATH + File dir = getFile("exampledocs").toFile(); int num = postTool.postFiles(new String[] {dir.toString()}, 0, null, null); assertEquals(2, num); } @@ -303,7 +305,7 @@ public void testDoFilesMode() { public void testDetectingIfRecursionPossibleInFilesMode() throws IOException { PostTool postTool = new PostTool(); postTool.recursive = 1; // This is the default - File dir = getFile("exampledocs"); + File dir = getFile("exampledocs").toFile(); File doc = File.createTempFile("temp", ".json"); assertTrue(postTool.recursionPossible(new String[] {dir.toString()})); assertFalse(postTool.recursionPossible(new String[] {doc.toString()})); @@ -311,12 +313,13 @@ public void testDetectingIfRecursionPossibleInFilesMode() throws IOException { } @Test - public void testRecursionAppliesToFilesMode() { + public void testRecursionAppliesToFilesMode() throws IOException { PostTool postTool = new PostTool(); postTool.recursive = 1; // This is the default postTool.dryRun = true; postTool.solrUpdateUrl = URI.create("http://localhost:8983/solr/fake/update"); - File dir = getFile("exampledocs"); + // TODO SOLR-8282 move to PATH + File dir = getFile("exampledocs").toFile(); int num = postTool.postFiles(new String[] {dir.toString()}, 0, null, null); assertEquals(2, num); } diff --git a/solr/core/src/test/org/apache/solr/cli/SolrCLITest.java b/solr/core/src/test/org/apache/solr/cli/SolrCLITest.java index 4b5290a1ff7..d045ae35501 100644 --- a/solr/core/src/test/org/apache/solr/cli/SolrCLITest.java +++ b/solr/core/src/test/org/apache/solr/cli/SolrCLITest.java @@ -20,15 +20,6 @@ import org.junit.Test; public class SolrCLITest extends SolrTestCase { - @Test - public void testResolveSolrUrl() { - assertEquals(SolrCLI.normalizeSolrUrl("http://localhost:8983/solr"), "http://localhost:8983"); - assertEquals(SolrCLI.normalizeSolrUrl("http://localhost:8983/solr/"), "http://localhost:8983"); - assertEquals(SolrCLI.normalizeSolrUrl("http://localhost:8983/"), "http://localhost:8983"); - assertEquals(SolrCLI.normalizeSolrUrl("http://localhost:8983"), "http://localhost:8983"); - assertEquals( - SolrCLI.normalizeSolrUrl("http://localhost:8983/solr/", false), "http://localhost:8983"); - } @Test public void testUptime() { diff --git a/solr/core/src/test/org/apache/solr/cli/StreamToolTest.java b/solr/core/src/test/org/apache/solr/cli/StreamToolTest.java new file mode 100644 index 00000000000..926e8aa91e3 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cli/StreamToolTest.java @@ -0,0 +1,373 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cli; + +import static org.apache.solr.cli.SolrCLI.findTool; +import static org.apache.solr.cli.SolrCLI.parseCmdLine; + +import java.io.BufferedWriter; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileWriter; +import java.io.LineNumberReader; +import java.io.PrintWriter; +import java.io.StringReader; +import java.io.StringWriter; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import org.apache.commons.cli.CommandLine; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrResponse; +import org.apache.solr.client.solrj.io.Tuple; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.util.SecurityJson; +import org.junit.BeforeClass; +import org.junit.Test; + +public class StreamToolTest extends SolrCloudTestCase { + + @BeforeClass + public static void setupClusterWithSecurityEnabled() throws Exception { + configureCluster(2).withSecurityJson(SecurityJson.SIMPLE).configure(); + } + + private > T withBasicAuth(T req) { + req.setBasicAuthCredentials(SecurityJson.USER, SecurityJson.PASS); + return req; + } + + @Test + @SuppressWarnings({"unchecked", "rawtypes"}) + public void testGetHeaderFromFirstTuple() { + Tuple tuple = new Tuple(new HashMap()); + tuple.put("field1", "blah"); + tuple.put("field2", "blah"); + tuple.put("field3", "blah"); + + String[] headers = StreamTool.getHeadersFromFirstTuple(tuple); + + assertEquals(headers.length, 3); + assertEquals(headers[0], "field1"); + assertEquals(headers[1], "field2"); + assertEquals(headers[2], "field3"); + } + + @Test + public void testGetOutputFields() { + String[] args = + new String[] { + "--fields", "field9, field2, field3, field4", + }; + StreamTool streamTool = new StreamTool(); + CommandLine cli = SolrCLI.processCommandLineArgs(streamTool, args); + String[] outputFields = StreamTool.getOutputFields(cli); + assert outputFields != null; + assertEquals(outputFields.length, 4); + assertEquals(outputFields[0], "field9"); + assertEquals(outputFields[1], "field2"); + assertEquals(outputFields[2], "field3"); + assertEquals(outputFields[3], "field4"); + } + + @Test + public void testReadExpression() throws Exception { + // This covers parameter substitution and expanded comments support. + + String[] args = {"file.expr", "one", "two", "three"}; + StringWriter stringWriter = new StringWriter(); + PrintWriter buf = new PrintWriter(stringWriter); + buf.println("/*"); + buf.println("Multi-line comment Comment..."); + buf.println("*/"); + buf.println(" /*"); + buf.println("Multi-line comment Comment..."); + buf.println(" */"); + buf.println("/*"); + buf.println("Multi-line comment ending with closing chars... */"); + buf.println("// Single line comment"); + buf.println("# Single line comment"); + buf.println(" // Single line comment"); + buf.println(" # Single line comment"); + buf.println("let(a=$1, b=$2,"); + buf.println("search($3))"); + buf.println(")"); + + String expr = stringWriter.toString(); + + LineNumberReader reader = new LineNumberReader(new StringReader(expr)); + String finalExpression = StreamTool.readExpression(reader, args); + // Strip the comment and insert the params in order. + assertEquals(finalExpression, "let(a=one, b=two,search(three)))"); + } + + @Test + public void testReadExpression2() throws Exception { + // This covers parameter substitution and expanded comments support. + + String[] args = {"file.expr", "id", "desc_s", "desc"}; + StringWriter stringWriter = new StringWriter(); + PrintWriter buf = new PrintWriter(stringWriter); + + buf.println("# Try me"); + buf.println("search(my_collection,q='*:*',fl='$1, $2',sort='id $3')"); + + String expr = stringWriter.toString(); + + LineNumberReader reader = new LineNumberReader(new StringReader(expr)); + String finalExpression = StreamTool.readExpression(reader, args); + // Strip the comment and insert the params in order. + assertEquals(finalExpression, "search(my_collection,q='*:*',fl='id, desc_s',sort='id desc')"); + } + + @Test + @SuppressWarnings({"unchecked", "rawtypes"}) + public void testReadStream() throws Exception { + StreamTool.StandardInStream inStream = new StreamTool.StandardInStream(); + List tuples = new ArrayList(); + try { + StringWriter stringWriter = new StringWriter(); + PrintWriter buf = new PrintWriter(stringWriter); + + buf.println("one two"); + buf.println("three four"); + buf.println("five six"); + + String expr = stringWriter.toString(); + ByteArrayInputStream inputStream = + new ByteArrayInputStream(expr.getBytes(Charset.defaultCharset())); + inStream.setInputStream(inputStream); + inStream.open(); + while (true) { + Tuple tuple = inStream.read(); + if (tuple.EOF) { + break; + } else { + tuples.add(tuple); + } + } + + } finally { + inStream.close(); + } + + assertEquals(tuples.size(), 3); + + String line1 = tuples.get(0).getString("line"); + String line2 = tuples.get(1).getString("line"); + String line3 = tuples.get(2).getString("line"); + + assertEquals("one two", line1); + assertEquals("three four", line2); + assertEquals("five six", line3); + } + + @Test + @SuppressWarnings({"unchecked", "rawtypes"}) + public void testLocalCatStream() throws Exception { + File localFile = File.createTempFile("topLevel1", ".txt"); + populateFileWithData(localFile.toPath()); + + StreamTool.LocalCatStream catStream = + new StreamTool.LocalCatStream(localFile.getAbsolutePath(), -1); + List tuples = new ArrayList(); + try { + catStream.open(); + while (true) { + Tuple tuple = catStream.read(); + if (tuple.EOF) { + break; + } else { + tuples.add(tuple); + } + } + + } finally { + catStream.close(); + } + + assertEquals(4, tuples.size()); + + for (int i = 0; i < 4; i++) { + Tuple t = tuples.get(i); + assertEquals(localFile.getName() + " line " + (i + 1), t.get("line")); + assertEquals(localFile.getAbsolutePath(), t.get("file")); + } + } + + @Test + @SuppressWarnings({"unchecked", "rawtypes"}) + public void testListToString() { + List stuff = new ArrayList(); + stuff.add("test1"); + stuff.add(3); + stuff.add(111.32322); + stuff.add("test3"); + String s = StreamTool.listToString(stuff, "|"); + assertEquals("test1|3|111.32322|test3", s); + } + + @Test + public void testStdInFailsWithRemoteWorker() throws Exception { + String expression = "echo(stdin())"; + + String[] args = + new String[] { + "stream", + "--execution", + "remote", + "--name", + "fakeCollection", + "--verbose", + "--zk-host", + cluster.getZkClient().getZkServerAddress(), + expression + }; + assertEquals(1, runTool(args)); + } + + @Test + public void testStdInSucceedsWithLocalWorker() throws Exception { + String expression = "echo(stdin())"; + + String[] args = + new String[] { + "stream", + "--execution", + "local", + "-v", + "-z", + cluster.getZkClient().getZkServerAddress(), + expression + }; + assertEquals(0, runTool(args)); + } + + @Test + public void testRunEchoStreamLocally() throws Exception { + + String expression = "echo(Hello)"; + File expressionFile = File.createTempFile("expression", ".EXPR"); + FileWriter writer = new FileWriter(expressionFile, Charset.defaultCharset()); + writer.write(expression); + writer.close(); + + // test passing in the file + // notice that we do not pass in zkHost or solrUrl for a simple echo run locally. + String[] args = { + "stream", + "--execution", + "local", + "--verbose", + "-zk-host", + cluster.getZkClient().getZkServerAddress(), + expressionFile.getAbsolutePath() + }; + + assertEquals(0, runTool(args)); + + // test passing in the expression directly + args = + new String[] { + "stream", + "--execution", + "local", + "--verbose", + "--zk-host", + cluster.getZkClient().getZkServerAddress(), + expression + }; + + assertEquals(0, runTool(args)); + } + + @Test + public void testRunEchoStreamRemotely() throws Exception { + String collectionName = "streamWorkerCollection"; + withBasicAuth(CollectionAdminRequest.createCollection(collectionName, "_default", 1, 1)) + .processAndWait(cluster.getSolrClient(), 10); + waitForState( + "Expected collection to be created with 1 shard and 1 replicas", + collectionName, + clusterShape(1, 1)); + + String expression = "echo(Hello)"; + File expressionFile = File.createTempFile("expression", ".EXPR"); + FileWriter writer = new FileWriter(expressionFile, Charset.defaultCharset()); + writer.write(expression); + writer.close(); + + // test passing in the file + String[] args = { + "stream", + "--execution", + "remote", + "-c", + collectionName, + "--verbose", + "-z", + cluster.getZkClient().getZkServerAddress(), + "--credentials", + SecurityJson.USER_PASS, + expressionFile.getAbsolutePath() + }; + + assertEquals(0, runTool(args)); + + // test passing in the expression directly + args = + new String[] { + "stream", + "--execution", + "remote", + "--name", + collectionName, + "--verbose", + "--zk-host", + cluster.getZkClient().getZkServerAddress(), + "--credentials", + SecurityJson.USER_PASS, + expression + }; + + assertEquals(0, runTool(args)); + } + + private int runTool(String[] args) throws Exception { + Tool tool = findTool(args); + assertTrue(tool instanceof StreamTool); + CommandLine cli = parseCmdLine(tool, args); + return tool.runTool(cli); + } + + // Copied from StreamExpressionTest.java + private static void populateFileWithData(Path dataFile) throws Exception { + // Files.createFile(dataFile); + try (final BufferedWriter writer = Files.newBufferedWriter(dataFile, StandardCharsets.UTF_8)) { + for (int i = 1; i <= 4; i++) { + writer.write(dataFile.getFileName() + " line " + i); + writer.newLine(); + } + } + } +} diff --git a/solr/core/src/test/org/apache/solr/cli/TestSolrCLIRunExample.java b/solr/core/src/test/org/apache/solr/cli/TestSolrCLIRunExample.java index 6c0522fd937..6da81edaa29 100644 --- a/solr/core/src/test/org/apache/solr/cli/TestSolrCLIRunExample.java +++ b/solr/core/src/test/org/apache/solr/cli/TestSolrCLIRunExample.java @@ -403,15 +403,6 @@ protected void testExample(String exampleName) throws Exception { // dump all the output written by the SolrCLI commands to stdout // System.out.println("\n\n"+toolOutput+"\n\n"); - File exampleSolrHomeDir = new File(solrExampleDir, exampleName + "/solr"); - assertTrue( - exampleSolrHomeDir.getAbsolutePath() - + " not found! run " - + exampleName - + " example failed; output: " - + toolOutput, - exampleSolrHomeDir.isDirectory()); - if ("techproducts".equals(exampleName)) { try (SolrClient solrClient = getHttpSolrClient("http://localhost:" + bindPort + "/solr", exampleName)) { @@ -506,7 +497,7 @@ public void testInteractiveSolrCloudExample() throws Exception { // verify Solr is running on the expected port and verify the collection exists String solrUrl = "http://localhost:" + bindPort + "/solr"; - if (!SolrCLI.safeCheckCollectionExists(solrUrl, collectionName, null)) { + if (!CLIUtils.safeCheckCollectionExists(solrUrl, collectionName, null)) { fail( "After running Solr cloud example, test collection '" + collectionName diff --git a/solr/core/src/test/org/apache/solr/cloud/AssignBackwardCompatibilityTest.java b/solr/core/src/test/org/apache/solr/cloud/AssignBackwardCompatibilityTest.java deleted file mode 100644 index 0587fada196..00000000000 --- a/solr/core/src/test/org/apache/solr/cloud/AssignBackwardCompatibilityTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.HashSet; -import java.util.Set; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.response.CollectionAdminResponse; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.util.NumberUtils; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.data.Stat; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -// TODO Remove in Solr 9.0 -/** - * Test for backward compatibility when users update from 6.x or 7.0 to 7.1, then the counter of - * collection does not exist in Zk - */ -public class AssignBackwardCompatibilityTest extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static final String COLLECTION = "collection1"; - - @BeforeClass - public static void setupCluster() throws Exception { - configureCluster(4) - .addConfig( - "conf1", TEST_PATH().resolve("configsets").resolve("cloud-dynamic").resolve("conf")) - .configure(); - CollectionAdminRequest.createCollection(COLLECTION, 1, 4).process(cluster.getSolrClient()); - cluster.waitForActiveCollection(COLLECTION, 1, 4); - } - - @Test - public void test() - throws IOException, SolrServerException, KeeperException, InterruptedException { - Set coreNames = new HashSet<>(); - Set coreNodeNames = new HashSet<>(); - - int numOperations = random().nextInt(15) + 15; - int numLiveReplicas = 4; - - boolean clearedCounter = false; - for (int i = 0; i < numOperations; i++) { - if (log.isInfoEnabled()) { - log.info("Collection counter={} i={}", getCounter(), i); - } - boolean deleteReplica = random().nextBoolean() && numLiveReplicas > 1; - // No need to clear counter more than one time - if (random().nextBoolean() && i > 5 && !clearedCounter) { - log.info("Clear collection counter"); - // clear counter - cluster.getZkClient().delete("/collections/" + COLLECTION + "/counter", -1, true); - clearedCounter = true; - } - if (deleteReplica) { - cluster.waitForActiveCollection(COLLECTION, 1, numLiveReplicas); - DocCollection dc = getCollectionState(COLLECTION); - Replica replica = - getRandomReplica(dc.getSlice("shard1"), (r) -> r.getState() == Replica.State.ACTIVE); - CollectionAdminRequest.deleteReplica(COLLECTION, "shard1", replica.getName()) - .process(cluster.getSolrClient()); - coreNames.remove(replica.getCoreName()); - numLiveReplicas--; - } else { - CollectionAdminResponse response = - CollectionAdminRequest.addReplicaToShard(COLLECTION, "shard1") - .process(cluster.getSolrClient()); - assertTrue(response.isSuccess()); - String coreName = response.getCollectionCoresStatus().keySet().iterator().next(); - assertFalse( - "Core name is not unique coreName=" + coreName + " " + coreNames, - coreNames.contains(coreName)); - coreNames.add(coreName); - numLiveReplicas++; - cluster.waitForActiveCollection(COLLECTION, 1, numLiveReplicas); - - Replica newReplica = - getCollectionState(COLLECTION).getReplicas().stream() - .filter(r -> r.getCoreName().equals(coreName)) - .findAny() - .get(); - String coreNodeName = newReplica.getName(); - assertFalse("Core node name is not unique", coreNodeNames.contains(coreName)); - coreNodeNames.add(coreNodeName); - } - } - } - - private int getCounter() throws InterruptedException { - try { - byte[] data = - cluster - .getZkClient() - .getData("/collections/" + COLLECTION + "/counter", null, new Stat(), true); - int count = NumberUtils.bytesToInt(data); - if (count < 0) throw new AssertionError("Found negative collection counter " + count); - return count; - } catch (KeeperException e) { - return -1; - } - } -} diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java index cba11e7d064..0fe1b755d62 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java @@ -37,6 +37,7 @@ import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; import org.apache.lucene.tests.util.TestUtil; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; @@ -132,10 +133,10 @@ public void testCreateCollWithDefaultClusterPropertiesNewFormat() throws Excepti String COLL_NAME = "CollWithDefaultClusterProperties"; try { V2Response rsp = - new V2Request.Builder("/cluster") - .withMethod(SolrRequest.METHOD.POST) + new V2Request.Builder("/cluster/properties") + .withMethod(SolrRequest.METHOD.PUT) .withPayload( - "{set-obj-property:{defaults : {collection:{numShards : 2 , nrtReplicas : 2}}}}") + "{\"defaults\": {\"collection\": {\"numShards\": 2, \"nrtReplicas\": 2}}}") .build() .process(cluster.getSolrClient()); @@ -171,15 +172,13 @@ public void testCreateCollWithDefaultClusterPropertiesNewFormat() throws Excepti // unset only a single value rsp = - new V2Request.Builder("/cluster") - .withMethod(SolrRequest.METHOD.POST) + new V2Request.Builder("/cluster/properties") + .withMethod(SolrRequest.METHOD.PUT) .withPayload( "{\n" - + " \"set-obj-property\": {\n" - + " \"defaults\" : {\n" - + " \"collection\": {\n" - + " \"nrtReplicas\": null\n" - + " }\n" + + " \"defaults\" : {\n" + + " \"collection\": {\n" + + " \"nrtReplicas\": null\n" + " }\n" + " }\n" + "}") @@ -198,9 +197,9 @@ public void testCreateCollWithDefaultClusterPropertiesNewFormat() throws Excepti assertNull(clusterProperty); rsp = - new V2Request.Builder("/cluster") - .withMethod(SolrRequest.METHOD.POST) - .withPayload("{set-obj-property:{defaults: {collection:null}}}") + new V2Request.Builder("/cluster/properties") + .withMethod(SolrRequest.METHOD.PUT) + .withPayload("{\"defaults\": {\"collection\": null}}") .build() .process(cluster.getSolrClient()); // assert that it is really gone in both old and new paths @@ -215,9 +214,9 @@ public void testCreateCollWithDefaultClusterPropertiesNewFormat() throws Excepti assertNull(clusterProperty); } finally { V2Response rsp = - new V2Request.Builder("/cluster") - .withMethod(SolrRequest.METHOD.POST) - .withPayload("{set-obj-property:{defaults: null}}") + new V2Request.Builder("/cluster/properties") + .withMethod(SolrRequest.METHOD.PUT) + .withPayload("{\"defaults\": null}") .build() .process(cluster.getSolrClient()); } @@ -1027,7 +1026,7 @@ public void testDeleteAliasedCollection() throws Exception { delete.setFollowAliases(false); delete.process(solrClient); ClusterState state = solrClient.getClusterState(); - assertFalse(state.getCollectionsMap().toString(), state.hasCollection(collectionName1)); + assertFalse(collectionNamesString(state), state.hasCollection(collectionName1)); // search should still work, returning results from collection 2 assertDoc(solrClient, collectionName1, "2"); // aliased assertDoc(solrClient, collectionName2, "2"); // direct @@ -1048,7 +1047,7 @@ public void testDeleteAliasedCollection() throws Exception { state = solrClient.getClusterState(); // the collection is gone - assertFalse(state.getCollectionsMap().toString(), state.hasCollection(collectionName2)); + assertFalse(collectionNamesString(state), state.hasCollection(collectionName2)); // and the alias is gone RetryUtil.retryUntil( @@ -1067,6 +1066,10 @@ public void testDeleteAliasedCollection() throws Exception { }); } + private static String collectionNamesString(ClusterState state) { + return state.collectionStream().map(Object::toString).collect(Collectors.joining(",")); + } + private void assertDoc(CloudSolrClient solrClient, String collection, String id) throws Exception { QueryResponse rsp = solrClient.query(collection, params(CommonParams.Q, "*:*")); diff --git a/solr/core/src/test/org/apache/solr/cloud/DistribDocExpirationUpdateProcessorTest.java b/solr/core/src/test/org/apache/solr/cloud/DistribDocExpirationUpdateProcessorTest.java index 143f1f4dee8..dfd9aa4dc14 100644 --- a/solr/core/src/test/org/apache/solr/cloud/DistribDocExpirationUpdateProcessorTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/DistribDocExpirationUpdateProcessorTest.java @@ -384,8 +384,7 @@ public String toString() { @Override public boolean equals(Object other) { - if (other instanceof ReplicaData) { - ReplicaData that = (ReplicaData) other; + if (other instanceof ReplicaData that) { return this.shardName.equals(that.shardName) && this.coreName.equals(that.coreName) && (this.indexVersion == that.indexVersion) diff --git a/solr/core/src/test/org/apache/solr/cloud/DistributedQueueTest.java b/solr/core/src/test/org/apache/solr/cloud/DistributedQueueTest.java index 213a5db6d6d..152a5742bd9 100644 --- a/solr/core/src/test/org/apache/solr/cloud/DistributedQueueTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/DistributedQueueTest.java @@ -295,7 +295,13 @@ private void forceSessionExpire() throws InterruptedException, TimeoutException zkClient .getCuratorFramework() .getConnectionStateListenable() - .addListener((OnDisconnect) hasDisconnected::countDown); + .addListener( + (OnDisconnect) + ((sessionExpired) -> { + if (sessionExpired) { + hasDisconnected.countDown(); + } + })); long sessionId = zkClient.getZkSessionId(); zkServer.expire(sessionId); hasDisconnected.await(10, TimeUnit.SECONDS); diff --git a/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java index 5da2f862ea2..32f921747d9 100644 --- a/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java @@ -73,10 +73,9 @@ public void testSimpleSliceLeaderElection() throws Exception { "shard1", jetty .getCoreContainer() - .getCores() + .getCoreDescriptors() .iterator() .next() - .getCoreDescriptor() .getCloudDescriptor() .getShardId()); String jettyNodeName = jetty.getNodeName(); // must get before shutdown diff --git a/solr/core/src/test/org/apache/solr/cloud/MockSimpleZkController.java b/solr/core/src/test/org/apache/solr/cloud/MockSimpleZkController.java index a553e1064b0..2c678132a86 100644 --- a/solr/core/src/test/org/apache/solr/cloud/MockSimpleZkController.java +++ b/solr/core/src/test/org/apache/solr/cloud/MockSimpleZkController.java @@ -17,23 +17,16 @@ package org.apache.solr.cloud; import java.io.IOException; -import java.util.List; import java.util.concurrent.TimeoutException; -import java.util.function.Supplier; import org.apache.solr.core.CloudConfig; import org.apache.solr.core.CoreContainer; -import org.apache.solr.core.CoreDescriptor; public class MockSimpleZkController extends ZkController { public MockSimpleZkController( - CoreContainer cc, - String zkServerAddress, - int zkClientConnectTimeout, - CloudConfig cloudConfig, - Supplier> descriptorsSupplier) + CoreContainer cc, String zkServerAddress, int zkClientConnectTimeout, CloudConfig cloudConfig) throws InterruptedException, TimeoutException, IOException { - super(cc, zkServerAddress, zkClientConnectTimeout, cloudConfig, descriptorsSupplier); + super(cc, zkServerAddress, zkClientConnectTimeout, cloudConfig); } @Override diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java index 8e7e45333f4..e611902898f 100644 --- a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java @@ -1412,16 +1412,14 @@ public void testDoubleAssignment() throws Exception { reader.forceUpdateCollection(COLLECTION); ClusterState state = reader.getClusterState(); - int numFound = 0; - Map collectionsMap = state.getCollectionsMap(); - for (Map.Entry entry : collectionsMap.entrySet()) { - DocCollection collection = entry.getValue(); - for (Slice slice : collection.getSlices()) { - if (slice.getReplicasMap().get("core_node1") != null) { - numFound++; - } - } - } + long numFound = + state + .collectionStream() + .map(DocCollection::getSlices) + .flatMap(Collection::stream) + .filter(slice -> slice.getReplicasMap().get("core_node1") != null) + .count(); + assertEquals("Shard was found more than once in ClusterState", 1, numFound); } finally { close(overseerClient); @@ -1947,17 +1945,16 @@ public Void answer(InvocationOnMock invocation) { } private SolrCloudManager getCloudDataProvider(ZkStateReader zkStateReader) { - var httpSolrClient = + var httpSolrClientBuilder = new Http2SolrClient.Builder() .withIdleTimeout(30000, TimeUnit.MILLISECONDS) - .withConnectionTimeout(15000, TimeUnit.MILLISECONDS) - .build(); + .withConnectionTimeout(15000, TimeUnit.MILLISECONDS); var cloudSolrClient = new CloudHttp2SolrClient.Builder(new ZkClientClusterStateProvider(zkStateReader)) - .withHttpClient(httpSolrClient) + .withInternalClientBuilder(httpSolrClientBuilder) .build(); solrClients.add(cloudSolrClient); - solrClients.add(httpSolrClient); + solrClients.add(httpSolrClientBuilder.build()); SolrClientCloudManager sccm = new SolrClientCloudManager(cloudSolrClient, null); sccm.getClusterStateProvider().connect(); return sccm; diff --git a/solr/core/src/test/org/apache/solr/cloud/ReindexCollectionTest.java b/solr/core/src/test/org/apache/solr/cloud/ReindexCollectionTest.java index 200e7b68974..0b79aa1c336 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ReindexCollectionTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ReindexCollectionTest.java @@ -367,7 +367,8 @@ public void testFailure() throws Exception { // verify that the target and checkpoint collections don't exist cloudManager .getClusterState() - .forEachCollection( + .collectionStream() + .forEach( coll -> { assertFalse( coll.getName() + " still exists", diff --git a/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java b/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java index 1716d09a45f..8e2e728944f 100644 --- a/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java @@ -24,6 +24,7 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.cli.CommandLine; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.cli.CLIUtils; import org.apache.solr.cli.CreateTool; import org.apache.solr.cli.DeleteTool; import org.apache.solr.cli.HealthcheckTool; @@ -174,7 +175,7 @@ protected void doTestDeleteAction(String testCollectionName, String solrUrl) thr CommandLine cli = SolrCLI.processCommandLineArgs(tool, args); assertEquals("Delete action failed!", 0, tool.runTool(cli)); assertFalse( - SolrCLI.safeCheckCollectionExists( + CLIUtils.safeCheckCollectionExists( solrUrl, testCollectionName, null)); // it should not exist anymore } } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java index b41d698ac4c..886ffb6afca 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java @@ -287,7 +287,7 @@ public void testCreateWithTrust() throws Exception { private void setupBaseConfigSet(String baseConfigSetName, Map oldProps) throws Exception { - final Path configDir = getFile("solr").toPath().resolve("configsets/configset-2/conf"); + final Path configDir = getFile("solr").resolve("configsets/configset-2/conf"); final Path tmpConfigDir = createTempDir(); tmpConfigDir.toFile().deleteOnExit(); PathUtils.copyDirectory(configDir, tmpConfigDir); @@ -1421,48 +1421,6 @@ public void testUploadWithScriptUpdateProcessor() throws Exception { scriptRequest("newcollection2"); } - @Test - public void testUploadWithLibDirective() throws Exception { - final String untrustedSuffix = "-untrusted"; - uploadConfigSetWithAssertions("with-lib-directive", untrustedSuffix, null); - // try to create a collection with the uploaded configset - ignoreException("without any authentication in place"); - Throwable thrown = - expectThrows( - SolrClient.RemoteSolrException.class, - () -> { - createCollection( - "newcollection3", - "with-lib-directive" + untrustedSuffix, - 1, - 1, - cluster.getSolrClient()); - }); - unIgnoreException("without any authentication in place"); - - assertThat(thrown.getMessage(), containsString("Underlying core creation failed")); - - // Authorization on - final String trustedSuffix = "-trusted"; - uploadConfigSetWithAssertions("with-lib-directive", trustedSuffix, "solr"); - // try to create a collection with the uploaded configset - CollectionAdminResponse resp = - createCollection( - "newcollection3", "with-lib-directive" + trustedSuffix, 1, 1, cluster.getSolrClient()); - - SolrInputDocument doc = sdoc("id", "4055", "subject", "Solr"); - cluster.getSolrClient().add("newcollection3", doc); - cluster.getSolrClient().commit("newcollection3"); - assertEquals( - "4055", - cluster - .getSolrClient() - .query("newcollection3", params("q", "*:*")) - .getResults() - .get(0) - .get("id")); - } - @Test public void testUploadWithForbiddenContent() throws Exception { // Uploads a config set containing a script, a class file and jar file, will return 400 error @@ -1560,7 +1518,7 @@ private long uploadBadConfigSet(String configSetName, String suffix, String user // Read single file from sample configs. This should fail the unzipping return uploadGivenConfigSet( - SolrTestCaseJ4.getFile("solr/configsets/upload/regular/solrconfig.xml"), + SolrTestCaseJ4.getFile("solr/configsets/upload/regular/solrconfig.xml").toFile(), configSetName, suffix, username, @@ -1605,7 +1563,7 @@ private long uploadGivenConfigSet( try { return (new Upload()) .setConfigSetName(configSetName + suffix) - .setUploadFile(file, "application/zip") + .setUploadFile(file.toPath(), "application/zip") .setOverwrite(overwrite ? true : null) // expect server default to be 'false' .setCleanup(cleanup ? true : null) // expect server default to be 'false' .setBasicAuthCredentials(username, username) // for our MockAuthenticationPlugin @@ -1629,7 +1587,7 @@ private long uploadSingleConfigSetFile( boolean v2) throws IOException { // Read single file from sample configs - final File file = SolrTestCaseJ4.getFile(localFilePath); + final File file = SolrTestCaseJ4.getFile(localFilePath).toFile(); if (v2) { // TODO: switch to use V2Request @@ -1662,7 +1620,7 @@ private long uploadSingleConfigSetFile( .setConfigSetName(configSetName + suffix) .setFilePath(uploadPath) // NOTE: server doesn't actually care, and test plumbing doesn't tell us - .setUploadFile(file, "application/octet-stream") + .setUploadFile(file.toPath(), "application/octet-stream") .setOverwrite(overwrite ? true : null) // expect server default to be 'false' .setCleanup(cleanup ? true : null) // expect server default to be 'false' .setBasicAuthCredentials(username, username) // for our MockAuthenticationPlugin @@ -1682,7 +1640,7 @@ private long uploadSingleConfigSetFile( private File createTempZipFile(String directoryPath) { try { final File zipFile = createTempFile("configset", "zip").toFile(); - final File directory = SolrTestCaseJ4.getFile(directoryPath); + final File directory = SolrTestCaseJ4.getFile(directoryPath).toFile(); if (log.isInfoEnabled()) { log.info("Directory: {}", directory.getAbsolutePath()); } @@ -1703,7 +1661,7 @@ private File createTempZipFile(String directoryPath) { private File createTempZipFileWithForbiddenTypes(String file) { try { final File zipFile = createTempFile("configset", "zip").toFile(); - final File directory = SolrTestCaseJ4.getFile(file); + final File directory = SolrTestCaseJ4.getFile(file).toFile(); if (log.isInfoEnabled()) { log.info("Directory: {}", directory.getAbsolutePath()); } @@ -1721,7 +1679,7 @@ private File createTempZipFileWithForbiddenTypes(String file) { private File createTempZipFileWithForbiddenContent(String resourcePath) { try { final File zipFile = createTempFile("configset", "zip").toFile(); - final File directory = SolrTestCaseJ4.getFile(resourcePath); + final File directory = SolrTestCaseJ4.getFile(resourcePath).toFile(); if (log.isInfoEnabled()) { log.info("Directory: {}", directory.getAbsolutePath()); } @@ -1892,7 +1850,7 @@ private static Object getObjectByPath(Map root, List hierarchy) { private byte[] readFile(String fname) throws IOException { byte[] buf = null; - try (FileInputStream fis = new FileInputStream(getFile(fname))) { + try (FileInputStream fis = new FileInputStream(getFile(fname).toFile())) { buf = new byte[fis.available()]; fis.read(buf); } @@ -1903,7 +1861,7 @@ private byte[] readFile(String fname) throws IOException { public void testDeleteErrors() throws Exception { final String baseUrl = cluster.getJettySolrRunners().get(0).getBaseUrl().toString(); final SolrClient solrClient = getHttpSolrClient(baseUrl); - final Path configDir = getFile("solr").toPath().resolve("configsets/configset-2/conf"); + final Path configDir = getFile("solr").resolve("configsets/configset-2/conf"); final Path tmpConfigDir = createTempDir(); tmpConfigDir.toFile().deleteOnExit(); // Ensure ConfigSet is immutable diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIZkFailure.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIZkFailure.java index 3afb80e9e03..bc644f1a362 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIZkFailure.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIZkFailure.java @@ -141,7 +141,7 @@ public void testCreateZkFailure() throws Exception { private void setupBaseConfigSet(String baseConfigSetName, Map oldProps) throws Exception { - final Path configDir = getFile("solr").toPath().resolve("configsets/configset-2/conf"); + final Path configDir = getFile("solr").resolve("configsets/configset-2/conf"); final Path tmpConfigDir = createTempDir(); tmpConfigDir.toFile().deleteOnExit(); PathUtils.copyDirectory(configDir, tmpConfigDir); diff --git a/solr/core/src/test/org/apache/solr/cloud/TestLeaderElectionZkExpiry.java b/solr/core/src/test/org/apache/solr/cloud/TestLeaderElectionZkExpiry.java index a871fc7176c..2c8520ddc22 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestLeaderElectionZkExpiry.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestLeaderElectionZkExpiry.java @@ -18,7 +18,6 @@ import java.lang.invoke.MethodHandles; import java.nio.file.Path; -import java.util.Collections; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import org.apache.lucene.tests.util.LuceneTestCase.BadApple; @@ -63,8 +62,7 @@ public void testLeaderElectionWithZkExpiry() throws Exception { ExecutorUtil.newMDCAwareSingleThreadExecutor( new SolrNamedThreadFactory(this.getTestName())); try (ZkController zkController = - new ZkController( - cc, server.getZkAddress(), 15000, cloudConfig, Collections::emptyList)) { + new ZkController(cc, server.getZkAddress(), 15000, cloudConfig)) { threadExecutor.execute( () -> { TimeOut timeout = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME); diff --git a/solr/core/src/test/org/apache/solr/cloud/TestOnReconnectListenerSupport.java b/solr/core/src/test/org/apache/solr/cloud/TestOnReconnectListenerSupport.java index 040fbc5e7eb..e080a4ddc7f 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestOnReconnectListenerSupport.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestOnReconnectListenerSupport.java @@ -85,8 +85,7 @@ public void test() throws Exception { assertNotNull("ZkController returned null OnReconnect listeners", listeners); ZkIndexSchemaReader expectedListener = null; for (OnReconnect listener : listeners) { - if (listener instanceof ZkIndexSchemaReader) { - ZkIndexSchemaReader reader = (ZkIndexSchemaReader) listener; + if (listener instanceof ZkIndexSchemaReader reader) { if (leaderCoreId.equals(reader.getUniqueCoreId())) { expectedListener = reader; break; @@ -122,8 +121,7 @@ public void test() throws Exception { expectedListener = null; // reset for (OnReconnect listener : listeners) { - if (listener instanceof ZkIndexSchemaReader) { - ZkIndexSchemaReader reader = (ZkIndexSchemaReader) listener; + if (listener instanceof ZkIndexSchemaReader reader) { if (leaderCoreId.equals(reader.getUniqueCoreId())) { fail( "Previous core " @@ -153,8 +151,7 @@ public void test() throws Exception { listeners = zkController.getCurrentOnReconnectListeners(); for (OnReconnect listener : listeners) { - if (listener instanceof ZkIndexSchemaReader) { - ZkIndexSchemaReader reader = (ZkIndexSchemaReader) listener; + if (listener instanceof ZkIndexSchemaReader reader) { if (reloadedLeaderCoreId.equals(reader.getUniqueCoreId())) { fail( "Previous core " diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java index 6bda0dd6b53..c5be38b570c 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java @@ -166,8 +166,7 @@ public void testGetHostName() throws Exception { try { CloudConfig cloudConfig = new CloudConfig.CloudConfigBuilder("127.0.0.1", 8983).build(); - zkController = - new ZkController(cc, server.getZkAddress(), TIMEOUT, cloudConfig, () -> null); + zkController = new ZkController(cc, server.getZkAddress(), TIMEOUT, cloudConfig); } catch (IllegalArgumentException e) { fail("ZkController did not normalize host name correctly"); } finally { @@ -233,8 +232,7 @@ public List getCoreDescriptors() { Boolean.getBoolean("solr.distributedCollectionConfigSetExecution")) .build(); zkController = - new ZkController( - cc, cluster.getZkServer().getZkAddress(), TIMEOUT, cloudConfig, () -> null); + new ZkController(cc, cluster.getZkServer().getZkAddress(), TIMEOUT, cloudConfig); zkControllerRef.set(zkController); zkController @@ -399,7 +397,7 @@ public void testTouchConfDir() throws Exception { try { CloudConfig cloudConfig = new CloudConfig.CloudConfigBuilder("127.0.0.1", 8983).build(); try (ZkController zkController = - new ZkController(cc, server.getZkAddress(), TIMEOUT, cloudConfig, () -> null)) { + new ZkController(cc, server.getZkAddress(), TIMEOUT, cloudConfig)) { final Path dir = createTempDir(); final String configsetName = "testconfigset"; try (ZkSolrResourceLoader loader = @@ -478,8 +476,7 @@ public void testCheckNoOldClusterstate() throws Exception { cc, server.getZkAddress(), TIMEOUT, - new CloudConfig.CloudConfigBuilder("127.0.0.1", 8983 + index).build(), - () -> null); + new CloudConfig.CloudConfigBuilder("127.0.0.1", 8983 + index).build()); } catch (Exception e) { exception.compareAndSet(null, e); } finally { diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java index 28335c920eb..7e610b2c2da 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java @@ -1320,8 +1320,7 @@ protected void deleteAndUpdateCount( public static int getHashRangeIdx(DocRouter router, List ranges, String id) { int hash = 0; - if (router instanceof HashBasedRouter) { - HashBasedRouter hashBasedRouter = (HashBasedRouter) router; + if (router instanceof HashBasedRouter hashBasedRouter) { hash = hashBasedRouter.sliceHash(id, null, null, null); } for (int i = 0; i < ranges.size(); i++) { diff --git a/solr/core/src/test/org/apache/solr/core/ExitableDirectoryReaderTest.java b/solr/core/src/test/org/apache/solr/core/ExitableDirectoryReaderTest.java index 1ceb9636b20..f2e4aaa3c8f 100644 --- a/solr/core/src/test/org/apache/solr/core/ExitableDirectoryReaderTest.java +++ b/solr/core/src/test/org/apache/solr/core/ExitableDirectoryReaderTest.java @@ -32,7 +32,7 @@ */ public class ExitableDirectoryReaderTest extends SolrTestCaseJ4 { - static int NUM_DOCS = 100; + static final int NUM_DOCS = 100; static final String assertionString = "/response/numFound==" + NUM_DOCS; static final String failureAssertionString = "/responseHeader/partialResults==true]"; static final String longTimeout = "10000"; @@ -70,9 +70,26 @@ public void testPrefixQuery() throws Exception { // this time we should get a query cache hit and hopefully no exception? this may change in the // future if time checks are put into other places. - assertJQ(req("q", q, "timeAllowed", "1", "sleep", sleep), assertionString); + + // 2024-4-15: it did change..., and now this fails with 1 or 2 ms and passes with 3ms... I see + // no way this won't be terribly brittle. Maybe TestInjection of some sort to bring this back? + + // assertJQ(req("q", q, "timeAllowed", "2", "sleep", sleep), assertionString); + + // The idea that the request won't time out due to caching is a flawed test methodology, + // It relies on the test running quickly and not stalling. The above test should possibly + // be doing something along the lines of this (but we lack api for it) + // + // SolrCores solrCores = ExitableDirectoryReaderTest.h.getCoreContainer().solrCores; + // List cores = solrCores.getCores(); + // for (SolrCore core : cores) { + // if (<<< find the right core >>> ) { + // ((SolrCache)core.getSearcher().get().<<>> + // } + // } // now do the same for the filter cache + // 2024-4-15: this still passes probably because *:* is so fast, but it still worries me assertJQ(req("q", "*:*", "fq", q, "timeAllowed", "1", "sleep", sleep), failureAssertionString); // make sure that the result succeeds this time, and that a bad filter wasn't cached diff --git a/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java b/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java index 8fd60619122..54f7406635d 100644 --- a/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java +++ b/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.Map; +import java.util.Set; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.lucene99.Lucene99Codec.Mode; import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; @@ -238,7 +239,7 @@ public void testCompressionModeDefault() throws IOException { config.get("codecFactory").attr("class")); assertTrue( "Unexpected configuration of codec factory for this test. Expecting empty element", - config.get("codecFactory").getAll(null, (String) null).isEmpty()); + config.get("codecFactory").getAll(null, Set.of()).isEmpty()); IndexSchema schema = IndexSchemaFactory.buildIndexSchema("schema_codec.xml", config); CoreContainer coreContainer = h.getCoreContainer(); diff --git a/solr/core/src/test/org/apache/solr/core/TestConfLoadPerf.java b/solr/core/src/test/org/apache/solr/core/TestConfLoadPerf.java index ee9177b2105..4c348104b35 100644 --- a/solr/core/src/test/org/apache/solr/core/TestConfLoadPerf.java +++ b/solr/core/src/test/org/apache/solr/core/TestConfLoadPerf.java @@ -84,7 +84,7 @@ public InputStream openResource(String resource) throws IOException { long startTime = System.currentTimeMillis(); int numReads = 100; for (int i = 0; i < numReads; i++) { - allConfigs.add(SolrConfig.readFromResourceLoader(srl, "solrconfig.xml", true, null)); + allConfigs.add(SolrConfig.readFromResourceLoader(srl, "solrconfig.xml", null)); } assertEquals(numReads, allConfigs.size()); System.gc(); diff --git a/solr/core/src/test/org/apache/solr/core/TestConfig.java b/solr/core/src/test/org/apache/solr/core/TestConfig.java index 2c17cbf1e1a..745fa21c4c5 100644 --- a/solr/core/src/test/org/apache/solr/core/TestConfig.java +++ b/solr/core/src/test/org/apache/solr/core/TestConfig.java @@ -45,20 +45,6 @@ public static void beforeClass() throws Exception { public void testLib() throws IOException { SolrResourceLoader loader = h.getCore().getResourceLoader(); InputStream data = null; - String[] expectedFiles = - new String[] { - "empty-file-main-lib.txt", - "empty-file-a1.txt", - "empty-file-a2.txt", - "empty-file-b1.txt", - "empty-file-b2.txt", - "empty-file-c1.txt" - }; - for (String f : expectedFiles) { - data = loader.openResource(f); - assertNotNull("Should have found file " + f, data); - data.close(); - } String[] unexpectedFiles = new String[] {"empty-file-c2.txt", "empty-file-d2.txt"}; for (String f : unexpectedFiles) { data = null; diff --git a/solr/core/src/test/org/apache/solr/core/TestConfigSets.java b/solr/core/src/test/org/apache/solr/core/TestConfigSets.java index df9d72cb508..8bec679b63d 100644 --- a/solr/core/src/test/org/apache/solr/core/TestConfigSets.java +++ b/solr/core/src/test/org/apache/solr/core/TestConfigSets.java @@ -91,7 +91,8 @@ public void testConfigSetServiceFindsConfigSets() { @Test public void testNonExistentConfigSetThrowsException() { - final CoreContainer container = setupContainer(getFile("solr/configsets").getAbsolutePath()); + final CoreContainer container = + setupContainer(getFile("solr/configsets").toAbsolutePath().toString()); try { Exception thrown = expectThrows( @@ -112,7 +113,7 @@ public void testConfigSetOnCoreReload() throws IOException { Path testDirectory = createTempDir("core-reload"); Path configSetsDir = testDirectory.resolve("configsets"); - PathUtils.copyDirectory(getFile("solr/configsets").toPath(), configSetsDir); + PathUtils.copyDirectory(getFile("solr/configsets"), configSetsDir); String csd = configSetsDir.toAbsolutePath().toString(); System.setProperty("configsets", csd); @@ -129,7 +130,7 @@ public void testConfigSetOnCoreReload() throws IOException { // Now copy in a config with a /dump handler and reload Files.copy( - getFile("solr/collection1/conf/solrconfig-withgethandler.xml").toPath(), + getFile("solr/collection1/conf/solrconfig-withgethandler.xml"), configSetsDir.resolve("configset-2/conf").resolve("solrconfig.xml"), StandardCopyOption.REPLACE_EXISTING); container.reload("core1"); diff --git a/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java b/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java index 4aaea27c857..bc93c3d6ad0 100644 --- a/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java +++ b/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java @@ -61,7 +61,7 @@ public class TestCoreContainer extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() { oldSolrHome = System.getProperty(SOLR_HOME_PROP); - System.setProperty("configsets", getFile("solr/configsets").getAbsolutePath()); + System.setProperty("configsets", getFile("solr/configsets").toAbsolutePath().toString()); } @AfterClass @@ -320,7 +320,7 @@ public void testDeleteBadCores() { MockCoresLocator cl = new MockCoresLocator(); Path solrHome = createTempDir(); - System.setProperty("configsets", getFile("solr/configsets").getAbsolutePath()); + System.setProperty("configsets", getFile("solr/configsets").toAbsolutePath().toString()); final CoreContainer cc = new CoreContainer(SolrXmlConfig.fromString(solrHome, CONFIGSETS_SOLR_XML), cl); @@ -939,7 +939,7 @@ public void testCoreInitFailuresOnReload() throws Exception { Path solrHome = createTempDir(); - System.setProperty("configsets", getFile("solr/configsets").getAbsolutePath()); + System.setProperty("configsets", getFile("solr/configsets").toAbsolutePath().toString()); final CoreContainer cc = new CoreContainer(SolrXmlConfig.fromString(solrHome, CONFIGSETS_SOLR_XML), cl); @@ -986,11 +986,11 @@ public void testCoreInitFailuresOnReload() throws Exception { Path confDir = Path.of(cc.getSolrHome(), "col_bad", "conf"); Files.createDirectories(confDir); Files.copy( - getFile("solr/collection1/conf/solrconfig-defaults.xml").toPath(), + getFile("solr/collection1/conf/solrconfig-defaults.xml"), confDir.resolve("solrconfig.xml"), StandardCopyOption.REPLACE_EXISTING); Files.copy( - getFile("solr/collection1/conf/schema-minimal.xml").toPath(), + getFile("solr/collection1/conf/schema-minimal.xml"), confDir.resolve("schema.xml"), StandardCopyOption.REPLACE_EXISTING); cc.create("col_bad", Map.of()); @@ -1107,7 +1107,7 @@ public void testCoreInitFailuresOnReload() throws Exception { // ---- // fix col_bad's config (again) and RELOAD to fix failure Files.copy( - getFile("solr/collection1/conf/solrconfig-defaults.xml").toPath(), + getFile("solr/collection1/conf/solrconfig-defaults.xml"), confDir.resolve("solrconfig.xml"), StandardCopyOption.REPLACE_EXISTING); cc.reload("col_bad"); diff --git a/solr/core/src/test/org/apache/solr/core/TestFileSystemConfigSetService.java b/solr/core/src/test/org/apache/solr/core/TestFileSystemConfigSetService.java index f9695a990e5..e4ceb1b831c 100644 --- a/solr/core/src/test/org/apache/solr/core/TestFileSystemConfigSetService.java +++ b/solr/core/src/test/org/apache/solr/core/TestFileSystemConfigSetService.java @@ -17,7 +17,9 @@ package org.apache.solr.core; import static org.apache.solr.core.FileSystemConfigSetService.METADATA_FILE; +import static org.hamcrest.Matchers.hasItem; +import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -49,13 +51,43 @@ public static void afterClass() throws Exception { fileSystemConfigSetService = null; } + @Test + public void testIgnoresFileUploadsOutsideOfConfigSetDirectory() throws IOException { + final var initialNumConfigs = fileSystemConfigSetService.listConfigs().size(); + final String configName = "fileEscapeTestConfig"; + final var specificConfigSetBase = configSetBase.resolve(configName); + + fileSystemConfigSetService.uploadConfig(configName, configset("cloud-minimal")); + assertEquals(fileSystemConfigSetService.listConfigs().size(), initialNumConfigs + 1); + assertTrue(fileSystemConfigSetService.checkConfigExists(configName)); + + // This succeeds, as the file is an allowed type and the path doesn't attempt to escape the + // configset's root + byte[] testdata = "test data".getBytes(StandardCharsets.UTF_8); + fileSystemConfigSetService.uploadFileToConfig(configName, "validPath", testdata, true); + final var knownFiles = fileSystemConfigSetService.getAllConfigFiles(configName); + assertThat(knownFiles, hasItem("validPath")); + assertTrue(Files.exists(specificConfigSetBase.resolve("validPath"))); + + // Each of these will fail "quietly" as ConfigSetService opts to log warnings but otherwise not + // surface validation errors to enable bulk uploading + final var invalidFilePaths = + List.of( + ".." + File.separator + "escapePath", + "foo" + File.separator + ".." + File.separator + ".." + File.separator + "bar"); + for (String invalidFilePath : invalidFilePaths) { + fileSystemConfigSetService.uploadFileToConfig(configName, invalidFilePath, testdata, true); + assertFalse(Files.exists(specificConfigSetBase.resolve(invalidFilePath))); + } + } + @Test public void testUploadAndDeleteConfig() throws IOException { + final var initialNumConfigs = fileSystemConfigSetService.listConfigs().size(); String configName = "testconfig"; fileSystemConfigSetService.uploadConfig(configName, configset("cloud-minimal")); - - assertEquals(fileSystemConfigSetService.listConfigs().size(), 1); + assertEquals(fileSystemConfigSetService.listConfigs().size(), initialNumConfigs + 1); assertTrue(fileSystemConfigSetService.checkConfigExists(configName)); byte[] testdata = "test data".getBytes(StandardCharsets.UTF_8); @@ -79,7 +111,7 @@ public void testUploadAndDeleteConfig() throws IOException { assertEquals("[schema.xml, solrconfig.xml]", allConfigFiles.toString()); fileSystemConfigSetService.copyConfig(configName, "copytestconfig"); - assertEquals(fileSystemConfigSetService.listConfigs().size(), 2); + assertEquals(fileSystemConfigSetService.listConfigs().size(), initialNumConfigs + 2); allConfigFiles = fileSystemConfigSetService.getAllConfigFiles("copytestconfig"); assertEquals("[schema.xml, solrconfig.xml]", allConfigFiles.toString()); diff --git a/solr/core/src/test/org/apache/solr/core/TestMinimalConfig.java b/solr/core/src/test/org/apache/solr/core/TestMinimalConfig.java index 1efa74d2d03..b04420664e8 100644 --- a/solr/core/src/test/org/apache/solr/core/TestMinimalConfig.java +++ b/solr/core/src/test/org/apache/solr/core/TestMinimalConfig.java @@ -29,8 +29,7 @@ public static void beforeClass() throws Exception { initCore("solrconfig-minimal.xml", "schema-minimal.xml"); } - // Make sure the content of the lib/ core subfolder is loaded even if there is no node in - // the solrconfig + // Make sure the content of the lib/ core subfolder is loaded @Test public void testLib() throws IOException { SolrResourceLoader loader = h.getCore().getResourceLoader(); diff --git a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java index 9ef6256d991..7e4eb1db75f 100644 --- a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java +++ b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java @@ -78,7 +78,7 @@ public static ByteBuffer getFileContent(String f) throws IOException { */ public static ByteBuffer getFileContent(String f, boolean loadFromClassPath) throws IOException { ByteBuffer jar; - File file = loadFromClassPath ? getFile(f) : new File(f); + File file = loadFromClassPath ? getFile(f).toFile() : new File(f); try (FileInputStream fis = new FileInputStream(file)) { byte[] buf = new byte[fis.available()]; // TODO: This should check that we read the entire stream @@ -732,9 +732,7 @@ public static LinkedHashMapWriter testForResponseElement( } Object actual = Utils.getObjectByPath(m, false, jsonPath); - if (expected instanceof ValidatingJsonMap.PredicateWithErrMsg) { - ValidatingJsonMap.PredicateWithErrMsg predicate = - (ValidatingJsonMap.PredicateWithErrMsg) expected; + if (expected instanceof ValidatingJsonMap.PredicateWithErrMsg predicate) { if (predicate.test(actual) == null) { success = true; break; @@ -982,8 +980,7 @@ public void testReqParams() throws Exception { new ValidatingJsonMap.PredicateWithErrMsg<>() { @Override public String test(Object o) { - if (o instanceof Map) { - Map m = (Map) o; + if (o instanceof Map m) { if ("part1_Value".equals(m.get("part1")) && "part2_Value".equals(m.get("part2"))) return null; } diff --git a/solr/core/src/test/org/apache/solr/handler/ReplicationTestHelper.java b/solr/core/src/test/org/apache/solr/handler/ReplicationTestHelper.java index 2e69ba4915a..2efe775a58d 100644 --- a/solr/core/src/test/org/apache/solr/handler/ReplicationTestHelper.java +++ b/solr/core/src/test/org/apache/solr/handler/ReplicationTestHelper.java @@ -26,6 +26,7 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Properties; import java.util.concurrent.TimeUnit; @@ -42,7 +43,6 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.embedded.JettyConfig; import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.util.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,9 +54,9 @@ public final class ReplicationTestHelper { "solr" + File.separator + "collection1" + File.separator + "conf" + File.separator; public static JettySolrRunner createAndStartJetty(SolrInstance instance) throws Exception { - FileUtils.copyFile( - new File(SolrTestCaseJ4.TEST_HOME(), "solr.xml"), - new File(instance.getHomeDir(), "solr.xml")); + Files.copy( + Path.of(SolrTestCaseJ4.TEST_HOME(), "solr.xml"), + Path.of(instance.getHomeDir(), "solr.xml")); Properties nodeProperties = new Properties(); nodeProperties.setProperty("solr.data.dir", instance.getDataDir()); JettyConfig jettyConfig = JettyConfig.builder().setPort(0).build(); @@ -96,9 +96,9 @@ public static int index(SolrClient s, Object... fields) throws Exception { * character copy of file using UTF-8. If port is non-null, will be substituted any time * "TEST_PORT" is found. */ - private static void copyFile(File src, File dst, Integer port, boolean internalCompression) + private static void copyFile(Path src, File dst, Integer port, boolean internalCompression) throws IOException { - try (BufferedReader in = Files.newBufferedReader(src.toPath(), StandardCharsets.UTF_8); + try (BufferedReader in = Files.newBufferedReader(src, StandardCharsets.UTF_8); Writer out = Files.newBufferedWriter(dst.toPath(), StandardCharsets.UTF_8)) { for (String line = in.readLine(); null != line; line = in.readLine()) { if (null != port) { diff --git a/solr/core/src/test/org/apache/solr/handler/RequestHandlerMetricsTest.java b/solr/core/src/test/org/apache/solr/handler/RequestHandlerMetricsTest.java index e3eecfc8cb3..b5b73de2597 100644 --- a/solr/core/src/test/org/apache/solr/handler/RequestHandlerMetricsTest.java +++ b/solr/core/src/test/org/apache/solr/handler/RequestHandlerMetricsTest.java @@ -90,11 +90,10 @@ public void testAggregateNodeLevelMetrics() throws SolrServerException, IOExcept final double[] minUpdateTime = {Double.MAX_VALUE}; final double[] maxUpdateTime = {-1.0}; Set> coreMetrics = new HashSet<>(); - metrics.forEachKey( - (key) -> { + metrics.forEach( + (key, coreMetric) -> { if (key.startsWith("solr.core.testRequestHandlerMetrics")) { - NamedList coreMetric = (NamedList) metrics.get(key); - coreMetrics.add(coreMetric); + coreMetrics.add((NamedList) coreMetric); } }); assertEquals(2, coreMetrics.size()); diff --git a/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java b/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java index bbff8694af3..2126747c19d 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java +++ b/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java @@ -83,7 +83,7 @@ void loadLocal(String... args) throws Exception { // TODO: stop using locally defined streams once stream.file and // stream.body work everywhere List cs = new ArrayList<>(1); - ContentStreamBase f = new ContentStreamBase.FileStream(new File(filename)); + ContentStreamBase f = new ContentStreamBase.FileStream(Path.of(filename)); f.setContentType("text/csv"); cs.add(f); req.setContentStreams(cs); diff --git a/solr/core/src/test/org/apache/solr/handler/TestHealthCheckHandlerLegacyMode.java b/solr/core/src/test/org/apache/solr/handler/TestHealthCheckHandlerLegacyMode.java index b376e52bd9b..fe25a08da3c 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestHealthCheckHandlerLegacyMode.java +++ b/solr/core/src/test/org/apache/solr/handler/TestHealthCheckHandlerLegacyMode.java @@ -74,8 +74,6 @@ public void setUp() throws Exception { buildUrl(followerJetty.getLocalPort()), DEFAULT_TEST_CORENAME); followerClientHealthCheck = ReplicationTestHelper.createNewSolrClient(buildUrl(followerJetty.getLocalPort())); - - System.setProperty("solr.indexfetcher.sotimeout2", "45000"); } public void clearIndexWithReplication() throws Exception { @@ -116,7 +114,6 @@ public void tearDown() throws Exception { followerClientHealthCheck.close(); followerClientHealthCheck = null; } - System.clearProperty("solr.indexfetcher.sotimeout"); } @Test diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java index 8f424e97803..a467010ca5e 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java +++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java @@ -72,12 +72,12 @@ import org.apache.solr.core.StandardDirectoryFactory; import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager; import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.handler.admin.api.ReplicationAPIBase; import org.apache.solr.security.AllowListUrlChecker; import org.apache.solr.util.TestInjection; import org.apache.solr.util.TimeOut; import org.junit.After; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -102,20 +102,12 @@ public class TestReplicationHandler extends SolrTestCaseJ4 { // index from previous test method static int nDocs = 500; - /* For testing backward compatibility, remove for 10.x */ - private static boolean useLegacyParams = false; - - @BeforeClass - public static void beforeClass() { - useLegacyParams = rarely(); - } - @Override @Before public void setUp() throws Exception { super.setUp(); systemSetPropertySolrDisableUrlAllowList("true"); - // System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory"); + System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory"); // For manual testing only // useFactory(null); // force an FS factory. leader = new SolrInstance(createTempDir("solr-instance").toFile(), "leader", null); @@ -133,8 +125,6 @@ public void setUp() throws Exception { followerClient = ReplicationTestHelper.createNewSolrClient( buildUrl(followerJetty.getLocalPort()), DEFAULT_TEST_CORENAME); - - System.setProperty("solr.indexfetcher.sotimeout2", "45000"); } public void clearIndexWithReplication() throws Exception { @@ -167,7 +157,6 @@ public void tearDown() throws Exception { followerClient.close(); followerClient = null; } - System.clearProperty("solr.indexfetcher.sotimeout"); } static JettySolrRunner createAndStartJetty(SolrInstance instance) throws Exception { @@ -315,8 +304,8 @@ public void doTestDetails() throws Exception { // check details on the follower a couple of times before & after fetching for (int i = 0; i < 3; i++) { NamedList details = getDetails(followerClient); - assertNotNull(i + ": " + details); - assertNotNull(i + ": " + details.toString(), details.get("follower")); + assertNotNull(i + ": " + details, details); + assertNotNull(i + ": " + details, details.get("follower")); if (i > 0) { rQuery(i, "*:*", followerClient); @@ -390,38 +379,6 @@ public void doTestDetails() throws Exception { } } - @Test - public void testLegacyConfiguration() throws Exception { - SolrInstance solrInstance = null; - JettySolrRunner instanceJetty = null; - SolrClient client = null; - try { - solrInstance = - new SolrInstance( - createTempDir("solr-instance").toFile(), - "replication-legacy", - leaderJetty.getLocalPort()); - solrInstance.setUp(); - instanceJetty = createAndStartJetty(solrInstance); - client = - ReplicationTestHelper.createNewSolrClient( - buildUrl(instanceJetty.getLocalPort()), DEFAULT_TEST_CORENAME); - - NamedList details = getDetails(client); - - assertEquals("repeater isLeader?", "true", details.get("isLeader")); - assertEquals("repeater isFollower?", "true", details.get("isFollower")); - assertNotNull("repeater has leader section", details.get("leader")); - assertNotNull("repeater has follower section", details.get("follower")); - - } finally { - if (instanceJetty != null) { - instanceJetty.stop(); - } - if (client != null) client.close(); - } - } - /** * Verify that empty commits and/or commits with openSearcher=false on the leader do not cause * subsequent replication problems on the follower @@ -502,7 +459,7 @@ public void doTestReplicateAfterWrite2Follower() throws Exception { index(followerClient, "id", 555, "name", "name = " + 555); followerClient.commit(true, true); - // this doc is added to follower so it should show an item w/ that result + // this doc is added to follower, so it should show an item w/ that result assertEquals(1, numFound(rQuery(1, "id:555", followerClient))); // Let's fetch the index rather than rely on the polling. @@ -571,7 +528,7 @@ public void doTestIndexAndConfigReplication() throws Exception { followerJetty.stop(); - // setup an sub directory /foo/ in order to force subdir file replication + // set up a subdirectory /foo/ in order to force subdir file replication File leaderFooDir = new File(leader.getConfDir() + File.separator + "foo"); File leaderBarFile = new File(leaderFooDir, "bar.txt"); assertTrue("could not make dir " + leaderFooDir, leaderFooDir.mkdirs()); @@ -594,7 +551,7 @@ public void doTestIndexAndConfigReplication() throws Exception { followerQueryRsp = rQuery(1, "*:*", followerClient); assertVersions(leaderClient, followerClient); SolrDocument d = ((SolrDocumentList) followerQueryRsp.get("response")).get(0); - assertEquals("newname = 2000", (String) d.getFieldValue("newname")); + assertEquals("newname = 2000", d.getFieldValue("newname")); assertTrue(followerFooDir.isDirectory()); assertTrue(followerBarFile.exists()); @@ -639,8 +596,8 @@ public void doTestStopPoll() throws Exception { // get docs from leader and check if number is equal to leader assertEquals(nDocs + 1, numFound(rQuery(nDocs + 1, "*:*", leaderClient))); - // NOTE: this test is wierd, we want to verify it DOESNT replicate... - // for now, add a sleep for this.., but the logic is wierd. + // NOTE: this test is weird, we want to verify it DOESN'T replicate... + // for now, add a sleep for this... but the logic is weird. Thread.sleep(3000); // get docs from follower and check if number is not equal to leader; polling is disabled @@ -781,11 +738,8 @@ private NamedList getFollowerDetails() throws SolrServerException, IOExc ModifiableSolrParams params = new ModifiableSolrParams(); params.set(CommonParams.QT, "/replication"); params.set("command", "details"); - if (useLegacyParams) { - params.set("slave", "true"); - } else { - params.set("follower", "true"); - } + params.set("follower", "true"); + QueryResponse response = followerClient.query(params); // details/follower/timesIndexReplicated @@ -827,9 +781,6 @@ public void doTestIndexFetchWithLeaderUrl() throws Exception { assertEquals(nDocs, leaderQueryResult.getNumFound()); String urlKey = "leaderUrl"; - if (useLegacyParams) { - urlKey = "masterUrl"; - } // index fetch String leaderUrl = @@ -1518,9 +1469,9 @@ public void doTestIllegalFilePaths() { Arrays.asList(absFile, "../dir/traversal", "illegal\rfile\nname\t"); List params = Arrays.asList( - ReplicationHandler.FILE, - ReplicationHandler.CONF_FILE_SHORT, - ReplicationHandler.TLOG_FILE); + ReplicationAPIBase.FILE, + ReplicationAPIBase.CONF_FILE_SHORT, + ReplicationAPIBase.TLOG_FILE); for (String param : params) { for (String filename : illegalFilenames) { expectThrows( @@ -1632,7 +1583,7 @@ public void testEmptyBackups() throws Exception { index(leaderClient, "id", "1", "name", "foo"); - { // second backup w/uncommited doc + { // second backup w/uncommitted doc final String backupName = "empty_backup2"; final GenericSolrRequest req = new GenericSolrRequest( @@ -1669,48 +1620,6 @@ public void testEmptyBackups() throws Exception { } } - public void testGetBoolWithBackwardCompatibility() { - assertTrue(ReplicationHandler.getBoolWithBackwardCompatibility(params(), "foo", "bar", true)); - assertFalse(ReplicationHandler.getBoolWithBackwardCompatibility(params(), "foo", "bar", false)); - assertTrue( - ReplicationHandler.getBoolWithBackwardCompatibility( - params("foo", "true"), "foo", "bar", false)); - assertTrue( - ReplicationHandler.getBoolWithBackwardCompatibility( - params("bar", "true"), "foo", "bar", false)); - assertTrue( - ReplicationHandler.getBoolWithBackwardCompatibility( - params("foo", "true", "bar", "false"), "foo", "bar", false)); - } - - public void testGetObjectWithBackwardCompatibility() { - assertEquals( - "aaa", - ReplicationHandler.getObjectWithBackwardCompatibility(params(), "foo", "bar", "aaa")); - assertEquals( - "bbb", - ReplicationHandler.getObjectWithBackwardCompatibility( - params("foo", "bbb"), "foo", "bar", "aaa")); - assertEquals( - "bbb", - ReplicationHandler.getObjectWithBackwardCompatibility( - params("bar", "bbb"), "foo", "bar", "aaa")); - assertEquals( - "bbb", - ReplicationHandler.getObjectWithBackwardCompatibility( - params("foo", "bbb", "bar", "aaa"), "foo", "bar", "aaa")); - assertNull(ReplicationHandler.getObjectWithBackwardCompatibility(params(), "foo", "bar", null)); - } - - public void testGetObjectWithBackwardCompatibilityFromNL() { - NamedList nl = new NamedList<>(); - assertNull(ReplicationHandler.getObjectWithBackwardCompatibility(nl, "foo", "bar")); - nl.add("bar", "bbb"); - assertEquals("bbb", ReplicationHandler.getObjectWithBackwardCompatibility(nl, "foo", "bar")); - nl.add("foo", "aaa"); - assertEquals("aaa", ReplicationHandler.getObjectWithBackwardCompatibility(nl, "foo", "bar")); - } - private static class AddExtraDocs implements Runnable { SolrClient leaderClient; @@ -1786,7 +1695,7 @@ private Date watchCoreStartAt(JettySolrRunner jettySolrRunner, final Date min) return startTime; } } catch (SolrException e) { - // workarround for SOLR-4668 + // workaround for SOLR-4668 if (500 != e.code()) { throw e; } // else server possibly from the core reload in progress... @@ -1796,7 +1705,140 @@ private Date watchCoreStartAt(JettySolrRunner jettySolrRunner, final Date min) Thread.sleep(sleepInterval); } fail("timed out waiting for collection1 startAt time to exceed: " + min); - return min; // compilation neccessity + return min; // compilation necessity + } + } + + @Test + public void doTestIndexFollowerAfterRestartWhenReplicationIsDisabled() throws Exception { + // failed before changes to IndexFetcher + testReplicationRestartFollower("disablereplication"); + } + + @Test + public void doTestIndexFollowerAfterRestartWhenReplicationIsEnabled() throws Exception { + testReplicationRestartFollower("enablereplication"); + } + + private void testReplicationRestartFollower(String replicationCmd) throws Exception { + useFactory(null); + try { + clearIndexWithReplication(); + // change solrconfig having 'replicateAfter startup' option on leader + leader.copyConfigFile(CONF_DIR + "solrconfig-leader2.xml", "solrconfig.xml"); + + leaderJetty.stop(); + final TimeOut waitForLeaderToShutdown = + new TimeOut(300, TimeUnit.SECONDS, TimeSource.NANO_TIME); + waitForLeaderToShutdown.waitFor( + "Gave up after waiting an obscene amount of time for leader to shut down", + () -> leaderJetty.isStopped()); + + leaderJetty.start(); + final TimeOut waitForLeaderToStart = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME); + waitForLeaderToStart.waitFor( + "Gave up after waiting an obscene amount of time for leader to start", + () -> leaderJetty.isRunning()); + + // close and re-create leader client because its connection pool has stale connections + leaderClient.close(); + leaderClient = + createNewSolrClient(buildUrl(leaderJetty.getLocalPort()), DEFAULT_TEST_CORENAME); + + NamedList leaderQueryRsp = rQuery(0, "*:*", leaderClient); + SolrDocumentList leaderQueryResult = (SolrDocumentList) leaderQueryRsp.get("response"); + assertEquals(0, numFound(leaderQueryRsp)); + + // get docs from follower and check if number is equal to leader + NamedList followerQueryRsp = rQuery(0, "*:*", followerClient); + SolrDocumentList followerQueryResult = (SolrDocumentList) followerQueryRsp.get("response"); + assertEquals(0, numFound(followerQueryRsp)); + + // compare results + String cmp = + BaseDistributedSearchTestCase.compare(leaderQueryResult, followerQueryResult, 0, null); + assertNull(cmp); + + nDocs--; + for (int i = 0; i < nDocs; i++) { + index(leaderClient, "id", i, "name", "name = " + i); + } + + leaderClient.commit(); + + leaderQueryRsp = rQuery(nDocs, "*:*", leaderClient); + leaderQueryResult = (SolrDocumentList) leaderQueryRsp.get("response"); + assertEquals(nDocs, numFound(leaderQueryRsp)); + + // get docs from follower and check if number is equal to leader + followerQueryRsp = rQuery(nDocs, "*:*", followerClient); + followerQueryResult = (SolrDocumentList) followerQueryRsp.get("response"); + assertEquals(nDocs, numFound(followerQueryRsp)); + + // compare results + cmp = BaseDistributedSearchTestCase.compare(leaderQueryResult, followerQueryResult, 0, null); + assertNull(cmp); + + String timesReplicatedString = getFollowerDetails("timesIndexReplicated"); + String timesFailed; + Integer previousTimesFailed = null; + if (timesReplicatedString == null) { + timesFailed = "0"; + } else { + int timesReplicated = Integer.parseInt(timesReplicatedString); + timesFailed = getFollowerDetails("timesFailed"); + if (null == timesFailed) { + timesFailed = "0"; + } + + previousTimesFailed = Integer.parseInt(timesFailed); + // Sometimes replication will fail because leader's core is still loading; make sure there + // was one success + assertEquals(1, timesReplicated - previousTimesFailed); + } + + followerJetty.stop(); + + invokeReplicationCommand( + buildUrl(leaderJetty.getLocalPort()) + "/" + DEFAULT_TEST_CORENAME, replicationCmd); + + final TimeOut waitForFollowerToShutdown = + new TimeOut(300, TimeUnit.SECONDS, TimeSource.NANO_TIME); + waitForFollowerToShutdown.waitFor( + "Gave up after waiting an obscene amount of time for leader to shut down", + () -> followerJetty.isStopped()); + + log.info("FOLLOWER START ********************************************"); + followerJetty.start(); + + final TimeOut waitForFollowerToStart = + new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME); + waitForFollowerToStart.waitFor( + "Gave up after waiting an obscene amount of time for leader to start", + () -> followerJetty.isRunning()); + + // poll interval on follower is 1 second, so we just sleep for a few seconds + Thread.sleep(3000); + followerClient.close(); + followerClient = + createNewSolrClient(buildUrl(followerJetty.getLocalPort()), DEFAULT_TEST_CORENAME); + NamedList details = getDetails(followerClient); + + leaderQueryRsp = rQuery(nDocs, "*:*", leaderClient); + leaderQueryResult = (SolrDocumentList) leaderQueryRsp.get("response"); + assertEquals(nDocs, numFound(leaderQueryRsp)); + + // get docs from follower and check if number is equal to leader + followerQueryRsp = rQuery(nDocs, "*:*", followerClient); + followerQueryResult = (SolrDocumentList) followerQueryRsp.get("response"); + assertEquals(nDocs, numFound(followerQueryRsp)); + + // compare results again + cmp = BaseDistributedSearchTestCase.compare(leaderQueryResult, followerQueryResult, 0, null); + assertNull(cmp); + + } finally { + resetFactory(); } } diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java index 418ce1d6551..83990058abc 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java +++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java @@ -46,7 +46,6 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.embedded.JettyConfig; import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.util.FileUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -72,9 +71,9 @@ public class TestReplicationHandlerBackup extends SolrJettyTestBase { private static JettySolrRunner createAndStartJetty(ReplicationTestHelper.SolrInstance instance) throws Exception { - FileUtils.copyFile( - new File(SolrTestCaseJ4.TEST_HOME(), "solr.xml"), - new File(instance.getHomeDir(), "solr.xml")); + Files.copy( + Path.of(SolrTestCaseJ4.TEST_HOME(), "solr.xml"), + Path.of(instance.getHomeDir(), "solr.xml")); Properties nodeProperties = new Properties(); nodeProperties.setProperty("solr.data.dir", instance.getDataDir()); JettyConfig jettyConfig = JettyConfig.builder().setPort(0).build(); @@ -238,7 +237,7 @@ public void doTestBackup() throws Exception { } } - private void testDeleteNamedBackup(String backupNames[]) throws Exception { + private void testDeleteNamedBackup(String[] backupNames) throws Exception { final BackupStatusChecker backupStatus = new BackupStatusChecker(leaderClient, "/" + DEFAULT_TEST_CORENAME + "/replication"); for (int i = 0; i < 2; i++) { diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerDiskOverFlow.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerDiskOverFlow.java index c66a1dd7201..7bd54f68c77 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerDiskOverFlow.java +++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerDiskOverFlow.java @@ -92,8 +92,6 @@ public void setUp() throws Exception { followerClient = ReplicationTestHelper.createNewSolrClient( TestReplicationHandler.buildUrl(followerJetty.getLocalPort()), DEFAULT_TEST_CORENAME); - - System.setProperty("solr.indexfetcher.sotimeout2", "45000"); } @Override @@ -118,7 +116,6 @@ public void tearDown() throws Exception { followerClient = null; } System.clearProperty(TEST_URL_ALLOW_LIST); - System.clearProperty("solr.indexfetcher.sotimeout"); IndexFetcher.usableDiskSpaceProvider = originalDiskSpaceprovider; IndexFetcher.testWait = originalTestWait; @@ -214,6 +211,7 @@ public boolean getAsBoolean() { .add("qt", "/replication") .add("command", CMD_FETCH_INDEX) .add("wait", "true")); + assertEquals("Replication command status", "OK", response._getStr("status", null)); assertEquals( diff --git a/solr/core/src/test/org/apache/solr/handler/TestRestoreCore.java b/solr/core/src/test/org/apache/solr/handler/TestRestoreCore.java index 593f0af7bd9..a4de0796d69 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestRestoreCore.java +++ b/solr/core/src/test/org/apache/solr/handler/TestRestoreCore.java @@ -36,7 +36,6 @@ import org.apache.solr.common.SolrInputDocument; import org.apache.solr.embedded.JettyConfig; import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.util.FileUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -57,9 +56,9 @@ public class TestRestoreCore extends SolrJettyTestBase { private static JettySolrRunner createAndStartJetty(ReplicationTestHelper.SolrInstance instance) throws Exception { - FileUtils.copyFile( - new File(SolrTestCaseJ4.TEST_HOME(), "solr.xml"), - new File(instance.getHomeDir(), "solr.xml")); + Files.copy( + Path.of(SolrTestCaseJ4.TEST_HOME(), "solr.xml"), + Path.of(instance.getHomeDir(), "solr.xml")); Properties nodeProperties = new Properties(); nodeProperties.setProperty("solr.data.dir", instance.getDataDir()); JettyConfig jettyConfig = JettyConfig.builder().setPort(0).build(); diff --git a/solr/core/src/test/org/apache/solr/handler/TestSampleDocumentsLoader.java b/solr/core/src/test/org/apache/solr/handler/TestSampleDocumentsLoader.java index 6ecb27c7f02..7bd82ec60b1 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestSampleDocumentsLoader.java +++ b/solr/core/src/test/org/apache/solr/handler/TestSampleDocumentsLoader.java @@ -109,12 +109,12 @@ public static String guessContentTypeFromFilename(String name) { return "application/octet-stream"; } - protected ContentStream getContentStream(File file) { + protected ContentStream getContentStream(File file) throws IOException { return getContentStream(file, guessContentTypeFromFilename(file.getName())); } - protected ContentStream getContentStream(File file, String contentType) { - ContentStreamBase.FileStream stream = new ContentStreamBase.FileStream(file); + protected ContentStream getContentStream(File file, String contentType) throws IOException { + ContentStreamBase.FileStream stream = new ContentStreamBase.FileStream(file.toPath()); stream.setContentType(contentType); return stream; } diff --git a/solr/core/src/test/org/apache/solr/handler/V2ApiIntegrationTest.java b/solr/core/src/test/org/apache/solr/handler/V2ApiIntegrationTest.java index 651d4a8ed93..a7f4606e160 100644 --- a/solr/core/src/test/org/apache/solr/handler/V2ApiIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/handler/V2ApiIntegrationTest.java @@ -155,18 +155,17 @@ public void testSetPropertyValidationOfCluster() throws IOException, SolrServerE cluster .getSolrClient() .request( - new V2Request.Builder("/cluster") - .withMethod(SolrRequest.METHOD.POST) - .withPayload("{set-property: {name: maxCoresPerNode, val:42}}") + new V2Request.Builder("/cluster/properties/maxCoresPerNode") + .withMethod(SolrRequest.METHOD.PUT) + .withPayload("{\"value\": \"42\"}") .build()); assertTrue(resp.toString().contains("status=0")); resp = cluster .getSolrClient() .request( - new V2Request.Builder("/cluster") - .withMethod(SolrRequest.METHOD.POST) - .withPayload("{set-property: {name: maxCoresPerNode, val:null}}") + new V2Request.Builder("/cluster/properties/maxCoresPerNode") + .withMethod(SolrRequest.METHOD.DELETE) .build()); assertTrue(resp.toString().contains("status=0")); } diff --git a/solr/core/src/test/org/apache/solr/handler/V2ClusterAPIMappingTest.java b/solr/core/src/test/org/apache/solr/handler/V2ClusterAPIMappingTest.java index 94f091fba2a..6410dc60a47 100644 --- a/solr/core/src/test/org/apache/solr/handler/V2ClusterAPIMappingTest.java +++ b/solr/core/src/test/org/apache/solr/handler/V2ClusterAPIMappingTest.java @@ -19,7 +19,6 @@ import static org.apache.solr.cloud.api.collections.CollectionHandlingUtils.REQUESTID; import static org.apache.solr.common.params.CommonParams.ACTION; -import static org.apache.solr.common.params.CommonParams.NAME; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -128,29 +127,11 @@ public void testRemoveRoleAllParams() throws Exception { assertEquals("some_role", v1Params.get("role")); } - @Test - public void testSetPropertyAllParams() throws Exception { - final SolrParams v1Params = - captureConvertedV1Params( - "/cluster", - "POST", - "{'set-property': {" + "'name': 'some_prop_name', " + "'val':'some_value'}}"); - - assertEquals(CollectionParams.CollectionAction.CLUSTERPROP.toString(), v1Params.get(ACTION)); - assertEquals("some_prop_name", v1Params.get(NAME)); - assertEquals("some_value", v1Params.get("val")); - } - private SolrParams captureConvertedV1Params(String path, String method, String v2RequestBody) throws Exception { return doCaptureParams(path, method, v2RequestBody, mockCollectionsHandler); } - private SolrParams captureConvertedConfigsetV1Params( - String path, String method, String v2RequestBody) throws Exception { - return doCaptureParams(path, method, v2RequestBody, mockConfigSetHandler); - } - private SolrParams doCaptureParams( String path, String method, String v2RequestBody, RequestHandlerBase mockHandler) throws Exception { diff --git a/solr/core/src/test/org/apache/solr/handler/XmlUpdateRequestHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/XmlUpdateRequestHandlerTest.java index 0394018e0bf..8dcb465094a 100644 --- a/solr/core/src/test/org/apache/solr/handler/XmlUpdateRequestHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/XmlUpdateRequestHandlerTest.java @@ -105,7 +105,7 @@ public void testRequestParams() throws Exception { @Test public void testExternalEntities() throws Exception { - String file = getFile("mailing_lists.pdf").toURI().toASCIIString(); + String file = getFile("mailing_lists.pdf").toUri().toASCIIString(); String xml = "" + diff --git a/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerActionTest.java b/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerActionTest.java index 235db7af21e..6b5daa5642d 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerActionTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/CoreAdminHandlerActionTest.java @@ -17,7 +17,6 @@ package org.apache.solr.handler.admin; import java.util.Locale; -import java.util.Map; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CoreAdminParams; @@ -82,11 +81,11 @@ private void testAction(String action, String propertyName, String propertyValue admin.handleRequestBody(req(CoreAdminParams.ACTION, action), response); - Map actionResponse = ((NamedList) response.getResponse()).asMap(); + var actionResponse = (NamedList) response.getResponse(); assertTrue( String.format(Locale.ROOT, "Action response should contain %s property", propertyName), - actionResponse.containsKey(propertyName)); + actionResponse.get(propertyName) != null); assertEquals( String.format( Locale.ROOT, diff --git a/solr/core/src/test/org/apache/solr/handler/admin/IndexSizeEstimatorTest.java b/solr/core/src/test/org/apache/solr/handler/admin/IndexSizeEstimatorTest.java index e175497aa36..5523ad23c15 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/IndexSizeEstimatorTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/IndexSizeEstimatorTest.java @@ -227,8 +227,6 @@ public void testIntegration() throws Exception { (k, v) -> { double size = fromHumanReadableUnits((String) v); double sampledSize = fromHumanReadableUnits((String) sampledFieldsBySize.get(k)); - assertNotNull( - "sampled size missing for field " + k + " in " + sampledFieldsBySize, sampledSize); double delta = size * 0.5; assertEquals("sampled size of " + k + " is wildly off", size, sampledSize, delta); }); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MBeansHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MBeansHandlerTest.java index c4fa6c35e2e..05232b7f05e 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/MBeansHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/MBeansHandlerTest.java @@ -130,7 +130,7 @@ public void testAddedMBeanDiff() throws Exception { @Test public void testXMLDiffWithExternalEntity() { - String file = getFile("mailing_lists.pdf").toURI().toASCIIString(); + String file = getFile("mailing_lists.pdf").toUri().toASCIIString(); String xml = "\n" + " { - return new ObjectMapper() - .readValue(is, ZookeeperReadAPI.ListZkChildrenResponse.class); + return new ObjectMapper().readValue(is, ZooKeeperListChildrenResponse.class); }); // At the top level, the response contains a key with the value of the specified zkPath @@ -145,7 +146,7 @@ public void testCanListChildNodes() throws Exception { // node. // The actual stat values vary a good bit so aren't very useful to assert on, so let's just // make sure all of the expected child nodes were found. - final Map childStatsByPath = + final Map childStatsByPath = response.unknownProperties().get("/configs/_default"); assertEquals(6, childStatsByPath.size()); assertThat( diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/ClusterPropsAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/ClusterPropsAPITest.java new file mode 100644 index 00000000000..f690dbc1498 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/ClusterPropsAPITest.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler.admin.api; + +import static org.apache.solr.common.util.Utils.getObjectByPath; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.not; + +import java.net.URL; +import java.util.List; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.entity.StringEntity; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.util.Utils; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class ClusterPropsAPITest extends SolrCloudTestCase { + + private URL baseUrl; + private String baseUrlV2ClusterProps; + + private static final String testClusterProperty = "ext.test"; + private static final String testClusterPropertyValue = "test value"; + private static final String testClusterPropertyNestedKeyAndValue = + " \"defaults\": {" + + " \"collection\": {" + + " \"numShards\": 4," + + " \"nrtReplicas\": 2," + + " \"tlogReplicas\": 2," + + " \"pullReplicas\": 2" + + " }" + + " }"; + private static final String testClusterPropertyBulkAndNestedValues = + "{" + + testClusterPropertyNestedKeyAndValue + + "," + + " \"" + + testClusterProperty + + "\": " + + "\"" + + testClusterPropertyValue + + "\"" + + " }"; + + @BeforeClass + public static void setupCluster() throws Exception { + configureCluster(1).addConfig("conf", configset("cloud-minimal")).configure(); + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + + baseUrl = cluster.getJettySolrRunner(0).getBaseUrl(); + baseUrlV2ClusterProps = + cluster.getJettySolrRunner(0).getBaseURLV2().toString() + "/cluster/properties"; + } + + @After + @Override + public void tearDown() throws Exception { + super.tearDown(); + } + + @Test + public void testClusterPropertyOpsAllGood() throws Exception { + try (HttpSolrClient client = new HttpSolrClient.Builder(baseUrl.toString()).build()) { + // List Properties, confirm the test property does not exist + // This ignores eventually existing other properties + Object o = + Utils.executeGET(client.getHttpClient(), baseUrlV2ClusterProps, Utils.JSONCONSUMER); + assertNotNull(o); + @SuppressWarnings("unchecked") + List initProperties = (List) getObjectByPath(o, true, "clusterProperties"); + assertThat(initProperties, not(hasItem(testClusterProperty))); + + // Create a single cluster property + String path = baseUrlV2ClusterProps + "/" + testClusterProperty; + HttpPut httpPut = new HttpPut(path); + httpPut.setHeader("Content-Type", "application/json"); + httpPut.setEntity(new StringEntity("{\"value\":\"" + testClusterPropertyValue + "\"}")); + o = Utils.executeHttpMethod(client.getHttpClient(), path, Utils.JSONCONSUMER, httpPut); + assertNotNull(o); + + // List Properties, this time there should be the just added property + o = Utils.executeGET(client.getHttpClient(), baseUrlV2ClusterProps, Utils.JSONCONSUMER); + assertNotNull(o); + @SuppressWarnings("unchecked") + List updatedProperties = (List) getObjectByPath(o, true, "clusterProperties"); + assertThat(updatedProperties, hasItem(testClusterProperty)); + + // Fetch Cluster Property + // Same path as used in the Create step above + o = Utils.executeGET(client.getHttpClient(), path, Utils.JSONCONSUMER); + assertNotNull(o); + assertEquals(testClusterProperty, (String) getObjectByPath(o, true, "clusterProperty/name")); + assertEquals( + testClusterPropertyValue, (String) getObjectByPath(o, true, "clusterProperty/value")); + + // Delete Cluster Property + // Same path as used in the Create step above + HttpDelete httpDelete = new HttpDelete(path); + o = Utils.executeHttpMethod(client.getHttpClient(), path, Utils.JSONCONSUMER, httpDelete); + assertNotNull(o); + + // List Properties, the test property should be gone + o = Utils.executeGET(client.getHttpClient(), baseUrlV2ClusterProps, Utils.JSONCONSUMER); + assertNotNull(o); + @SuppressWarnings("unchecked") + List clearedProperties = (List) getObjectByPath(o, true, "clusterProperties"); + assertThat(clearedProperties, not(hasItem(testClusterProperty))); + } + } + + @Test + public void testClusterPropertyNestedBulkSet() throws Exception { + try (HttpSolrClient client = new HttpSolrClient.Builder(baseUrl.toString()).build()) { + // Create a single cluster property using the Bulk/Nested set ClusterProp API + HttpPut httpPut = new HttpPut(baseUrlV2ClusterProps); + httpPut.setHeader("Content-Type", "application/json"); + httpPut.setEntity(new StringEntity(testClusterPropertyBulkAndNestedValues)); + Object o = + Utils.executeHttpMethod( + client.getHttpClient(), baseUrlV2ClusterProps, Utils.JSONCONSUMER, httpPut); + assertNotNull(o); + + // Fetch Cluster Property checking the not-nested property set above + String path = baseUrlV2ClusterProps + "/" + testClusterProperty; + o = Utils.executeGET(client.getHttpClient(), path, Utils.JSONCONSUMER); + assertNotNull(o); + assertEquals(testClusterProperty, (String) getObjectByPath(o, true, "clusterProperty/name")); + assertEquals( + testClusterPropertyValue, (String) getObjectByPath(o, true, "clusterProperty/value")); + + // Fetch Cluster Property checking the nested property set above + path = baseUrlV2ClusterProps + "/" + "defaults"; + o = Utils.executeGET(client.getHttpClient(), path, Utils.JSONCONSUMER); + assertNotNull(o); + assertEquals("defaults", (String) getObjectByPath(o, true, "clusterProperty/name")); + assertEquals(4L, getObjectByPath(o, true, "clusterProperty/value/collection/numShards")); + + // Clean up to leave the state unchanged + HttpDelete httpDelete = new HttpDelete(path); + Utils.executeHttpMethod(client.getHttpClient(), path, Utils.JSONCONSUMER, httpDelete); + path = baseUrlV2ClusterProps + "/" + testClusterProperty; + httpDelete = new HttpDelete(path); + Utils.executeHttpMethod(client.getHttpClient(), path, Utils.JSONCONSUMER, httpDelete); + } + } + + @Test + public void testClusterPropertyFetchNonExistentProperty() throws Exception { + try (HttpSolrClient client = new HttpSolrClient.Builder(baseUrl.toString()).build()) { + // Fetch Cluster Property that doesn't exist + String path = baseUrlV2ClusterProps + "/ext.clusterPropThatDoesNotExist"; + HttpGet fetchClusterPropertyGet = new HttpGet(path); + HttpResponse httpResponse = client.getHttpClient().execute(fetchClusterPropertyGet); + assertEquals(404, httpResponse.getStatusLine().getStatusCode()); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/CoreReplicationAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/CoreReplicationAPITest.java index e6fe9edad0b..2e3a321bc5e 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/CoreReplicationAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/CoreReplicationAPITest.java @@ -21,26 +21,32 @@ import static org.mockito.Mockito.when; import io.opentelemetry.api.trace.Span; +import java.io.IOException; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.api.model.FileListResponse; +import org.apache.solr.client.api.model.FileMetaData; +import org.apache.solr.client.api.model.IndexVersionResponse; import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.handler.ReplicationHandler; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.update.UpdateHandler; +import org.apache.solr.update.UpdateLog; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -/** Unit tests for {@link CoreReplicationAPI} */ +/** Unit tests for {@link CoreReplication} */ public class CoreReplicationAPITest extends SolrTestCaseJ4 { - private CoreReplicationAPI coreReplicationAPI; + private CoreReplication coreReplicationAPI; private SolrCore mockCore; private ReplicationHandler mockReplicationHandler; - private SolrQueryRequest mockQueryRequest; - private SolrQueryResponse queryResponse; @BeforeClass public static void ensureWorkingMockito() { @@ -52,19 +58,18 @@ public static void ensureWorkingMockito() { public void setUp() throws Exception { super.setUp(); setUpMocks(); - mockQueryRequest = mock(SolrQueryRequest.class); + final var mockQueryRequest = mock(SolrQueryRequest.class); when(mockQueryRequest.getSpan()).thenReturn(Span.getInvalid()); - queryResponse = new SolrQueryResponse(); + final var queryResponse = new SolrQueryResponse(); coreReplicationAPI = new CoreReplicationAPIMock(mockCore, mockQueryRequest, queryResponse); } @Test public void testGetIndexVersion() throws Exception { - CoreReplicationAPI.IndexVersionResponse expected = - new CoreReplicationAPI.IndexVersionResponse(123L, 123L, "testGeneration"); + IndexVersionResponse expected = new IndexVersionResponse(123L, 123L, "testGeneration"); when(mockReplicationHandler.getIndexVersionResponse()).thenReturn(expected); - CoreReplicationAPI.IndexVersionResponse actual = coreReplicationAPI.doFetchIndexVersion(); + IndexVersionResponse actual = coreReplicationAPI.doFetchIndexVersion(); assertEquals(expected.indexVersion, actual.indexVersion); assertEquals(expected.generation, actual.generation); assertEquals(expected.status, actual.status); @@ -73,26 +78,53 @@ public void testGetIndexVersion() throws Exception { @Test @SuppressWarnings("unchecked") public void testFetchFiles() throws Exception { - CoreReplicationAPI.FileListResponse actualResponse = coreReplicationAPI.fetchFileList(-1); + FileListResponse actualResponse = coreReplicationAPI.fetchFileList(-1); assertEquals(123, actualResponse.fileList.get(0).size); assertEquals("test", actualResponse.fileList.get(0).name); assertEquals(123456789, actualResponse.fileList.get(0).checksum); } - private void setUpMocks() { + @Test + public void testFetchFile() throws Exception { + ReplicationAPIBase.DirectoryFileStream actual = + coreReplicationAPI.doFetchFile("./test", "file", null, null, false, false, 0, null); + assertNotNull(actual); + + actual = + coreReplicationAPI.doFetchFile("./test", "tlogFile", null, null, false, false, 0, null); + assertTrue(actual instanceof ReplicationAPIBase.LocalFsTlogFileStream); + + actual = coreReplicationAPI.doFetchFile("./test", "cf", null, null, false, false, 0, null); + assertTrue(actual instanceof ReplicationAPIBase.LocalFsConfFileStream); + } + + private void setUpMocks() throws IOException { mockCore = mock(SolrCore.class); mockReplicationHandler = mock(ReplicationHandler.class); + + // Mocks for LocalFsTlogFileStream + UpdateHandler mockUpdateHandler = mock(UpdateHandler.class); + UpdateLog mockUpdateLog = mock(UpdateLog.class); + when(mockUpdateHandler.getUpdateLog()).thenReturn(mockUpdateLog); + when(mockUpdateLog.getTlogDir()).thenReturn("ignore"); + + // Mocks for LocalFsConfFileStream + SolrResourceLoader mockSolrResourceLoader = mock(SolrResourceLoader.class); + Path mockPath = mock(Path.class); when(mockCore.getRequestHandler(ReplicationHandler.PATH)).thenReturn(mockReplicationHandler); + when(mockCore.getUpdateHandler()).thenReturn(mockUpdateHandler); + when(mockCore.getResourceLoader()).thenReturn(mockSolrResourceLoader); + when(mockSolrResourceLoader.getConfigPath()).thenReturn(mockPath); } - private static class CoreReplicationAPIMock extends CoreReplicationAPI { + private static class CoreReplicationAPIMock extends CoreReplication { public CoreReplicationAPIMock(SolrCore solrCore, SolrQueryRequest req, SolrQueryResponse rsp) { super(solrCore, req, rsp); } @Override protected FileListResponse getFileList(long generation, ReplicationHandler replicationHandler) { - final FileListResponse filesResponse = new FileListResponse(); + final var filesResponse = new FileListResponse(); List fileMetaData = Arrays.asList(new FileMetaData(123, "test", 123456789)); filesResponse.fileList = new ArrayList<>(fileMetaData); return filesResponse; diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateAliasAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateAliasAPITest.java index e7d1ff89001..b4f5e2d2760 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/CreateAliasAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/CreateAliasAPITest.java @@ -24,12 +24,16 @@ import java.util.List; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.api.model.CategoryRoutedAliasProperties; +import org.apache.solr.client.api.model.CreateAliasRequestBody; import org.apache.solr.client.api.model.CreateCollectionRequestBody; +import org.apache.solr.client.api.model.RoutedAliasProperties; +import org.apache.solr.client.api.model.TimeRoutedAliasProperties; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.ModifiableSolrParams; import org.junit.Test; -/** Unit tests for {@link CreateAliasAPI} */ +/** Unit tests for {@link CreateAlias} */ public class CreateAliasAPITest extends SolrTestCaseJ4 { @Test @@ -38,7 +42,7 @@ public void testReportsErrorIfRequestBodyMissing() { expectThrows( SolrException.class, () -> { - final var api = new CreateAliasAPI(null, null, null); + final var api = new CreateAlias(null, null, null); api.createAlias(null); }); @@ -48,11 +52,12 @@ public void testReportsErrorIfRequestBodyMissing() { @Test public void testReportsErrorIfAliasNameInvalid() { - final var requestBody = new CreateAliasAPI.CreateAliasRequestBody(); + final var requestBody = new CreateAliasRequestBody(); requestBody.name = "some@invalid$alias"; requestBody.collections = List.of("validColl1", "validColl2"); - final var thrown = expectThrows(SolrException.class, () -> requestBody.validate()); + final var thrown = + expectThrows(SolrException.class, () -> CreateAlias.validateRequestBody(requestBody)); assertThat(thrown.getMessage(), containsString("Invalid alias")); assertThat(thrown.getMessage(), containsString("some@invalid$alias")); assertThat( @@ -64,24 +69,26 @@ public void testReportsErrorIfAliasNameInvalid() { // Aliases can be normal or "routed', but not both. @Test public void testReportsErrorIfExplicitCollectionsAndRoutingParamsBothProvided() { - final var requestBody = new CreateAliasAPI.CreateAliasRequestBody(); + final var requestBody = new CreateAliasRequestBody(); requestBody.name = "validName"; requestBody.collections = List.of("validColl1"); - final var categoryRouter = new CreateAliasAPI.CategoryRoutedAliasProperties(); + final var categoryRouter = new CategoryRoutedAliasProperties(); categoryRouter.field = "someField"; requestBody.routers = List.of(categoryRouter); - final var thrown = expectThrows(SolrException.class, () -> requestBody.validate()); + final var thrown = + expectThrows(SolrException.class, () -> CreateAlias.validateRequestBody(requestBody)); assertEquals( "Collections cannot be specified when creating a routed alias.", thrown.getMessage()); } @Test public void testReportsErrorIfNeitherExplicitCollectionsNorRoutingParamsProvided() { - final var requestBody = new CreateAliasAPI.CreateAliasRequestBody(); + final var requestBody = new CreateAliasRequestBody(); requestBody.name = "validName"; - final var thrown = expectThrows(SolrException.class, () -> requestBody.validate()); + final var thrown = + expectThrows(SolrException.class, () -> CreateAlias.validateRequestBody(requestBody)); assertEquals(400, thrown.code()); assertEquals( "Alias creation requires either a list of either collections (for creating a traditional alias) or routers (for creating a routed alias)", @@ -90,16 +97,17 @@ public void testReportsErrorIfNeitherExplicitCollectionsNorRoutingParamsProvided @Test public void testRoutedAliasesMustProvideAConfigsetToUseOnCreatedCollections() { - final var requestBody = new CreateAliasAPI.CreateAliasRequestBody(); + final var requestBody = new CreateAliasRequestBody(); requestBody.name = "validName"; - final var categoryRouter = new CreateAliasAPI.CategoryRoutedAliasProperties(); + final var categoryRouter = new CategoryRoutedAliasProperties(); categoryRouter.field = "someField"; requestBody.routers = List.of(categoryRouter); final var createParams = new CreateCollectionRequestBody(); createParams.numShards = 3; requestBody.collCreationParameters = createParams; - final var thrown = expectThrows(SolrException.class, () -> requestBody.validate()); + final var thrown = + expectThrows(SolrException.class, () -> CreateAlias.validateRequestBody(requestBody)); assertEquals(400, thrown.code()); assertThat( thrown.getMessage(), containsString("Routed alias creation requires a configset name")); @@ -107,9 +115,9 @@ public void testRoutedAliasesMustProvideAConfigsetToUseOnCreatedCollections() { @Test public void testRoutedAliasesMustNotSpecifyANameInCollectionCreationParams() { - final var requestBody = new CreateAliasAPI.CreateAliasRequestBody(); + final var requestBody = new CreateAliasRequestBody(); requestBody.name = "validName"; - final var categoryRouter = new CreateAliasAPI.CategoryRoutedAliasProperties(); + final var categoryRouter = new CategoryRoutedAliasProperties(); categoryRouter.field = "someField"; requestBody.routers = List.of(categoryRouter); final var createParams = new CreateCollectionRequestBody(); @@ -120,7 +128,8 @@ public void testRoutedAliasesMustNotSpecifyANameInCollectionCreationParams() { createParams.name = "someCollectionName"; requestBody.collCreationParameters = createParams; - final var thrown = expectThrows(SolrException.class, () -> requestBody.validate()); + final var thrown = + expectThrows(SolrException.class, () -> CreateAlias.validateRequestBody(requestBody)); assertEquals(400, thrown.code()); assertThat(thrown.getMessage(), containsString("cannot specify the name")); @@ -128,17 +137,18 @@ public void testRoutedAliasesMustNotSpecifyANameInCollectionCreationParams() { @Test public void testReportsErrorIfCategoryRoutedAliasDoesntSpecifyAllRequiredParameters() { - final var requestBody = new CreateAliasAPI.CreateAliasRequestBody(); + final var requestBody = new CreateAliasRequestBody(); requestBody.name = "validName"; final var createParams = new CreateCollectionRequestBody(); createParams.numShards = 3; createParams.config = "someConfig"; requestBody.collCreationParameters = createParams; - final var categoryRouter = new CreateAliasAPI.CategoryRoutedAliasProperties(); + final var categoryRouter = new CategoryRoutedAliasProperties(); categoryRouter.maxCardinality = 123L; requestBody.routers = List.of(categoryRouter); - final var thrown = expectThrows(SolrException.class, () -> requestBody.validate()); + final var thrown = + expectThrows(SolrException.class, () -> CreateAlias.validateRequestBody(requestBody)); assertEquals(400, thrown.code()); assertEquals( @@ -149,12 +159,13 @@ public void testReportsErrorIfCategoryRoutedAliasDoesntSpecifyAllRequiredParamet public void testReportsErrorIfTimeRoutedAliasDoesntSpecifyAllRequiredParameters() { // No 'field' defined! { - final var timeRouter = new CreateAliasAPI.TimeRoutedAliasProperties(); + final var timeRouter = new TimeRoutedAliasProperties(); timeRouter.start = "NOW"; timeRouter.interval = "+5MINUTES"; final var requestBody = requestBodyWithProvidedRouter(timeRouter); - final var thrown = expectThrows(SolrException.class, () -> requestBody.validate()); + final var thrown = + expectThrows(SolrException.class, () -> CreateAlias.validateRequestBody(requestBody)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: 'field' on time routed alias", thrown.getMessage()); @@ -162,12 +173,13 @@ public void testReportsErrorIfTimeRoutedAliasDoesntSpecifyAllRequiredParameters( // No 'start' defined! { - final var timeRouter = new CreateAliasAPI.TimeRoutedAliasProperties(); + final var timeRouter = new TimeRoutedAliasProperties(); timeRouter.field = "someField"; timeRouter.interval = "+5MINUTES"; final var requestBody = requestBodyWithProvidedRouter(timeRouter); - final var thrown = expectThrows(SolrException.class, () -> requestBody.validate()); + final var thrown = + expectThrows(SolrException.class, () -> CreateAlias.validateRequestBody(requestBody)); assertEquals(400, thrown.code()); assertEquals("Missing required parameter: 'start' on time routed alias", thrown.getMessage()); @@ -175,12 +187,13 @@ public void testReportsErrorIfTimeRoutedAliasDoesntSpecifyAllRequiredParameters( // No 'interval' defined! { - final var timeRouter = new CreateAliasAPI.TimeRoutedAliasProperties(); + final var timeRouter = new TimeRoutedAliasProperties(); timeRouter.field = "someField"; timeRouter.start = "NOW"; final var requestBody = requestBodyWithProvidedRouter(timeRouter); - final var thrown = expectThrows(SolrException.class, () -> requestBody.validate()); + final var thrown = + expectThrows(SolrException.class, () -> CreateAlias.validateRequestBody(requestBody)); assertEquals(400, thrown.code()); assertEquals( @@ -190,13 +203,13 @@ public void testReportsErrorIfTimeRoutedAliasDoesntSpecifyAllRequiredParameters( @Test public void testRemoteMessageCreationForTraditionalAlias() { - final var requestBody = new CreateAliasAPI.CreateAliasRequestBody(); + final var requestBody = new CreateAliasRequestBody(); requestBody.name = "someAliasName"; requestBody.collections = List.of("validColl1", "validColl2"); requestBody.async = "someAsyncId"; final var remoteMessage = - CreateAliasAPI.createRemoteMessageForTraditionalAlias(requestBody).getProperties(); + CreateAlias.createRemoteMessageForTraditionalAlias(requestBody).getProperties(); assertEquals(4, remoteMessage.size()); assertEquals("createalias", remoteMessage.get(QUEUE_OPERATION)); @@ -207,9 +220,9 @@ public void testRemoteMessageCreationForTraditionalAlias() { @Test public void testRemoteMessageCreationForCategoryRoutedAlias() { - final var requestBody = new CreateAliasAPI.CreateAliasRequestBody(); + final var requestBody = new CreateAliasRequestBody(); requestBody.name = "someAliasName"; - final var categoryRouter = new CreateAliasAPI.CategoryRoutedAliasProperties(); + final var categoryRouter = new CategoryRoutedAliasProperties(); categoryRouter.field = "someField"; requestBody.routers = List.of(categoryRouter); final var createParams = new CreateCollectionRequestBody(); @@ -218,7 +231,7 @@ public void testRemoteMessageCreationForCategoryRoutedAlias() { requestBody.collCreationParameters = createParams; final var remoteMessage = - CreateAliasAPI.createRemoteMessageForRoutedAlias(requestBody).getProperties(); + CreateAlias.createRemoteMessageForRoutedAlias(requestBody).getProperties(); assertEquals(6, remoteMessage.size()); assertEquals("createalias", remoteMessage.get(QUEUE_OPERATION)); @@ -231,9 +244,9 @@ public void testRemoteMessageCreationForCategoryRoutedAlias() { @Test public void testRemoteMessageCreationForTimeRoutedAlias() { - final var requestBody = new CreateAliasAPI.CreateAliasRequestBody(); + final var requestBody = new CreateAliasRequestBody(); requestBody.name = "someAliasName"; - final var timeRouter = new CreateAliasAPI.TimeRoutedAliasProperties(); + final var timeRouter = new TimeRoutedAliasProperties(); timeRouter.field = "someField"; timeRouter.start = "NOW"; timeRouter.interval = "+1MONTH"; @@ -245,7 +258,7 @@ public void testRemoteMessageCreationForTimeRoutedAlias() { requestBody.collCreationParameters = createParams; final var remoteMessage = - CreateAliasAPI.createRemoteMessageForRoutedAlias(requestBody).getProperties(); + CreateAlias.createRemoteMessageForRoutedAlias(requestBody).getProperties(); assertEquals(9, remoteMessage.size()); assertEquals("createalias", remoteMessage.get(QUEUE_OPERATION)); @@ -261,14 +274,14 @@ public void testRemoteMessageCreationForTimeRoutedAlias() { @Test public void testRemoteMessageCreationForMultiDimensionalRoutedAlias() { - final var requestBody = new CreateAliasAPI.CreateAliasRequestBody(); + final var requestBody = new CreateAliasRequestBody(); requestBody.name = "someAliasName"; - final var timeRouter = new CreateAliasAPI.TimeRoutedAliasProperties(); + final var timeRouter = new TimeRoutedAliasProperties(); timeRouter.field = "someField"; timeRouter.start = "NOW"; timeRouter.interval = "+1MONTH"; timeRouter.maxFutureMs = 123456L; - final var categoryRouter = new CreateAliasAPI.CategoryRoutedAliasProperties(); + final var categoryRouter = new CategoryRoutedAliasProperties(); categoryRouter.field = "someField"; requestBody.routers = List.of(timeRouter, categoryRouter); final var createParams = new CreateCollectionRequestBody(); @@ -277,7 +290,7 @@ public void testRemoteMessageCreationForMultiDimensionalRoutedAlias() { requestBody.collCreationParameters = createParams; final var remoteMessage = - CreateAliasAPI.createRemoteMessageForRoutedAlias(requestBody).getProperties(); + CreateAlias.createRemoteMessageForRoutedAlias(requestBody).getProperties(); assertEquals(11, remoteMessage.size()); assertEquals("createalias", remoteMessage.get(QUEUE_OPERATION)); @@ -293,9 +306,8 @@ public void testRemoteMessageCreationForMultiDimensionalRoutedAlias() { assertEquals("someConfig", remoteMessage.get("create-collection.collection.configName")); } - private CreateAliasAPI.CreateAliasRequestBody requestBodyWithProvidedRouter( - CreateAliasAPI.RoutedAliasProperties router) { - final var requestBody = new CreateAliasAPI.CreateAliasRequestBody(); + private CreateAliasRequestBody requestBodyWithProvidedRouter(RoutedAliasProperties router) { + final var requestBody = new CreateAliasRequestBody(); requestBody.name = "validName"; final var createParams = new CreateCollectionRequestBody(); createParams.numShards = 3; @@ -321,23 +333,22 @@ public void testConvertsV1ParamsForMultiDimensionalAliasToV2RequestBody() { v1Params.add("create-collection.numShards", "3"); v1Params.add("create-collection.collection.configName", "someConfig"); - final var requestBody = CreateAliasAPI.createFromSolrParams(v1Params); + final var requestBody = CreateAlias.createFromSolrParams(v1Params); assertEquals("someAliasName", requestBody.name); assertEquals(2, requestBody.routers.size()); assertTrue( "Incorrect router type " + requestBody.routers.get(0) + " at index 0", - requestBody.routers.get(0) instanceof CreateAliasAPI.TimeRoutedAliasProperties); - final var timeRouter = (CreateAliasAPI.TimeRoutedAliasProperties) requestBody.routers.get(0); + requestBody.routers.get(0) instanceof TimeRoutedAliasProperties); + final var timeRouter = (TimeRoutedAliasProperties) requestBody.routers.get(0); assertEquals("someField", timeRouter.field); assertEquals("NOW", timeRouter.start); assertEquals("+1MONTH", timeRouter.interval); assertEquals(Long.valueOf(123456L), timeRouter.maxFutureMs); assertTrue( "Incorrect router type " + requestBody.routers.get(1) + " at index 1", - requestBody.routers.get(1) instanceof CreateAliasAPI.CategoryRoutedAliasProperties); - final var categoryRouter = - (CreateAliasAPI.CategoryRoutedAliasProperties) requestBody.routers.get(1); + requestBody.routers.get(1) instanceof CategoryRoutedAliasProperties); + final var categoryRouter = (CategoryRoutedAliasProperties) requestBody.routers.get(1); assertEquals("someOtherField", categoryRouter.field); assertEquals(Long.valueOf(20), categoryRouter.maxCardinality); final var createCollParams = requestBody.collCreationParameters; diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/GetSchemaFieldsAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/GetSchemaFieldsAPITest.java index 03e134602a0..3c5a9f0a0de 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/GetSchemaFieldsAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/GetSchemaFieldsAPITest.java @@ -30,13 +30,13 @@ import org.junit.Before; import org.junit.Test; -/** Unit tests for {@link GetSchemaFieldAPI} */ +/** Unit tests for {@link GetSchemaField} */ @SuppressWarnings("unchecked") public class GetSchemaFieldsAPITest extends SolrTestCaseJ4 { private IndexSchema mockSchema; private SolrParams mockParams; - private GetSchemaFieldAPI api; + private GetSchemaField api; private SimpleOrderedMap mockField; private ArrayList> mockFieldList; @@ -47,7 +47,7 @@ public void setUpMocks() { mockSchema = mock(IndexSchema.class); mockParams = mock(SolrParams.class); - api = new GetSchemaFieldAPI(mockSchema, mockParams); + api = new GetSchemaField(mockSchema, mockParams); mockField = new SimpleOrderedMap<>(); mockField.add("name", "id"); @@ -76,7 +76,7 @@ public void testReliesOnIndexSchemaWhenFetchingSpecificField() { final var response = api.getFieldInfo("id"); assertNotNull(response); - assertCorrectField(response.fieldInfo); + assertCorrectField((SimpleOrderedMap) response.fieldInfo); } @Test @@ -109,7 +109,7 @@ public void testReliesOnIndexSchemaWhenFetchingSpecificDynamicField() { final var response = api.getDynamicFieldInfo("id"); assertNotNull(response); - assertCorrectField(response.dynamicFieldInfo); + assertCorrectField((SimpleOrderedMap) response.dynamicFieldInfo); } @Test @@ -131,7 +131,7 @@ public void testReliesOnIndexSchemaWhenFetchingSpecificFieldType() { final var response = api.getFieldTypeInfo("id"); assertNotNull(response); - assertCorrectField(response.fieldTypeInfo); + assertCorrectField((SimpleOrderedMap) response.fieldTypeInfo); } private void assertCorrectListFields(List responseFields) { diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/NodeLoggingAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/NodeLoggingAPITest.java index 1572812cb06..8838ac9e483 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/NodeLoggingAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/NodeLoggingAPITest.java @@ -19,6 +19,7 @@ import static org.apache.solr.SolrTestCaseJ4.assumeWorkingMockito; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.instanceOf; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; @@ -27,6 +28,8 @@ import java.util.List; import org.apache.solr.SolrTestCase; +import org.apache.solr.client.api.model.LogLevelChange; +import org.apache.solr.client.api.model.SetThresholdRequestBody; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.core.CoreContainer; @@ -36,7 +39,7 @@ import org.junit.BeforeClass; import org.junit.Test; -/** Unit tests for the functionality offered in {@link NodeLoggingAPI} */ +/** Unit tests for the functionality offered in {@link NodeLogging} */ @SuppressWarnings({"unchecked", "rawtypes"}) public class NodeLoggingAPITest extends SolrTestCase { @@ -61,7 +64,7 @@ public void testReliesOnLogWatcherToListLogLevels() { .thenReturn(List.of("ERROR", "WARN", "INFO", "DEBUG", "TRACE")); when(mockLogWatcher.getAllLoggers()) .thenReturn(List.of(logInfo("org.a.s.Foo", "WARN", true), logInfo("org", null, false))); - final var responseBody = new NodeLoggingAPI(mockCoreContainer).listAllLoggersAndLevels(); + final var responseBody = new NodeLogging(mockCoreContainer).listAllLoggersAndLevels(); assertEquals(5, responseBody.levels.size()); assertThat(responseBody.levels, containsInAnyOrder("ERROR", "WARN", "INFO", "DEBUG", "TRACE")); @@ -80,8 +83,8 @@ public void testReliesOnLogWatcherToListLogLevels() { @Test public void testReliesOnLogWatcherToModifyLogLevels() { final var responseBody = - new NodeLoggingAPI(mockCoreContainer) - .modifyLocalLogLevel(List.of(new NodeLoggingAPI.LogLevelChange("o.a.s.Foo", "WARN"))); + new NodeLogging(mockCoreContainer) + .modifyLocalLogLevel(List.of(new LogLevelChange("o.a.s.Foo", "WARN"))); assertNotNull(responseBody); assertNull("Expected error to be null but was " + responseBody.error, responseBody.error); @@ -105,7 +108,7 @@ public void testReliesOnLogWatcherToFetchLogMessages() { when(mockLogWatcher.getLastEvent()).thenReturn(123456L); when(mockLogWatcher.getHistorySize()).thenReturn(321); - final var responseBody = new NodeLoggingAPI(mockCoreContainer).fetchLocalLogMessages(123L); + final var responseBody = new NodeLogging(mockCoreContainer).fetchLocalLogMessages(123L); assertNotNull(responseBody); assertNull("Expected error to be null but was " + responseBody.error, responseBody.error); @@ -113,16 +116,17 @@ public void testReliesOnLogWatcherToFetchLogMessages() { assertEquals(Long.valueOf(123), responseBody.info.boundingTimeMillis); assertEquals(123456, responseBody.info.lastRecordTimestampMillis); assertEquals(321, responseBody.info.buffer); - assertEquals(2, responseBody.docs.size()); - assertEquals("logmsg1", responseBody.docs.get(0).getFieldValue("message_s")); - assertEquals("logmsg2", responseBody.docs.get(1).getFieldValue("message_s")); + assertThat(responseBody.docs, instanceOf(SolrDocumentList.class)); + final var docList = (SolrDocumentList) responseBody.docs; + assertEquals(2, docList.size()); + assertEquals("logmsg1", docList.get(0).getFieldValue("message_s")); + assertEquals("logmsg2", docList.get(1).getFieldValue("message_s")); } @Test public void testReliesOnLogWatcherToSetMessageThreshold() { final var responseBody = - new NodeLoggingAPI(mockCoreContainer) - .setMessageThreshold(new NodeLoggingAPI.SetThresholdRequestBody("WARN")); + new NodeLogging(mockCoreContainer).setMessageThreshold(new SetThresholdRequestBody("WARN")); assertNotNull(responseBody); assertNull("Expected error to be null but was " + responseBody.error, responseBody.error); @@ -134,8 +138,7 @@ public void testReliesOnLogWatcherToSetMessageThreshold() { @Test public void testCorrectlyParsesV1LogLevelChanges() { final var levelChanges = - NodeLoggingAPI.LogLevelChange.createRequestBodyFromV1Params( - new String[] {"o.a.s.Foo:WARN", "o.a.s.Bar:INFO"}); + NodeLogging.parseLogLevelChanges(new String[] {"o.a.s.Foo:WARN", "o.a.s.Bar:INFO"}); assertEquals(2, levelChanges.size()); assertEquals("o.a.s.Foo", levelChanges.get(0).logger); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/ReloadCoreAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/ReloadCoreAPITest.java index 304717a167a..fc73c6781f9 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/ReloadCoreAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/ReloadCoreAPITest.java @@ -57,7 +57,6 @@ public void setUp() throws Exception { public void testValidReloadCoreAPIResponse() throws Exception { SolrJerseyResponse response = reloadCoreAPI.reloadCore(coreName, new ReloadCoreRequestBody()); assertEquals(0, response.responseHeader.status); - assertNotNull(response.responseHeader.qTime); } @Test diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/RestoreCollectionAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/RestoreCollectionAPITest.java index 922cc989726..e4aab80a2df 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/RestoreCollectionAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/RestoreCollectionAPITest.java @@ -37,6 +37,7 @@ import java.util.Map; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.api.model.CreateCollectionRequestBody; +import org.apache.solr.client.api.model.RestoreCollectionRequestBody; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; @@ -46,15 +47,15 @@ import org.junit.BeforeClass; import org.junit.Test; -/** Unit tests for {@link RestoreCollectionAPI} */ +/** Unit tests for {@link RestoreCollection} */ public class RestoreCollectionAPITest extends SolrTestCaseJ4 { - private static RestoreCollectionAPI restoreCollectionAPI; + private static RestoreCollection restoreCollectionAPI; @BeforeClass public static void setUpApi() { restoreCollectionAPI = - new RestoreCollectionAPI( + new RestoreCollection( new CoreContainer( new NodeConfig.NodeConfigBuilder("testnode", createTempDir()).build()), new LocalSolrQueryRequest(null, new NamedList<>()), @@ -63,7 +64,7 @@ public static void setUpApi() { @Test public void testReportsErrorIfBackupNameMissing() { - final var requestBody = new RestoreCollectionAPI.RestoreCollectionRequestBody(); + final var requestBody = new RestoreCollectionRequestBody(); requestBody.collection = "someCollection"; final SolrException thrown = expectThrows( @@ -92,7 +93,7 @@ public void testReportsErrorIfRequestBodyMissing() { @Test public void testReportsErrorIfCollectionNameMissing() { // No 'collection' set on the requestBody - final var requestBody = new RestoreCollectionAPI.RestoreCollectionRequestBody(); + final var requestBody = new RestoreCollectionRequestBody(); final SolrException thrown = expectThrows( SolrException.class, @@ -106,7 +107,7 @@ public void testReportsErrorIfCollectionNameMissing() { @Test public void testReportsErrorIfProvidedCollectionNameIsInvalid() { - final var requestBody = new RestoreCollectionAPI.RestoreCollectionRequestBody(); + final var requestBody = new RestoreCollectionRequestBody(); requestBody.collection = "invalid$collection@name"; final SolrException thrown = expectThrows( @@ -122,7 +123,7 @@ public void testReportsErrorIfProvidedCollectionNameIsInvalid() { @Test public void testCreatesValidRemoteMessageForExistingCollectionRestore() { - final var requestBody = new RestoreCollectionAPI.RestoreCollectionRequestBody(); + final var requestBody = new RestoreCollectionRequestBody(); requestBody.collection = "someCollectionName"; requestBody.location = "/some/location/path"; requestBody.backupId = 123; @@ -145,7 +146,7 @@ public void testCreatesValidRemoteMessageForExistingCollectionRestore() { @Test public void testCreatesValidRemoteMessageForNewCollectionRestore() { - final var requestBody = new RestoreCollectionAPI.RestoreCollectionRequestBody(); + final var requestBody = new RestoreCollectionRequestBody(); requestBody.collection = "someCollectionName"; requestBody.location = "/some/location/path"; requestBody.backupId = 123; @@ -197,8 +198,7 @@ public void testCanConvertV1ParamsIntoV2RequestBody() { v1Params.add("createNodeSet", "node1,node2"); v1Params.add("createNodeSet.shuffle", "false"); - final var requestBody = - RestoreCollectionAPI.RestoreCollectionRequestBody.fromV1Params(v1Params); + final var requestBody = RestoreCollection.createRequestBodyFromV1Params(v1Params); assertEquals("someCollectionName", requestBody.collection); assertEquals("/some/location/str", requestBody.location); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/SnapshotBackupAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/SnapshotBackupAPITest.java index dee50c1ebcb..aede987f2b6 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/SnapshotBackupAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/SnapshotBackupAPITest.java @@ -24,6 +24,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import org.apache.solr.SolrTestCase; +import org.apache.solr.client.api.model.ReplicationBackupRequestBody; import org.apache.solr.common.SolrException; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; @@ -65,7 +66,7 @@ public void testMissingBody() throws Exception { @Test public void testSuccessfulBackupCommand() throws Exception { when(replicationHandlerConfig.getNumberBackupsToKeep()).thenReturn(11); - final var backupRequestBody = new SnapshotBackupAPI.BackupReplicationRequestBody(); + final var backupRequestBody = new ReplicationBackupRequestBody(); backupRequestBody.name = "test"; backupRequestBody.numberToKeep = 7; diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/UnloadCoreAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/UnloadCoreAPITest.java index d2dcc732b2c..c823f33dea1 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/UnloadCoreAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/UnloadCoreAPITest.java @@ -56,7 +56,6 @@ public void setUp() throws Exception { public void testValidUnloadCoreAPIResponse() throws Exception { SolrJerseyResponse response = unloadCoreAPI.unloadCore(coreName, getUnloadCoreRequestBodyObj()); assertEquals(0, response.responseHeader.status); - assertNotNull(response.responseHeader.qTime); } @Test diff --git a/solr/core/src/test/org/apache/solr/handler/api/V2ApiUtilsTest.java b/solr/core/src/test/org/apache/solr/handler/api/V2ApiUtilsTest.java index 51b89243dbe..78f2f8d5fb9 100644 --- a/solr/core/src/test/org/apache/solr/handler/api/V2ApiUtilsTest.java +++ b/solr/core/src/test/org/apache/solr/handler/api/V2ApiUtilsTest.java @@ -17,7 +17,7 @@ package org.apache.solr.handler.api; import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE_V2; -import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM; +import static org.apache.solr.handler.admin.api.ReplicationAPIBase.FILE_STREAM; import jakarta.ws.rs.core.MediaType; import org.apache.solr.SolrTestCaseJ4; diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallTest.java index 68de31559c4..97253a63518 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedFacetPivotSmallTest.java @@ -2523,8 +2523,7 @@ public ComparablePivotField(String f, Object v, int count, List pivo public boolean equals(Object obj) { if (this == obj) return true; if (obj == null) return false; - if (!(obj instanceof PivotField)) return false; - PivotField other = (PivotField) obj; + if (!(obj instanceof PivotField other)) return false; if (getCount() != other.getCount()) return false; if (getField() == null) { if (other.getField() != null) return false; @@ -2595,8 +2594,7 @@ public int hashCode() { @Override public boolean equals(Object o) { boolean equal = false; - if (o instanceof ArrayList) { - List otherList = (List) o; + if (o instanceof List otherList) { if (size() == otherList.size()) { equal = true; for (Object objectInOtherList : otherList) { diff --git a/solr/core/src/test/org/apache/solr/handler/component/TestShardHandlerFactory.java b/solr/core/src/test/org/apache/solr/handler/component/TestShardHandlerFactory.java index 541d2d845f3..970fa7bb48a 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/TestShardHandlerFactory.java +++ b/solr/core/src/test/org/apache/solr/handler/component/TestShardHandlerFactory.java @@ -17,21 +17,16 @@ package org.apache.solr.handler.component; import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.hasItem; import static org.hamcrest.CoreMatchers.instanceOf; -import static org.hamcrest.CoreMatchers.is; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Set; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.impl.LBSolrClient; import org.apache.solr.client.solrj.request.QueryRequest; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.ShardParams; @@ -155,18 +150,6 @@ public void getShardsAllowList() { } } - @Test - public void testLiveNodesToHostUrl() { - Set liveNodes = - new HashSet<>(Arrays.asList("1.2.3.4:8983_solr", "1.2.3.4:9000_", "1.2.3.4:9001_solr-2")); - ClusterState cs = new ClusterState(liveNodes, new HashMap<>()); - Set hostSet = cs.getHostAllowList(); - assertThat(hostSet.size(), is(3)); - assertThat(hostSet, hasItem("1.2.3.4:8983")); - assertThat(hostSet, hasItem("1.2.3.4:9000")); - assertThat(hostSet, hasItem("1.2.3.4:9001")); - } - @Test public void testXML() { Path home = TEST_PATH(); diff --git a/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerAPI.java b/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerAPI.java index f059acb8a6c..983d0fa3ab1 100644 --- a/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerAPI.java +++ b/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerAPI.java @@ -188,7 +188,7 @@ public void testAddTechproductsProgressively() throws Exception { when(req.getParams()).thenReturn(reqParams); // POST some sample JSON docs - ContentStreamBase.FileStream stream = new ContentStreamBase.FileStream(next); + ContentStreamBase.FileStream stream = new ContentStreamBase.FileStream(next.toPath()); stream.setContentType(TestSampleDocumentsLoader.guessContentTypeFromFilename(next.getName())); when(req.getContentStreams()).thenReturn(Collections.singletonList(stream)); @@ -300,7 +300,7 @@ public void testSuggestFilmsXml() throws Exception { when(req.getParams()).thenReturn(reqParams); // POST some sample XML docs - ContentStreamBase.FileStream stream = new ContentStreamBase.FileStream(filmsXml); + ContentStreamBase.FileStream stream = new ContentStreamBase.FileStream(filmsXml.toPath()); stream.setContentType("application/xml"); when(req.getContentStreams()).thenReturn(Collections.singletonList(stream)); @@ -354,7 +354,7 @@ public void testBasicUserWorkflow() throws Exception { // POST some sample JSON docs File booksJson = new File(ExternalPaths.SOURCE_HOME, "example/exampledocs/books.json"); - ContentStreamBase.FileStream stream = new ContentStreamBase.FileStream(booksJson); + ContentStreamBase.FileStream stream = new ContentStreamBase.FileStream(booksJson.toPath()); stream.setContentType(JSON_MIME); when(req.getContentStreams()).thenReturn(Collections.singletonList(stream)); diff --git a/solr/core/src/test/org/apache/solr/handler/tagger/TaggerTestCase.java b/solr/core/src/test/org/apache/solr/handler/tagger/TaggerTestCase.java index 7e68744dc85..da0a8d753f9 100644 --- a/solr/core/src/test/org/apache/solr/handler/tagger/TaggerTestCase.java +++ b/solr/core/src/test/org/apache/solr/handler/tagger/TaggerTestCase.java @@ -241,8 +241,7 @@ public String toString() { @Override public boolean equals(Object obj) { - if (!(obj instanceof TestTag)) return false; - TestTag that = (TestTag) obj; + if (!(obj instanceof TestTag that)) return false; return this.startOffset == that.startOffset && this.endOffset == that.endOffset && Objects.equals(this.docName, that.docName); diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java b/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java index 0d94bcf1f5a..26d8261c07f 100644 --- a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java @@ -51,8 +51,8 @@ public class PackageStoreSchemaPluginsTest extends SolrCloudTestCase { } } - private final Path pluginJarPath = getFile("runtimecode/schema-plugins.jar.bin").toPath(); - private final Path bogusJarPath = getFile("runtimecode/runtimelibs.jar.bin").toPath(); + private final Path pluginJarPath = getFile("runtimecode/schema-plugins.jar.bin"); + private final Path bogusJarPath = getFile("runtimecode/runtimelibs.jar.bin"); private SolrClient client; diff --git a/solr/core/src/test/org/apache/solr/request/TestWriterPerf.java b/solr/core/src/test/org/apache/solr/request/TestWriterPerf.java index f8d1942945e..4ba2c17bedf 100644 --- a/solr/core/src/test/org/apache/solr/request/TestWriterPerf.java +++ b/solr/core/src/test/org/apache/solr/request/TestWriterPerf.java @@ -179,8 +179,7 @@ void doPerf(String writerName, SolrQueryRequest req, int encIter, int decIter) t System.gc(); RTimer timer = new RTimer(); for (int i = 0; i < encIter; i++) { - if (w instanceof BinaryQueryResponseWriter) { - BinaryQueryResponseWriter binWriter = (BinaryQueryResponseWriter) w; + if (w instanceof BinaryQueryResponseWriter binWriter) { out = new ByteArrayOutputStream(); binWriter.write(out, req, rsp); out.close(); diff --git a/solr/core/src/test/org/apache/solr/rest/TestRestManager.java b/solr/core/src/test/org/apache/solr/rest/TestRestManager.java index a95bd3d3edd..b8ef9b0de84 100644 --- a/solr/core/src/test/org/apache/solr/rest/TestRestManager.java +++ b/solr/core/src/test/org/apache/solr/rest/TestRestManager.java @@ -17,6 +17,7 @@ package org.apache.solr.rest; import java.io.File; +import java.io.IOException; import java.nio.file.Paths; import java.util.Arrays; import org.apache.solr.common.util.NamedList; @@ -120,7 +121,7 @@ public void testRestManagerEndpoints() throws Exception { } @Test - public void testReloadFromPersistentStorage() { + public void testReloadFromPersistentStorage() throws IOException { SolrResourceLoader loader = new SolrResourceLoader(Paths.get("./")); File unitTestStorageDir = createTempDir("testRestManager").toFile(); assertTrue( diff --git a/solr/core/src/test/org/apache/solr/schema/ExternalFileFieldSortTest.java b/solr/core/src/test/org/apache/solr/schema/ExternalFileFieldSortTest.java index 1a554d551b5..23a1152c3eb 100644 --- a/solr/core/src/test/org/apache/solr/schema/ExternalFileFieldSortTest.java +++ b/solr/core/src/test/org/apache/solr/schema/ExternalFileFieldSortTest.java @@ -26,9 +26,9 @@ public class ExternalFileFieldSortTest extends SolrTestCaseJ4 { static void updateExternalFile() throws IOException { - final String testHome = SolrTestCaseJ4.getFile("solr/collection1").getParent(); + final Path testHome = SolrTestCaseJ4.getFile("solr/collection1").getParent(); String filename = "external_eff"; - Files.copy(Path.of(testHome, filename), Path.of(h.getCore().getDataDir(), filename)); + Files.copy(testHome.resolve(filename), Path.of(h.getCore().getDataDir(), filename)); } private void addDocuments() { diff --git a/solr/core/src/test/org/apache/solr/schema/SchemaVersionSpecificBehaviorTest.java b/solr/core/src/test/org/apache/solr/schema/SchemaVersionSpecificBehaviorTest.java index 41d64fb3cb6..1f37e4c9c37 100644 --- a/solr/core/src/test/org/apache/solr/schema/SchemaVersionSpecificBehaviorTest.java +++ b/solr/core/src/test/org/apache/solr/schema/SchemaVersionSpecificBehaviorTest.java @@ -50,8 +50,7 @@ public void testVersionBehavior() throws Exception { field.omitTermFreqAndPositions()); // 1.4: autoGeneratePhraseQueries default changed to false - if (field.getType() instanceof TextField) { - TextField ft = (TextField) field.getType(); + if (field.getType() instanceof TextField ft) { assertEquals( f + " field's autoPhrase is wrong for ver=" + ver, (v < 1.4F), diff --git a/solr/core/src/test/org/apache/solr/schema/TestCollationField.java b/solr/core/src/test/org/apache/solr/schema/TestCollationField.java index 0ac2d36e4bb..437c397ea59 100644 --- a/solr/core/src/test/org/apache/solr/schema/TestCollationField.java +++ b/solr/core/src/test/org/apache/solr/schema/TestCollationField.java @@ -64,14 +64,11 @@ public static String setupSolrHome() throws Exception { // copy over configuration files Files.copy( - getFile("solr/collection1/conf/solrconfig-basic.xml").toPath(), - confDir.resolve("solrconfig.xml")); + getFile("solr/collection1/conf/solrconfig-basic.xml"), confDir.resolve("solrconfig.xml")); Files.copy( - getFile("solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml").toPath(), + getFile("solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml"), confDir.resolve("solrconfig.snippet.randomindexconfig.xml")); - Files.copy( - getFile("solr/collection1/conf/schema-collate.xml").toPath(), - confDir.resolve("schema.xml")); + Files.copy(getFile("solr/collection1/conf/schema-collate.xml"), confDir.resolve("schema.xml")); // generate custom collation rules (DIN 5007-2), saving to customrules.dat RuleBasedCollator baseCollator = diff --git a/solr/core/src/test/org/apache/solr/schema/TestCollationFieldDocValues.java b/solr/core/src/test/org/apache/solr/schema/TestCollationFieldDocValues.java index b38b65f81fc..1443d2f1f90 100644 --- a/solr/core/src/test/org/apache/solr/schema/TestCollationFieldDocValues.java +++ b/solr/core/src/test/org/apache/solr/schema/TestCollationFieldDocValues.java @@ -64,14 +64,12 @@ public static String setupSolrHome() throws Exception { // copy over configuration files Files.copy( - getFile("solr/collection1/conf/solrconfig-basic.xml").toPath(), - confDir.resolve("solrconfig.xml")); + getFile("solr/collection1/conf/solrconfig-basic.xml"), confDir.resolve("solrconfig.xml")); Files.copy( - getFile("solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml").toPath(), + getFile("solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml"), confDir.resolve("solrconfig.snippet.randomindexconfig.xml")); Files.copy( - getFile("solr/collection1/conf/schema-collate-dv.xml").toPath(), - confDir.resolve("schema.xml")); + getFile("solr/collection1/conf/schema-collate-dv.xml"), confDir.resolve("schema.xml")); // generate custom collation rules (DIN 5007-2), saving to customrules.dat RuleBasedCollator baseCollator = diff --git a/solr/core/src/test/org/apache/solr/schema/TestManagedSchemaThreadSafety.java b/solr/core/src/test/org/apache/solr/schema/TestManagedSchemaThreadSafety.java index bd452178f63..2cc881cecc2 100644 --- a/solr/core/src/test/org/apache/solr/schema/TestManagedSchemaThreadSafety.java +++ b/solr/core/src/test/org/apache/solr/schema/TestManagedSchemaThreadSafety.java @@ -183,8 +183,7 @@ private Runnable indexSchemaLoader(String configsetName, final ZkController zkCo try { SolrResourceLoader loader = new ZkSolrResourceLoader(loaderPath, configsetName, null, zkController); - SolrConfig solrConfig = - SolrConfig.readFromResourceLoader(loader, "solrconfig.xml", true, null); + SolrConfig solrConfig = SolrConfig.readFromResourceLoader(loader, "solrconfig.xml", null); ManagedIndexSchemaFactory factory = new ManagedIndexSchemaFactory(); factory.init(new NamedList<>()); diff --git a/solr/core/src/test/org/apache/solr/search/ComponentStageLimitsTest.java b/solr/core/src/test/org/apache/solr/search/ComponentStageLimitsTest.java new file mode 100644 index 00000000000..c3c5d72548e --- /dev/null +++ b/solr/core/src/test/org/apache/solr/search/ComponentStageLimitsTest.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.search; + +import static org.apache.solr.response.SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_DETAILS_KEY; + +import java.lang.invoke.MethodHandles; +import java.nio.file.Files; +import java.nio.file.Path; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.index.NoMergePolicyFactory; +import org.apache.solr.util.TestInjection; +import org.apache.solr.util.ThreadCpuTimer; +import org.junit.AfterClass; +import org.junit.Assume; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ComponentStageLimitsTest extends SolrCloudTestCase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static final String COLLECTION = "test"; + + private static Path createConfigSet() throws Exception { + Path configSet = createTempDir(); + copyMinConf(configSet.toFile()); + // insert an expensive search component + Path solrConfig = configSet.resolve("conf/solrconfig.xml"); + Files.writeString( + solrConfig, + Files.readString(solrConfig) + .replace( + "\n" + + "\n" + + " ", + "class=\"solr.SearchHandler\">\n" + + " \n" + + " expensiveSearchComponent\n" + + " \n")); + return configSet.resolve("conf"); + } + + @BeforeClass + public static void setupClass() throws Exception { + // Note: copied from TestCpuAllowedLimit.java, Segments not likely important + // at the moment, but setup retained for now. + + // Using NoMergePolicy and 100 commits we should get 100 segments (across all shards). + // At this point of writing MAX_SEGMENTS_PER_SLICE in lucene is 5, so we should be + // ensured that any multithreaded testing will create 20 executable tasks for the + // executor that was provided to index-searcher. + systemSetPropertySolrTestsMergePolicyFactory(NoMergePolicyFactory.class.getName()); + System.setProperty(ThreadCpuTimer.ENABLE_CPU_TIME, "true"); + Path configset = createConfigSet(); + configureCluster(1).addConfig("conf", configset).configure(); + SolrClient solrClient = cluster.getSolrClient(); + CollectionAdminRequest.Create create = + CollectionAdminRequest.createCollection(COLLECTION, "conf", 3, 2); + create.process(solrClient); + waitForState("active", COLLECTION, clusterShape(3, 6)); + for (int j = 0; j < 100; j++) { + solrClient.add(COLLECTION, sdoc("id", "id-" + j, "val_i", j % 5)); + solrClient.commit(COLLECTION); // need to commit every doc to create many segments. + } + } + + @AfterClass + public static void tearDownClass() { + TestInjection.cpuTimerDelayInjectedNS = null; + systemClearPropertySolrTestsMergePolicyFactory(); + } + + @Test + public void testLimitPrepare() throws Exception { + Assume.assumeTrue("Thread CPU time monitoring is not available", ThreadCpuTimer.isSupported()); + SolrClient solrClient = cluster.getSolrClient(); + long sleepMs = 1000; + log.info("--- 500ms limit, 1000ms prepare phase - partial results ---"); + QueryResponse rsp = + solrClient.query( + COLLECTION, + params( + "q", + "id:*", + "sort", + "id asc", + ExpensiveSearchComponent.SLEEP_MS_PARAM, + String.valueOf(sleepMs), + "stages", + "prepare", + "timeAllowed", + "500")); + System.err.println("rsp=" + rsp.jsonStr()); + assertEquals(rsp.getHeader().get("status"), 0); + assertNotNull("should have partial results", rsp.getHeader().get("partialResults")); + String details = (String) rsp.getHeader().get(RESPONSE_HEADER_PARTIAL_RESULTS_DETAILS_KEY); + assertNotNull(details); + assertTrue(details.contains("[prepare]")); + assertTrue( + details.contains("exceeded prior to query in [expensiveSearchComponent, query, facet,")); + } + + @Test + public void testLimitProcess() throws Exception { + Assume.assumeTrue("Thread CPU time monitoring is not available", ThreadCpuTimer.isSupported()); + SolrClient solrClient = cluster.getSolrClient(); + String msg = "--- 500ms limit, 1000ms process phase - partial results ---"; + log.info(msg); + System.out.println(msg); + int sleepMs = 2000; + QueryResponse rsp = + solrClient.query( + COLLECTION, + params( + "q", + "id:*", + "sort", + "id asc", + ExpensiveSearchComponent.SLEEP_MS_PARAM, + String.valueOf(sleepMs), + "stages", + "process", + "timeAllowed", + "1000")); + System.err.println("rsp=" + rsp.jsonStr()); + assertEquals(rsp.getHeader().get("status"), 0); + String details = (String) rsp.getHeader().get(RESPONSE_HEADER_PARTIAL_RESULTS_DETAILS_KEY); + System.out.println("details=" + details); + assertNotNull(details); + assertTrue(details.contains("[process]")); + assertTrue( + details.contains("exceeded prior to query in [expensiveSearchComponent, query, facet,")); + assertNotNull("should have partial results", rsp.getHeader().get("partialResults")); + } + + @Test + public void testLimitFinish() throws Exception { + Assume.assumeTrue("Thread CPU time monitoring is not available", ThreadCpuTimer.isSupported()); + SolrClient solrClient = cluster.getSolrClient(); + long sleepMs = 1000; + log.info("--- 500ms limit, 1000ms finish phase - partial results ---"); + sleepMs = 1000; + QueryResponse rsp = + solrClient.query( + COLLECTION, + params( + "q", + "id:*", + "sort", + "id asc", + ExpensiveSearchComponent.SLEEP_MS_PARAM, + String.valueOf(sleepMs), + "stages", + "finish", + "timeAllowed", + "500")); + System.err.println("rsp=" + rsp.jsonStr()); + assertEquals(rsp.getHeader().get("status"), 0); + String details = (String) rsp.getHeader().get(RESPONSE_HEADER_PARTIAL_RESULTS_DETAILS_KEY); + assertNotNull(details); + assertTrue(details.contains("[finishStage stage:PARSE_QUERY]")); + assertTrue( + details.contains("exceeded prior to query in [expensiveSearchComponent, query, facet,")); + assertNotNull("should have partial results", rsp.getHeader().get("partialResults")); + } + + @Test + public void testLimitDistrib() throws Exception { + Assume.assumeTrue("Thread CPU time monitoring is not available", ThreadCpuTimer.isSupported()); + SolrClient solrClient = cluster.getSolrClient(); + long sleepMs = 1000; + log.info("--- 500ms limit, 1000ms distrib phase - partial results ---"); + sleepMs = 1000; + QueryResponse rsp = + solrClient.query( + COLLECTION, + params( + "q", + "id:*", + "sort", + "id asc", + ExpensiveSearchComponent.SLEEP_MS_PARAM, + String.valueOf(sleepMs), + "stages", + "distrib", + "timeAllowed", + "500")); + System.err.println("rsp=" + rsp.jsonStr()); + assertEquals(rsp.getHeader().get("status"), 0); + String details = (String) rsp.getHeader().get(RESPONSE_HEADER_PARTIAL_RESULTS_DETAILS_KEY); + assertNotNull(details); + assertTrue(details.contains("[distrib]")); + assertTrue( + details.contains("exceeded prior to query in [expensiveSearchComponent, query, facet,")); + assertNotNull("should have partial results", rsp.getHeader().get("partialResults")); + } +} diff --git a/solr/core/src/test/org/apache/solr/search/RankQueryTestPlugin.java b/solr/core/src/test/org/apache/solr/search/RankQueryTestPlugin.java index c82661200b3..674836772f7 100644 --- a/solr/core/src/test/org/apache/solr/search/RankQueryTestPlugin.java +++ b/solr/core/src/test/org/apache/solr/search/RankQueryTestPlugin.java @@ -105,8 +105,7 @@ public int hashCode() { @Override public boolean equals(Object o) { - if (o instanceof TestRankQuery) { - TestRankQuery trq = (TestRankQuery) o; + if (o instanceof TestRankQuery trq) { return (trq.q.equals(q) && trq.collector == collector); } diff --git a/solr/core/src/test/org/apache/solr/search/TestCaffeineCache.java b/solr/core/src/test/org/apache/solr/search/TestCaffeineCache.java index 87ff29d5e6a..b4c69600fbd 100644 --- a/solr/core/src/test/org/apache/solr/search/TestCaffeineCache.java +++ b/solr/core/src/test/org/apache/solr/search/TestCaffeineCache.java @@ -308,7 +308,7 @@ public void testRamBytesSync() throws IOException { cache.put(0, "test"); long nonEmptySize = cache.ramBytesUsed(); - cache.put(0, "test"); + cache.put(0, random().nextBoolean() ? "test" : "rest"); assertEquals(nonEmptySize, cache.ramBytesUsed()); cache.remove(0); @@ -341,7 +341,7 @@ public void testRamBytesAsync() throws IOException { cache.put(0, "test"); long nonEmptySize = cache.ramBytesUsed(); - cache.put(0, "test"); + cache.put(0, random().nextBoolean() ? "test" : "rest"); assertEquals(nonEmptySize, cache.ramBytesUsed()); cache.remove(0); diff --git a/solr/core/src/test/org/apache/solr/search/TestCpuAllowedLimit.java b/solr/core/src/test/org/apache/solr/search/TestCpuAllowedLimit.java index 274994099f1..4611de10eca 100644 --- a/solr/core/src/test/org/apache/solr/search/TestCpuAllowedLimit.java +++ b/solr/core/src/test/org/apache/solr/search/TestCpuAllowedLimit.java @@ -169,6 +169,7 @@ public void testDistribLimit() throws Exception { assertEquals("should have partial results", true, rsp.getHeader().get("partialResults")); log.info("--- timeAllowed, partial results, multithreading ---"); + rsp = solrClient.query( COLLECTION, @@ -201,11 +202,11 @@ public void testDistribLimit() throws Exception { "stages", "prepare,process", "cpuAllowed", - "10000")); - // System.err.println("rsp=" + rsp.jsonStr()); + "100000")); + System.err.println("rsp=" + rsp.jsonStr()); assertNull("should have full results", rsp.getHeader().get("partialResults")); // cpuAllowed set, should return partial results - log.info("--- cpuAllowed 1, partial results ---"); + log.info("--- cpuAllowed 100, partial results ---"); rsp = solrClient.query( COLLECTION, @@ -222,7 +223,7 @@ public void testDistribLimit() throws Exception { "false")); // System.err.println("rsp=" + rsp.jsonStr()); assertNotNull("should have partial results", rsp.getHeader().get("partialResults")); - log.info("--- cpuAllowed 1, partial results omitted ---"); + log.info("--- cpuAllowed 100, partial results omitted ---"); rsp = solrClient.query( COLLECTION, @@ -243,10 +244,11 @@ public void testDistribLimit() throws Exception { "foo")); String s = rsp.jsonStr(); System.err.println("rsp=" + s); - assertEquals("should have partial results", "omitted", rsp.getHeader().get("partialResults")); + assertEquals( + "should not have partial results", "omitted", rsp.getHeader().get("partialResults")); // cpuAllowed set, should return partial results - log.info("--- cpuAllowed 2, partial results, multi-threaded ---"); + log.info("--- cpuAllowed 100, partial results, multi-threaded ---"); rsp = solrClient.query( COLLECTION, diff --git a/solr/core/src/test/org/apache/solr/search/TestExtendedDismaxParser.java b/solr/core/src/test/org/apache/solr/search/TestExtendedDismaxParser.java index df78f80215e..d6f20f0e49a 100644 --- a/solr/core/src/test/org/apache/solr/search/TestExtendedDismaxParser.java +++ b/solr/core/src/test/org/apache/solr/search/TestExtendedDismaxParser.java @@ -3124,8 +3124,7 @@ private boolean containsClause( Query query, String field, String value, int boost, boolean fuzzy) { float queryBoost = 1f; - if (query instanceof BoostQuery) { - BoostQuery bq = (BoostQuery) query; + if (query instanceof BoostQuery bq) { query = bq.getQuery(); queryBoost = bq.getBoost(); } diff --git a/solr/core/src/test/org/apache/solr/search/TestRecovery.java b/solr/core/src/test/org/apache/solr/search/TestRecovery.java index 6cb6747c716..36e33b772e2 100644 --- a/solr/core/src/test/org/apache/solr/search/TestRecovery.java +++ b/solr/core/src/test/org/apache/solr/search/TestRecovery.java @@ -1386,12 +1386,12 @@ public void testRemoveOldLogs() throws Exception { h.close(); - String[] files = ulog.getLogList(logDir); + String[] files = ulog.getLogList(logDir.toPath()); for (String file : files) { Files.delete(new File(logDir, file).toPath()); } - assertEquals(0, ulog.getLogList(logDir).length); + assertEquals(0, ulog.getLogList(logDir.toPath()).length); createCore(); @@ -1422,7 +1422,8 @@ public void testRemoveOldLogs() throws Exception { assertJQ( req("qt", "/get", "getVersions", "" + maxReq), "/versions==" + versions.subList(0, Math.min(maxReq, versExpected))); - assertEquals(Math.min(i, ulog.getMaxNumLogsToKeep()), ulog.getLogList(logDir).length); + assertEquals( + Math.min(i, ulog.getMaxNumLogsToKeep()), ulog.getLogList(logDir.toPath()).length); } docsPerBatch = ulog.getNumRecordsToKeep() + 20; @@ -1443,7 +1444,7 @@ public void testRemoveOldLogs() throws Exception { "/versions==" + versions.subList(0, Math.min(maxReq, versExpected))); // previous logs should be gone now - assertEquals(1, ulog.getLogList(logDir).length); + assertEquals(1, ulog.getLogList(logDir.toPath()).length); addDocs(1, numIndexed, versions); numIndexed += 1; @@ -1480,7 +1481,7 @@ public void testRemoveOldLogs() throws Exception { "/versions==" + versions.subList(0, Math.min(maxReq, expectedToRetain))); // previous logs should be gone now - assertEquals(1, ulog.getLogList(logDir).length); + assertEquals(1, ulog.getLogList(logDir.toPath()).length); // // test that a corrupt tlog file doesn't stop us from coming up, or seeing versions before @@ -1492,7 +1493,7 @@ public void testRemoveOldLogs() throws Exception { new LinkedList< Long>()); // don't add this to the versions list because we are going to lose it... h.close(); - files = ulog.getLogList(logDir); + files = ulog.getLogList(logDir.toPath()); Arrays.sort(files); try (RandomAccessFile raf = new RandomAccessFile(new File(logDir, files[files.length - 1]), "rw")) { @@ -1548,7 +1549,7 @@ public void testTruncatedLog() throws Exception { assertU(adoc("id", "F3")); h.close(); - String[] files = ulog.getLogList(logDir); + String[] files = ulog.getLogList(logDir.toPath()); Arrays.sort(files); try (RandomAccessFile raf = new RandomAccessFile(new File(logDir, files[files.length - 1]), "rw")) { @@ -1615,7 +1616,7 @@ public void testCorruptLog() throws Exception { h.close(); - String[] files = ulog.getLogList(logDir); + String[] files = ulog.getLogList(logDir.toPath()); Arrays.sort(files); try (RandomAccessFile raf = new RandomAccessFile(new File(logDir, files[files.length - 1]), "rw")) { @@ -1698,7 +1699,7 @@ public void testRecoveryMultipleLogs() throws Exception { assertU(adoc("id", "CCCCCC")); h.close(); - String[] files = ulog.getLogList(logDir); + String[] files = ulog.getLogList(logDir.toPath()); Arrays.sort(files); String fname = files[files.length - 1]; byte[] content; @@ -1925,12 +1926,12 @@ void deleteLogs() throws Exception { h.close(); try { - String[] files = ulog.getLogList(logDir); + String[] files = ulog.getLogList(logDir.toPath()); for (String file : files) { Files.delete(new File(logDir, file).toPath()); } - assertEquals(0, ulog.getLogList(logDir).length); + assertEquals(0, ulog.getLogList(logDir.toPath()).length); } finally { // make sure we create the core again, even if the assert operation fails, so it won't mess // up the next test. diff --git a/solr/core/src/test/org/apache/solr/search/facet/SpatialHeatmapFacetsTest.java b/solr/core/src/test/org/apache/solr/search/facet/SpatialHeatmapFacetsTest.java index 185e9693860..a385eeeb498 100644 --- a/solr/core/src/test/org/apache/solr/search/facet/SpatialHeatmapFacetsTest.java +++ b/solr/core/src/test/org/apache/solr/search/facet/SpatialHeatmapFacetsTest.java @@ -155,9 +155,11 @@ public void testClassicFacets() throws Exception { // AKA SimpleFacets .findRecursive("facet_counts", "facet_heatmaps", "course", "gridLevel")); assertTrue( ((NamedList) - response.getResponse().findRecursive("facet_counts", "facet_heatmaps", "course")) - .asMap(0) - .containsKey("counts_" + courseFormat)); + response + .getResponse() + .findRecursive("facet_counts", "facet_heatmaps", "course")) + .indexOf("counts_" + courseFormat, 0) + >= 0); } // ------ Index data diff --git a/solr/core/src/test/org/apache/solr/search/mlt/CloudMLTQContentParserTest.java b/solr/core/src/test/org/apache/solr/search/mlt/CloudMLTQContentParserTest.java index ec8dcae907e..326ca7aaff0 100644 --- a/solr/core/src/test/org/apache/solr/search/mlt/CloudMLTQContentParserTest.java +++ b/solr/core/src/test/org/apache/solr/search/mlt/CloudMLTQContentParserTest.java @@ -177,8 +177,7 @@ public void testMinDF() throws Exception { new String[] {"lowerfilt_u:bmw lowerfilt_u:usa", "lowerfilt_u:usa lowerfilt_u:bmw"}; String[] actualParsedQueries; - if (queryResponse.getDebugMap().get("parsedquery") instanceof String) { - String parsedQueryString = (String) queryResponse.getDebugMap().get("parsedquery"); + if (queryResponse.getDebugMap().get("parsedquery") instanceof String parsedQueryString) { assertTrue( parsedQueryString.equals(expectedQueryStrings[0]) || parsedQueryString.equals(expectedQueryStrings[1])); diff --git a/solr/core/src/test/org/apache/solr/search/mlt/CloudMLTQParserTest.java b/solr/core/src/test/org/apache/solr/search/mlt/CloudMLTQParserTest.java index 5912465f883..0ccf8862969 100644 --- a/solr/core/src/test/org/apache/solr/search/mlt/CloudMLTQParserTest.java +++ b/solr/core/src/test/org/apache/solr/search/mlt/CloudMLTQParserTest.java @@ -239,8 +239,7 @@ public void testMinDF() throws Exception { }; String[] actualParsedQueries; - if (queryResponse.getDebugMap().get("parsedquery") instanceof String) { - String parsedQueryString = (String) queryResponse.getDebugMap().get("parsedquery"); + if (queryResponse.getDebugMap().get("parsedquery") instanceof String parsedQueryString) { assertTrue( parsedQueryString.equals(expectedQueryStrings[0]) || parsedQueryString.equals(expectedQueryStrings[1])); diff --git a/solr/core/src/test/org/apache/solr/security/AllowListUrlCheckerTest.java b/solr/core/src/test/org/apache/solr/security/AllowListUrlCheckerTest.java index b32c2124c15..0a4f57ba5af 100644 --- a/solr/core/src/test/org/apache/solr/security/AllowListUrlCheckerTest.java +++ b/solr/core/src/test/org/apache/solr/security/AllowListUrlCheckerTest.java @@ -24,11 +24,14 @@ import java.net.MalformedURLException; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ClusterState; import org.junit.Test; /** Tests {@link AllowListUrlChecker}. */ @@ -196,6 +199,51 @@ public void testHostParsingNoProtocol() throws Exception { equalTo(AllowListUrlChecker.parseHostPorts(urls("https://abc-1.com:8983/solr")))); } + @Test + public void testLiveNodesToHostUrlCache() throws Exception { + // Given some live nodes defined in the cluster state. + Set liveNodes = Set.of("1.2.3.4:8983_solr", "1.2.3.4:9000_", "1.2.3.4:9001_solr-2"); + ClusterState clusterState1 = new ClusterState(liveNodes, new HashMap<>()); + + // When we call the AllowListUrlChecker.checkAllowList method on both valid and invalid urls. + AtomicInteger callCount = new AtomicInteger(); + AllowListUrlChecker checker = + new AllowListUrlChecker(List.of()) { + @Override + Set buildLiveHostUrls(Set liveNodes) { + callCount.incrementAndGet(); + return super.buildLiveHostUrls(liveNodes); + } + }; + for (int i = 0; i < 3; i++) { + checker.checkAllowList( + List.of("1.2.3.4:8983", "1.2.3.4:9000", "1.2.3.4:9001"), clusterState1); + SolrException exception = + expectThrows( + SolrException.class, + () -> checker.checkAllowList(List.of("1.1.3.4:8983"), clusterState1)); + assertThat(exception.code(), equalTo(SolrException.ErrorCode.FORBIDDEN.code)); + } + // Then we verify that the AllowListUrlChecker caches the live host urls and only builds them + // once. + assertThat(callCount.get(), equalTo(1)); + + // And when the ClusterState live nodes change. + liveNodes = Set.of("2.3.4.5:8983_solr", "2.3.4.5:9000_", "2.3.4.5:9001_solr-2"); + ClusterState clusterState2 = new ClusterState(liveNodes, new HashMap<>()); + for (int i = 0; i < 3; i++) { + checker.checkAllowList( + List.of("2.3.4.5:8983", "2.3.4.5:9000", "2.3.4.5:9001"), clusterState2); + SolrException exception = + expectThrows( + SolrException.class, + () -> checker.checkAllowList(List.of("1.1.3.4:8983"), clusterState2)); + assertThat(exception.code(), equalTo(SolrException.ErrorCode.FORBIDDEN.code)); + } + // Then the AllowListUrlChecker rebuilds the cache of live host urls. + assertThat(callCount.get(), equalTo(2)); + } + private static List urls(String... urls) { return Arrays.asList(urls); } diff --git a/solr/core/src/test/org/apache/solr/security/BaseTestRuleBasedAuthorizationPlugin.java b/solr/core/src/test/org/apache/solr/security/BaseTestRuleBasedAuthorizationPlugin.java index 04d4f7e3cba..196120ca628 100644 --- a/solr/core/src/test/org/apache/solr/security/BaseTestRuleBasedAuthorizationPlugin.java +++ b/solr/core/src/test/org/apache/solr/security/BaseTestRuleBasedAuthorizationPlugin.java @@ -22,9 +22,6 @@ import static java.util.Collections.singletonMap; import static org.apache.solr.common.util.CommandOperation.captureErrors; import static org.apache.solr.common.util.Utils.getObjectByPath; -import static org.hamcrest.CoreMatchers.instanceOf; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assume.assumeThat; import java.io.IOException; import java.io.StringReader; @@ -676,33 +673,6 @@ public void testAllPermissionDeniesActionsWhenUserIsNotCorrectRole() { FORBIDDEN); } - @Test - public void testShortNameResolvesPermissions() { - assumeThat( - "ExternalRBAPlugin doesn't use short name", - createPlugin(), - is(instanceOf(RuleBasedAuthorizationPlugin.class))); - - setUserRole("admin", "admin"); - addPermission("all", "admin"); - - Map values = - Map.of( - "userPrincipal", "admin@EXAMPLE", - "userName", "admin", - "resource", "/admin/info/properties", - "requestType", RequestType.ADMIN, - "handler", new PropertiesRequestHandler()); - - // Short names disabled, admin should fail, admin@EXAMPLE should succeed - rules.put("useShortName", "false"); - checkRules(values, FORBIDDEN); - - // Short names enabled, admin should succeed, admin@EXAMPLE should fail - rules.put("useShortName", "true"); - checkRules(values, STATUS_OK); - } - @Test public void testGetPermissionNamesForRoles() { // Tests the method that maps role(s) to permissions, used by SystemInfoHandler to provide UI diff --git a/solr/core/src/test/org/apache/solr/security/PrincipalWithUserRoles.java b/solr/core/src/test/org/apache/solr/security/PrincipalWithUserRoles.java index b07b8e5cee3..987b0a4deb6 100644 --- a/solr/core/src/test/org/apache/solr/security/PrincipalWithUserRoles.java +++ b/solr/core/src/test/org/apache/solr/security/PrincipalWithUserRoles.java @@ -65,9 +65,8 @@ public Set getVerifiedRoles() { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof PrincipalWithUserRoles)) return false; + if (!(o instanceof PrincipalWithUserRoles that)) return false; - PrincipalWithUserRoles that = (PrincipalWithUserRoles) o; if (!username.equals(that.username)) return false; return roles.equals(that.roles); } diff --git a/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java b/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java index e86e6c12c4b..f2634411e90 100644 --- a/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java +++ b/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java @@ -162,7 +162,7 @@ public void testStreamURL() throws Exception { @Test @SuppressWarnings({"try"}) public void testStreamFile() throws Exception { - Path file = getFile("README").toPath(); + Path file = getFile("README"); byte[] bytes = Files.readAllBytes(file); diff --git a/solr/core/src/test/org/apache/solr/update/DirectUpdateHandlerWithUpdateLogTest.java b/solr/core/src/test/org/apache/solr/update/DirectUpdateHandlerWithUpdateLogTest.java new file mode 100644 index 00000000000..49a3be19697 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/update/DirectUpdateHandlerWithUpdateLogTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.update; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.lucene.index.IndexWriter; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.util.TimeSource; +import org.apache.solr.core.SolrCore; +import org.apache.solr.util.LogLevel; +import org.apache.solr.util.TimeOut; +import org.junit.BeforeClass; +import org.junit.Test; + +/** Tests {@link DirectUpdateHandler2} with update log enabled. */ +@LogLevel("org.apache.solr.update=INFO") +public class DirectUpdateHandlerWithUpdateLogTest extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + System.setProperty("solr.updateHandler", SpyingUpdateHandler.class.getName()); + initCore("solrconfig.xml", "schema11.xml"); + } + + @Test + public void testShouldCommitHook() throws Exception { + // Given a core. + SolrCore core = h.getCore(); + assertNotNull(core); + SpyingUpdateHandler updater = (SpyingUpdateHandler) core.getUpdateHandler(); + updater.shouldCommitCallCount.set(0); + + // When we add a doc and commit. + assertU(adoc("id", "1")); + assertU(commit()); + // Then the shouldCommit hook is called. + assertEquals(1, updater.shouldCommitCallCount.get()); + + // When we add a doc and soft commit. + assertU(adoc("id", "2")); + assertU(commit("softCommit", "true")); + // Then the shouldCommit hook is not called. + assertEquals(1, updater.shouldCommitCallCount.get()); + // And when we commit. + assertU(commit()); + // Then the shouldCommit hook is called. + assertEquals(2, updater.shouldCommitCallCount.get()); + + // When we commit with no updates (empty commit). + assertU(commit()); + // Then the shouldCommit hook is called (may commit only user metadata). + assertEquals(3, updater.shouldCommitCallCount.get()); + + // When we add a doc, do not commit, and close the IndexWriter. + assertU(adoc("id", "3")); + h.close(); + // Then the shouldCommit hook is called. + new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME) + .waitFor( + "Timeout waiting for should commit hook", + () -> updater.shouldCommitCallCount.get() == 4); + } + + public static class SpyingUpdateHandler extends DirectUpdateHandler2 { + + final AtomicInteger shouldCommitCallCount = new AtomicInteger(); + + public SpyingUpdateHandler(SolrCore core) { + super(core); + } + + public SpyingUpdateHandler(SolrCore core, UpdateHandler updateHandler) { + super(core, updateHandler); + } + + @Override + protected boolean shouldCommit(CommitUpdateCommand cmd, IndexWriter writer) throws IOException { + shouldCommitCallCount.incrementAndGet(); + return super.shouldCommit(cmd, writer); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesStandalone.java b/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesStandalone.java index 6c26ff47a88..42d3d41c121 100644 --- a/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesStandalone.java +++ b/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesStandalone.java @@ -1256,10 +1256,9 @@ public void checkReplay(final String valField, Object... commands) throws Except final long version = addAndGetVersion(sdoc, null); final Object val = sdoc.getFieldValue(valField); - if (val instanceof Map) { + if (val instanceof Map atomicUpdate) { // atomic update of the field we're modeling - Map atomicUpdate = (Map) val; assertEquals(sdoc.toString(), 1, atomicUpdate.size()); if (atomicUpdate.containsKey("inc")) { // Solr treats inc on a non-existing doc (or doc w/o existing value) as if existing diff --git a/solr/core/src/test/org/apache/solr/util/FileUtilsTest.java b/solr/core/src/test/org/apache/solr/util/FileUtilsTest.java index 671157c9e72..e6e96380026 100644 --- a/solr/core/src/test/org/apache/solr/util/FileUtilsTest.java +++ b/solr/core/src/test/org/apache/solr/util/FileUtilsTest.java @@ -17,15 +17,56 @@ package org.apache.solr.util; import java.io.File; +import java.nio.file.Path; import org.apache.solr.SolrTestCase; +import org.junit.Test; public class FileUtilsTest extends SolrTestCase { - public void testResolve() { - String cwd = new File(".").getAbsolutePath(); - assertEquals(new File("conf/data"), FileUtils.resolvePath(new File("conf"), "data")); - assertEquals( - new File(cwd + "/conf/data"), FileUtils.resolvePath(new File(cwd + "/conf"), "data")); - assertEquals(new File(cwd + "/data"), FileUtils.resolvePath(new File("conf"), cwd + "/data")); + @Test + public void testDetectsPathEscape() { + final var parent = Path.of("."); + + // Allows simple child + assertTrue(FileUtils.isPathAChildOfParent(parent, parent.resolve("child"))); + + // Allows "./" prefixed child + assertTrue(FileUtils.isPathAChildOfParent(parent, parent.resolve(buildPath(".", "child")))); + + // Allows nested child + assertTrue( + FileUtils.isPathAChildOfParent(parent, parent.resolve(buildPath("nested", "child")))); + + // Allows backtracking, provided it stays "under" parent + assertTrue( + FileUtils.isPathAChildOfParent( + parent, parent.resolve(buildPath("child1", "..", "child2")))); + assertTrue( + FileUtils.isPathAChildOfParent( + parent, parent.resolve(buildPath("child", "grandchild1", "..", "grandchild2")))); + + // Prevents identical path + assertFalse(FileUtils.isPathAChildOfParent(parent, parent)); + + // Detects sibling of parent + assertFalse(FileUtils.isPathAChildOfParent(parent, parent.resolve(buildPath("..", "sibling")))); + + // Detects "grandparent" of parent + assertFalse(FileUtils.isPathAChildOfParent(parent, parent.resolve(".."))); + + // Detects many-layered backtracking + assertFalse( + FileUtils.isPathAChildOfParent(parent, parent.resolve(buildPath("..", "..", "..", "..")))); + } + + private static String buildPath(String... pathSegments) { + final var sb = new StringBuilder(); + for (int i = 0; i < pathSegments.length; i++) { + sb.append(pathSegments[i]); + if (i < pathSegments.length - 1) { + sb.append(File.separator); + } + } + return sb.toString(); } } diff --git a/solr/core/src/test/org/apache/solr/util/TestSystemIdResolver.java b/solr/core/src/test/org/apache/solr/util/TestSystemIdResolver.java index 71ba80b4bd1..8aea4ba8178 100644 --- a/solr/core/src/test/org/apache/solr/util/TestSystemIdResolver.java +++ b/solr/core/src/test/org/apache/solr/util/TestSystemIdResolver.java @@ -46,7 +46,7 @@ private void assertEntityResolving( } public void testResolving() throws Exception { - final Path testHome = SolrTestCaseJ4.getFile("solr/collection1").getParentFile().toPath(); + final Path testHome = SolrTestCaseJ4.getFile("solr/collection1").getParent(); final ResourceLoader loader = new SolrResourceLoader(testHome.resolve("collection1"), this.getClass().getClassLoader()); final SystemIdResolver resolver = new SystemIdResolver(loader); @@ -131,7 +131,7 @@ public void testResolving() throws Exception { public void testUnsafeResolving() throws Exception { System.setProperty(SolrResourceLoader.SOLR_ALLOW_UNSAFE_RESOURCELOADING_PARAM, "true"); - final Path testHome = SolrTestCaseJ4.getFile("solr/collection1").getParentFile().toPath(); + final Path testHome = SolrTestCaseJ4.getFile("solr/collection1").getParent(); final ResourceLoader loader = new SolrResourceLoader(testHome.resolve("collection1"), this.getClass().getClassLoader()); final SystemIdResolver resolver = new SystemIdResolver(loader); diff --git a/solr/cross-dc-manager/build.gradle b/solr/cross-dc-manager/build.gradle index b6cc162525e..1cf2b575b65 100644 --- a/solr/cross-dc-manager/build.gradle +++ b/solr/cross-dc-manager/build.gradle @@ -27,33 +27,49 @@ dependencies { implementation project(':solr:solrj-zookeeper') implementation project(':solr:modules:cross-dc') - implementation 'io.dropwizard.metrics:metrics-core' - implementation 'io.dropwizard.metrics:metrics-servlets' - implementation 'org.eclipse.jetty:jetty-server' - implementation 'org.eclipse.jetty:jetty-servlet' - implementation 'org.slf4j:slf4j-api' - runtimeOnly ('com.google.protobuf:protobuf-java-util') - runtimeOnly ('commons-codec:commons-codec') + implementation libs.dropwizard.metrics.core + implementation libs.dropwizard.metrics.servlets + implementation libs.eclipse.jetty.server + implementation libs.eclipse.jetty.servlet + implementation libs.slf4j.api + runtimeOnly libs.google.protobuf.javautils + runtimeOnly libs.commonscodec.commonscodec - implementation 'org.apache.kafka:kafka-clients' - runtimeOnly 'org.apache.kafka:kafka_2.13' - runtimeOnly 'org.apache.kafka:kafka-streams' + implementation libs.apache.kafka.clients + runtimeOnly libs.apache.kafka.kafka213 + runtimeOnly libs.apache.kafka.streams - runtimeOnly 'org.apache.logging.log4j:log4j-api' - runtimeOnly 'org.apache.logging.log4j:log4j-core' - runtimeOnly 'org.apache.logging.log4j:log4j-slf4j2-impl' - runtimeOnly 'com.lmax:disruptor' + runtimeOnly libs.apache.log4j.api + runtimeOnly libs.apache.log4j.core + runtimeOnly libs.apache.log4j.slf4j2impl + runtimeOnly libs.lmax.disruptor testImplementation project(':solr:test-framework') - testImplementation 'org.apache.lucene:lucene-test-framework' - testImplementation 'com.carrotsearch.randomizedtesting:randomizedtesting-runner' - testImplementation 'junit:junit' - testImplementation 'org.mockito:mockito-core' + testImplementation libs.apache.lucene.testframework + testImplementation libs.carrotsearch.randomizedtesting.runner + testImplementation libs.junit.junit + testImplementation libs.mockito.core - testImplementation 'org.apache.kafka:kafka-streams:3.7.1:test' - testImplementation 'org.apache.kafka:kafka-clients:3.7.1:test' - testRuntimeOnly 'org.apache.kafka:kafka_2.13:3.7.1:test' - testRuntimeOnly 'org.apache.kafka:kafka-server-common:3.7.1:test' + testImplementation(libs.apache.kafka.streams) { + artifact { + classifier = "test" + } + } + testImplementation(libs.apache.kafka.clients) { + artifact { + classifier = "test" + } + } + testRuntimeOnly(libs.apache.kafka.kafka213) { + artifact { + classifier = "test" + } + } + testRuntimeOnly(libs.apache.kafka.server.common) { + artifact { + classifier = "test" + } + } } ext { diff --git a/solr/cross-dc-manager/src/java/org/apache/solr/crossdc/manager/messageprocessor/SolrMessageProcessor.java b/solr/cross-dc-manager/src/java/org/apache/solr/crossdc/manager/messageprocessor/SolrMessageProcessor.java index b4e14feea78..2f4e090bab4 100644 --- a/solr/cross-dc-manager/src/java/org/apache/solr/crossdc/manager/messageprocessor/SolrMessageProcessor.java +++ b/solr/cross-dc-manager/src/java/org/apache/solr/crossdc/manager/messageprocessor/SolrMessageProcessor.java @@ -254,9 +254,8 @@ private void logRequest(SolrRequest request) { * @param request The SolrRequest to be cleaned up for submitting locally. */ private void prepareIfUpdateRequest(SolrRequest request) { - if (request instanceof UpdateRequest) { + if (request instanceof UpdateRequest updateRequest) { // Remove versions from add requests - UpdateRequest updateRequest = (UpdateRequest) request; List documents = updateRequest.getDocuments(); if (log.isTraceEnabled()) { @@ -318,8 +317,7 @@ private void logFirstAttemptLatency(MirroredSolrRequest mirroredSolrRequest) * @param mirroredSolrRequest MirroredSolrRequest object that is being processed. */ void preventCircularMirroring(MirroredSolrRequest mirroredSolrRequest) { - if (mirroredSolrRequest.getSolrRequest() instanceof UpdateRequest) { - UpdateRequest updateRequest = (UpdateRequest) mirroredSolrRequest.getSolrRequest(); + if (mirroredSolrRequest.getSolrRequest() instanceof UpdateRequest updateRequest) { ModifiableSolrParams params = updateRequest.getParams(); String shouldMirror = (params == null ? null : params.get(CrossDcConstants.SHOULD_MIRROR)); if (shouldMirror == null) { diff --git a/solr/cross-dc-manager/src/test/org/apache/solr/crossdc/manager/DeleteByQueryToIdTest.java b/solr/cross-dc-manager/src/test/org/apache/solr/crossdc/manager/DeleteByQueryToIdTest.java index ff13f34e301..475b9f0b73f 100644 --- a/solr/cross-dc-manager/src/test/org/apache/solr/crossdc/manager/DeleteByQueryToIdTest.java +++ b/solr/cross-dc-manager/src/test/org/apache/solr/crossdc/manager/DeleteByQueryToIdTest.java @@ -108,9 +108,8 @@ public String bootstrapServers() { solrCluster1 = SolrCloudTestCase.configureCluster(1) - .addConfig("conf", getFile("configs/cloud-minimal/conf").toPath().toAbsolutePath()) - .addConfig( - "confNoDbq", getFile("configs/cloud-minimal-no-dbq/conf").toPath().toAbsolutePath()) + .addConfig("conf", getFile("configs/cloud-minimal/conf").toAbsolutePath()) + .addConfig("confNoDbq", getFile("configs/cloud-minimal-no-dbq/conf").toAbsolutePath()) .configure(); props.setProperty("solr.crossdc.topicName", TOPIC); @@ -132,8 +131,8 @@ public String bootstrapServers() { solrCluster2 = SolrCloudTestCase.configureCluster(1) - .addConfig("conf", getFile("configs/cloud-minimal/conf").toPath()) - .addConfig("confNoDbq", getFile("configs/cloud-minimal-no-dbq/conf").toPath()) + .addConfig("conf", getFile("configs/cloud-minimal/conf")) + .addConfig("confNoDbq", getFile("configs/cloud-minimal-no-dbq/conf")) .configure(); solrCluster2.getZkClient().makePath("/crossdc.properties", data, true); diff --git a/solr/cross-dc-manager/src/test/org/apache/solr/crossdc/manager/RetryQueueIntegrationTest.java b/solr/cross-dc-manager/src/test/org/apache/solr/crossdc/manager/RetryQueueIntegrationTest.java index 147a7673df7..c654fc99187 100644 --- a/solr/cross-dc-manager/src/test/org/apache/solr/crossdc/manager/RetryQueueIntegrationTest.java +++ b/solr/cross-dc-manager/src/test/org/apache/solr/crossdc/manager/RetryQueueIntegrationTest.java @@ -157,7 +157,7 @@ private static MiniSolrCloudCluster startCluster( // new SolrCloudTestCase.Builder(1, baseDir).addConfig("conf", // getFile("configs/cloud-minimal/conf").toPath()).configure(); - cluster.uploadConfigSet(getFile("configs/cloud-minimal/conf").toPath(), "conf"); + cluster.uploadConfigSet(getFile("configs/cloud-minimal/conf"), "conf"); CollectionAdminRequest.Create create2 = CollectionAdminRequest.createCollection(COLLECTION, "conf", 1, 1); diff --git a/solr/cross-dc-manager/src/test/org/apache/solr/crossdc/manager/SolrAndKafkaIntegrationTest.java b/solr/cross-dc-manager/src/test/org/apache/solr/crossdc/manager/SolrAndKafkaIntegrationTest.java index af65035258e..65fe1aeb55b 100644 --- a/solr/cross-dc-manager/src/test/org/apache/solr/crossdc/manager/SolrAndKafkaIntegrationTest.java +++ b/solr/cross-dc-manager/src/test/org/apache/solr/crossdc/manager/SolrAndKafkaIntegrationTest.java @@ -121,9 +121,7 @@ public String bootstrapServers() { System.setProperty(INDEX_UNMIRRORABLE_DOCS, "false"); solrCluster1 = - configureCluster(1) - .addConfig("conf", getFile("configs/cloud-minimal/conf").toPath()) - .configure(); + configureCluster(1).addConfig("conf", getFile("configs/cloud-minimal/conf")).configure(); CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(COLLECTION, "conf", 1, 1); @@ -131,9 +129,7 @@ public String bootstrapServers() { solrCluster1.waitForActiveCollection(COLLECTION, 1, 1); solrCluster2 = - configureCluster(1) - .addConfig("conf", getFile("configs/cloud-minimal/conf").toPath()) - .configure(); + configureCluster(1).addConfig("conf", getFile("configs/cloud-minimal/conf")).configure(); CollectionAdminRequest.Create create2 = CollectionAdminRequest.createCollection(COLLECTION, "conf", 1, 1); diff --git a/solr/cross-dc-manager/src/test/org/apache/solr/crossdc/manager/SolrAndKafkaMultiCollectionIntegrationTest.java b/solr/cross-dc-manager/src/test/org/apache/solr/crossdc/manager/SolrAndKafkaMultiCollectionIntegrationTest.java index 26ecfd34e25..de942eb980e 100644 --- a/solr/cross-dc-manager/src/test/org/apache/solr/crossdc/manager/SolrAndKafkaMultiCollectionIntegrationTest.java +++ b/solr/cross-dc-manager/src/test/org/apache/solr/crossdc/manager/SolrAndKafkaMultiCollectionIntegrationTest.java @@ -117,9 +117,7 @@ public String bootstrapServers() { System.setProperty(INDEX_UNMIRRORABLE_DOCS, "false"); solrCluster1 = - configureCluster(1) - .addConfig("conf", getFile("configs/cloud-minimal/conf").toPath()) - .configure(); + configureCluster(1).addConfig("conf", getFile("configs/cloud-minimal/conf")).configure(); CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(COLLECTION, "conf", 1, 1); @@ -127,9 +125,7 @@ public String bootstrapServers() { solrCluster1.waitForActiveCollection(COLLECTION, 1, 1); solrCluster2 = - configureCluster(1) - .addConfig("conf", getFile("configs/cloud-minimal/conf").toPath()) - .configure(); + configureCluster(1).addConfig("conf", getFile("configs/cloud-minimal/conf")).configure(); CollectionAdminRequest.Create create2 = CollectionAdminRequest.createCollection(COLLECTION, "conf", 1, 1); diff --git a/solr/cross-dc-manager/src/test/org/apache/solr/crossdc/manager/SolrAndKafkaReindexTest.java b/solr/cross-dc-manager/src/test/org/apache/solr/crossdc/manager/SolrAndKafkaReindexTest.java index 5aa7c1ddf51..7a955fb6519 100644 --- a/solr/cross-dc-manager/src/test/org/apache/solr/crossdc/manager/SolrAndKafkaReindexTest.java +++ b/solr/cross-dc-manager/src/test/org/apache/solr/crossdc/manager/SolrAndKafkaReindexTest.java @@ -98,9 +98,7 @@ public String bootstrapServers() { System.setProperty("solr.crossdc.bootstrapServers", kafkaCluster.bootstrapServers()); solrCluster1 = - configureCluster(3) - .addConfig("conf", getFile("configs/cloud-minimal/conf").toPath()) - .configure(); + configureCluster(3).addConfig("conf", getFile("configs/cloud-minimal/conf")).configure(); CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(COLLECTION, "conf", 3, 2); @@ -108,9 +106,7 @@ public String bootstrapServers() { solrCluster1.waitForActiveCollection(COLLECTION, 3, 6); solrCluster2 = - configureCluster(3) - .addConfig("conf", getFile("configs/cloud-minimal/conf").toPath()) - .configure(); + configureCluster(3).addConfig("conf", getFile("configs/cloud-minimal/conf")).configure(); CollectionAdminRequest.Create create2 = CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 3); diff --git a/solr/cross-dc-manager/src/test/org/apache/solr/crossdc/manager/ZkConfigIntegrationTest.java b/solr/cross-dc-manager/src/test/org/apache/solr/crossdc/manager/ZkConfigIntegrationTest.java index f926e9178ea..a3c0f2b9626 100644 --- a/solr/cross-dc-manager/src/test/org/apache/solr/crossdc/manager/ZkConfigIntegrationTest.java +++ b/solr/cross-dc-manager/src/test/org/apache/solr/crossdc/manager/ZkConfigIntegrationTest.java @@ -102,9 +102,7 @@ public String bootstrapServers() { Properties props = new Properties(); solrCluster1 = - configureCluster(1) - .addConfig("conf", getFile("configs/cloud-minimal/conf").toPath()) - .configure(); + configureCluster(1).addConfig("conf", getFile("configs/cloud-minimal/conf")).configure(); props.setProperty(KafkaCrossDcConf.TOPIC_NAME, TOPIC2); props.setProperty(KafkaCrossDcConf.BOOTSTRAP_SERVERS, kafkaCluster.bootstrapServers()); @@ -123,9 +121,7 @@ public String bootstrapServers() { solrCluster1.waitForActiveCollection(COLLECTION, 1, 1); solrCluster2 = - configureCluster(1) - .addConfig("conf", getFile("configs/cloud-minimal/conf").toPath()) - .configure(); + configureCluster(1).addConfig("conf", getFile("configs/cloud-minimal/conf")).configure(); CollectionAdminRequest.Create create2 = CollectionAdminRequest.createCollection(COLLECTION, "conf", 1, 1); diff --git a/solr/docker/build.gradle b/solr/docker/build.gradle index cf360908bb2..6c3417504d8 100644 --- a/solr/docker/build.gradle +++ b/solr/docker/build.gradle @@ -22,7 +22,7 @@ import org.apache.commons.codec.digest.DigestUtils description = 'Solr Docker image' apply plugin: 'base' -def javaVersion = scriptDepVersions['min-java-version'] +def javaVersion = libs.versions.java.min.get() // Solr Docker inputs def dockerImageSolrDist = "${ -> propertyOrEnvOrDefault("solr.docker.dist", "SOLR_DOCKER_DIST", 'full') }" def isDistSlim = {String dist -> dist.toLowerCase(Locale.ROOT) == "slim"} @@ -127,7 +127,7 @@ buildscript { mavenCentral() } dependencies { - classpath "commons-codec:commons-codec:${scriptDepVersions['commons-codec']}" + classpath libs.commonscodec.commonscodec } } def checksum = { file -> diff --git a/solr/example/README.md b/solr/example/README.md index 09bb2dfb277..f0c8e46a4be 100644 --- a/solr/example/README.md +++ b/solr/example/README.md @@ -73,18 +73,6 @@ For a list of other tutorials and introductory articles. Notes About These Examples -------------------------- -### References to Jar Files Outside This Directory - -Various example SolrHome dirs contained in this directory may use "" -statements in the solrconfig.xml file to reference plugin jars outside of -this directory for loading modules via relative paths. - -If you make a copy of this example server and wish to use the -ExtractingRequestHandler (SolrCell), the clustering component, -or any other modules, you will need to -copy the required jars or update the paths to those jars in your -solrconfig.xml. - ### Logging By default, Jetty & Solr will log to the console and logs/solr.log. This can diff --git a/solr/licenses/commons-configuration2-2.11.0.jar.sha1 b/solr/licenses/commons-configuration2-2.11.0.jar.sha1 deleted file mode 100644 index fae01fd879c..00000000000 --- a/solr/licenses/commons-configuration2-2.11.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -af5a2c6abe587074c0be1107fcb27fa2fad91304 diff --git a/solr/licenses/commons-configuration2-LICENSE-ASL.txt b/solr/licenses/commons-configuration2-LICENSE-ASL.txt deleted file mode 100644 index dd726f2eaf1..00000000000 --- a/solr/licenses/commons-configuration2-LICENSE-ASL.txt +++ /dev/null @@ -1,403 +0,0 @@ - - - Apache License - - Version 2.0, January 2004 - - http://www.apache.org/licenses/ - - - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - - - 1. Definitions. - - - - "License" shall mean the terms and conditions for use, reproduction, - - and distribution as defined by Sections 1 through 9 of this document. - - - - "Licensor" shall mean the copyright owner or entity authorized by - - the copyright owner that is granting the License. - - - - "Legal Entity" shall mean the union of the acting entity and all - - other entities that control, are controlled by, or are under common - - control with that entity. For the purposes of this definition, - - "control" means (i) the power, direct or indirect, to cause the - - direction or management of such entity, whether by contract or - - otherwise, or (ii) ownership of fifty percent (50%) or more of the - - outstanding shares, or (iii) beneficial ownership of such entity. - - - - "You" (or "Your") shall mean an individual or Legal Entity - - exercising permissions granted by this License. - - - - "Source" form shall mean the preferred form for making modifications, - - including but not limited to software source code, documentation - - source, and configuration files. - - - - "Object" form shall mean any form resulting from mechanical - - transformation or translation of a Source form, including but - - not limited to compiled object code, generated documentation, - - and conversions to other media types. - - - - "Work" shall mean the work of authorship, whether in Source or - - Object form, made available under the License, as indicated by a - - copyright notice that is included in or attached to the work - - (an example is provided in the Appendix below). - - - - "Derivative Works" shall mean any work, whether in Source or Object - - form, that is based on (or derived from) the Work and for which the - - editorial revisions, annotations, elaborations, or other modifications - - represent, as a whole, an original work of authorship. For the purposes - - of this License, Derivative Works shall not include works that remain - - separable from, or merely link (or bind by name) to the interfaces of, - - the Work and Derivative Works thereof. - - - - "Contribution" shall mean any work of authorship, including - - the original version of the Work and any modifications or additions - - to that Work or Derivative Works thereof, that is intentionally - - submitted to Licensor for inclusion in the Work by the copyright owner - - or by an individual or Legal Entity authorized to submit on behalf of - - the copyright owner. For the purposes of this definition, "submitted" - - means any form of electronic, verbal, or written communication sent - - to the Licensor or its representatives, including but not limited to - - communication on electronic mailing lists, source code control systems, - - and issue tracking systems that are managed by, or on behalf of, the - - Licensor for the purpose of discussing and improving the Work, but - - excluding communication that is conspicuously marked or otherwise - - designated in writing by the copyright owner as "Not a Contribution." - - - - "Contributor" shall mean Licensor and any individual or Legal Entity - - on behalf of whom a Contribution has been received by Licensor and - - subsequently incorporated within the Work. - - - - 2. Grant of Copyright License. Subject to the terms and conditions of - - this License, each Contributor hereby grants to You a perpetual, - - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - - copyright license to reproduce, prepare Derivative Works of, - - publicly display, publicly perform, sublicense, and distribute the - - Work and such Derivative Works in Source or Object form. - - - - 3. Grant of Patent License. Subject to the terms and conditions of - - this License, each Contributor hereby grants to You a perpetual, - - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - - (except as stated in this section) patent license to make, have made, - - use, offer to sell, sell, import, and otherwise transfer the Work, - - where such license applies only to those patent claims licensable - - by such Contributor that are necessarily infringed by their - - Contribution(s) alone or by combination of their Contribution(s) - - with the Work to which such Contribution(s) was submitted. If You - - institute patent litigation against any entity (including a - - cross-claim or counterclaim in a lawsuit) alleging that the Work - - or a Contribution incorporated within the Work constitutes direct - - or contributory patent infringement, then any patent licenses - - granted to You under this License for that Work shall terminate - - as of the date such litigation is filed. - - - - 4. Redistribution. You may reproduce and distribute copies of the - - Work or Derivative Works thereof in any medium, with or without - - modifications, and in Source or Object form, provided that You - - meet the following conditions: - - - - (a) You must give any other recipients of the Work or - - Derivative Works a copy of this License; and - - - - (b) You must cause any modified files to carry prominent notices - - stating that You changed the files; and - - - - (c) You must retain, in the Source form of any Derivative Works - - that You distribute, all copyright, patent, trademark, and - - attribution notices from the Source form of the Work, - - excluding those notices that do not pertain to any part of - - the Derivative Works; and - - - - (d) If the Work includes a "NOTICE" text file as part of its - - distribution, then any Derivative Works that You distribute must - - include a readable copy of the attribution notices contained - - within such NOTICE file, excluding those notices that do not - - pertain to any part of the Derivative Works, in at least one - - of the following places: within a NOTICE text file distributed - - as part of the Derivative Works; within the Source form or - - documentation, if provided along with the Derivative Works; or, - - within a display generated by the Derivative Works, if and - - wherever such third-party notices normally appear. The contents - - of the NOTICE file are for informational purposes only and - - do not modify the License. You may add Your own attribution - - notices within Derivative Works that You distribute, alongside - - or as an addendum to the NOTICE text from the Work, provided - - that such additional attribution notices cannot be construed - - as modifying the License. - - - - You may add Your own copyright statement to Your modifications and - - may provide additional or different license terms and conditions - - for use, reproduction, or distribution of Your modifications, or - - for any such Derivative Works as a whole, provided Your use, - - reproduction, and distribution of the Work otherwise complies with - - the conditions stated in this License. - - - - 5. Submission of Contributions. Unless You explicitly state otherwise, - - any Contribution intentionally submitted for inclusion in the Work - - by You to the Licensor shall be under the terms and conditions of - - this License, without any additional terms or conditions. - - Notwithstanding the above, nothing herein shall supersede or modify - - the terms of any separate license agreement you may have executed - - with Licensor regarding such Contributions. - - - - 6. Trademarks. This License does not grant permission to use the trade - - names, trademarks, service marks, or product names of the Licensor, - - except as required for reasonable and customary use in describing the - - origin of the Work and reproducing the content of the NOTICE file. - - - - 7. Disclaimer of Warranty. Unless required by applicable law or - - agreed to in writing, Licensor provides the Work (and each - - Contributor provides its Contributions) on an "AS IS" BASIS, - - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - - implied, including, without limitation, any warranties or conditions - - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - - PARTICULAR PURPOSE. You are solely responsible for determining the - - appropriateness of using or redistributing the Work and assume any - - risks associated with Your exercise of permissions under this License. - - - - 8. Limitation of Liability. In no event and under no legal theory, - - whether in tort (including negligence), contract, or otherwise, - - unless required by applicable law (such as deliberate and grossly - - negligent acts) or agreed to in writing, shall any Contributor be - - liable to You for damages, including any direct, indirect, special, - - incidental, or consequential damages of any character arising as a - - result of this License or out of the use or inability to use the - - Work (including but not limited to damages for loss of goodwill, - - work stoppage, computer failure or malfunction, or any and all - - other commercial damages or losses), even if such Contributor - - has been advised of the possibility of such damages. - - - - 9. Accepting Warranty or Additional Liability. While redistributing - - the Work or Derivative Works thereof, You may choose to offer, - - and charge a fee for, acceptance of support, warranty, indemnity, - - or other liability obligations and/or rights consistent with this - - License. However, in accepting such obligations, You may act only - - on Your own behalf and on Your sole responsibility, not on behalf - - of any other Contributor, and only if You agree to indemnify, - - defend, and hold each Contributor harmless for any liability - - incurred by, or claims asserted against, such Contributor by reason - - of your accepting any such warranty or additional liability. - - - - END OF TERMS AND CONDITIONS - - - - APPENDIX: How to apply the Apache License to your work. - - - - To apply the Apache License to your work, attach the following - - boilerplate notice, with the fields enclosed by brackets "[]" - - replaced with your own identifying information. (Don't include - - the brackets!) The text should be enclosed in the appropriate - - comment syntax for the file format. We also recommend that a - - file or class name and description of purpose be included on the - - same "printed page" as the copyright notice for easier - - identification within third-party archives. - - - - Copyright [yyyy] [name of copyright owner] - - - - Licensed under the Apache License, Version 2.0 (the "License"); - - you may not use this file except in compliance with the License. - - You may obtain a copy of the License at - - - - http://www.apache.org/licenses/LICENSE-2.0 - - - - Unless required by applicable law or agreed to in writing, software - - distributed under the License is distributed on an "AS IS" BASIS, - - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - - See the License for the specific language governing permissions and - - limitations under the License. \ No newline at end of file diff --git a/solr/licenses/commons-configuration2-NOTICE.txt b/solr/licenses/commons-configuration2-NOTICE.txt deleted file mode 100644 index 51e4285634f..00000000000 --- a/solr/licenses/commons-configuration2-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache Commons Configuration -Copyright 2001-2018 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). diff --git a/solr/licenses/converter-jackson-2.9.0.jar.sha1 b/solr/licenses/converter-jackson-2.9.0.jar.sha1 new file mode 100644 index 00000000000..20ab4394574 --- /dev/null +++ b/solr/licenses/converter-jackson-2.9.0.jar.sha1 @@ -0,0 +1 @@ +19b4010914e747601e26f46c6a403044bbe0b2bf diff --git a/solr/licenses/kerb-client-LICENSE-ASL.txt b/solr/licenses/converter-jackson-LICENSE-ASL.txt similarity index 99% rename from solr/licenses/kerb-client-LICENSE-ASL.txt rename to solr/licenses/converter-jackson-LICENSE-ASL.txt index ad410e11302..7a4a3ea2424 100644 --- a/solr/licenses/kerb-client-LICENSE-ASL.txt +++ b/solr/licenses/converter-jackson-LICENSE-ASL.txt @@ -1,4 +1,5 @@ -Apache License + + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -178,7 +179,7 @@ Apache License APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -186,7 +187,7 @@ Apache License same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/solr/licenses/converter-jackson-NOTICE.txt b/solr/licenses/converter-jackson-NOTICE.txt new file mode 100644 index 00000000000..25fdd5926ad --- /dev/null +++ b/solr/licenses/converter-jackson-NOTICE.txt @@ -0,0 +1,13 @@ +Copyright 2013 Square, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/solr/licenses/curator-recipes-5.7.1.jar.sha1 b/solr/licenses/curator-recipes-5.7.1.jar.sha1 deleted file mode 100644 index fc1cd77aa2c..00000000000 --- a/solr/licenses/curator-recipes-5.7.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -39f8871614f0ed157f87f412bed8fd7b32c457ac diff --git a/solr/licenses/error_prone_annotations-2.28.0.jar.sha1 b/solr/licenses/error_prone_annotations-2.28.0.jar.sha1 deleted file mode 100644 index 4839239eabf..00000000000 --- a/solr/licenses/error_prone_annotations-2.28.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -59fc00087ce372de42e394d2c789295dff2d19f0 diff --git a/solr/licenses/error_prone_annotations-2.31.0.jar.sha1 b/solr/licenses/error_prone_annotations-2.31.0.jar.sha1 new file mode 100644 index 00000000000..1fa88710c6d --- /dev/null +++ b/solr/licenses/error_prone_annotations-2.31.0.jar.sha1 @@ -0,0 +1 @@ +c3ba307b915d6d506e98ffbb49e6d2d12edad65b diff --git a/solr/licenses/hadoop-annotations-3.4.0.jar.sha1 b/solr/licenses/hadoop-annotations-3.4.0.jar.sha1 deleted file mode 100644 index 44880116ccb..00000000000 --- a/solr/licenses/hadoop-annotations-3.4.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -612302e6e385d7e6a62bc0129dbc58689f8c7b2b diff --git a/solr/licenses/hadoop-annotations-LICENSE-ASL.txt b/solr/licenses/hadoop-annotations-LICENSE-ASL.txt deleted file mode 100644 index 9a8e847ee84..00000000000 --- a/solr/licenses/hadoop-annotations-LICENSE-ASL.txt +++ /dev/null @@ -1,244 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -APACHE HADOOP SUBCOMPONENTS: - -The Apache Hadoop project contains subcomponents with separate copyright -notices and license terms. Your use of the source code for the these -subcomponents is subject to the terms and conditions of the following -licenses. - -For the org.apache.hadoop.util.bloom.* classes: - -/** - * - * Copyright (c) 2005, European Commission project OneLab under contract - * 034819 (http://www.one-lab.org) - * All rights reserved. - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the distribution. - * - Neither the name of the University Catholique de Louvain - UCL - * nor the names of its contributors may be used to endorse or - * promote products derived from this software without specific prior - * written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ \ No newline at end of file diff --git a/solr/licenses/hadoop-annotations-NOTICE.txt b/solr/licenses/hadoop-annotations-NOTICE.txt deleted file mode 100644 index c56a5e4eac1..00000000000 --- a/solr/licenses/hadoop-annotations-NOTICE.txt +++ /dev/null @@ -1,2 +0,0 @@ -This product includes software developed by The Apache Software -Foundation (http://www.apache.org/). \ No newline at end of file diff --git a/solr/licenses/hadoop-auth-3.4.0.jar.sha1 b/solr/licenses/hadoop-auth-3.4.0.jar.sha1 deleted file mode 100644 index 0933800fb77..00000000000 --- a/solr/licenses/hadoop-auth-3.4.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6f5ab3a46b3d2734c0ae4d6338f0be3529efe083 diff --git a/solr/licenses/hadoop-auth-LICENSE-ASL.txt b/solr/licenses/hadoop-auth-LICENSE-ASL.txt deleted file mode 100644 index 9a8e847ee84..00000000000 --- a/solr/licenses/hadoop-auth-LICENSE-ASL.txt +++ /dev/null @@ -1,244 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -APACHE HADOOP SUBCOMPONENTS: - -The Apache Hadoop project contains subcomponents with separate copyright -notices and license terms. Your use of the source code for the these -subcomponents is subject to the terms and conditions of the following -licenses. - -For the org.apache.hadoop.util.bloom.* classes: - -/** - * - * Copyright (c) 2005, European Commission project OneLab under contract - * 034819 (http://www.one-lab.org) - * All rights reserved. - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the distribution. - * - Neither the name of the University Catholique de Louvain - UCL - * nor the names of its contributors may be used to endorse or - * promote products derived from this software without specific prior - * written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ \ No newline at end of file diff --git a/solr/licenses/hadoop-auth-NOTICE.txt b/solr/licenses/hadoop-auth-NOTICE.txt deleted file mode 100644 index c56a5e4eac1..00000000000 --- a/solr/licenses/hadoop-auth-NOTICE.txt +++ /dev/null @@ -1,2 +0,0 @@ -This product includes software developed by The Apache Software -Foundation (http://www.apache.org/). \ No newline at end of file diff --git a/solr/licenses/hadoop-common-3.4.0.jar.sha1 b/solr/licenses/hadoop-common-3.4.0.jar.sha1 deleted file mode 100644 index 00248c5160a..00000000000 --- a/solr/licenses/hadoop-common-3.4.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2b4b1694b695b31cdd8e345b6b59fd74d43a26bc diff --git a/solr/licenses/hadoop-common-LICENSE-ASL.txt b/solr/licenses/hadoop-common-LICENSE-ASL.txt deleted file mode 100644 index 9a8e847ee84..00000000000 --- a/solr/licenses/hadoop-common-LICENSE-ASL.txt +++ /dev/null @@ -1,244 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -APACHE HADOOP SUBCOMPONENTS: - -The Apache Hadoop project contains subcomponents with separate copyright -notices and license terms. Your use of the source code for the these -subcomponents is subject to the terms and conditions of the following -licenses. - -For the org.apache.hadoop.util.bloom.* classes: - -/** - * - * Copyright (c) 2005, European Commission project OneLab under contract - * 034819 (http://www.one-lab.org) - * All rights reserved. - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the distribution. - * - Neither the name of the University Catholique de Louvain - UCL - * nor the names of its contributors may be used to endorse or - * promote products derived from this software without specific prior - * written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ \ No newline at end of file diff --git a/solr/licenses/hadoop-common-NOTICE.txt b/solr/licenses/hadoop-common-NOTICE.txt deleted file mode 100644 index c56a5e4eac1..00000000000 --- a/solr/licenses/hadoop-common-NOTICE.txt +++ /dev/null @@ -1,2 +0,0 @@ -This product includes software developed by The Apache Software -Foundation (http://www.apache.org/). \ No newline at end of file diff --git a/solr/licenses/hadoop-minikdc-3.4.0.jar.sha1 b/solr/licenses/hadoop-minikdc-3.4.0.jar.sha1 deleted file mode 100644 index 3bb86c66079..00000000000 --- a/solr/licenses/hadoop-minikdc-3.4.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ec7c619e1deff5a9f2da0293fc9f5fcae66d159e diff --git a/solr/licenses/hadoop-minikdc-LICENSE-ASL.txt b/solr/licenses/hadoop-minikdc-LICENSE-ASL.txt deleted file mode 100644 index 9a8e847ee84..00000000000 --- a/solr/licenses/hadoop-minikdc-LICENSE-ASL.txt +++ /dev/null @@ -1,244 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -APACHE HADOOP SUBCOMPONENTS: - -The Apache Hadoop project contains subcomponents with separate copyright -notices and license terms. Your use of the source code for the these -subcomponents is subject to the terms and conditions of the following -licenses. - -For the org.apache.hadoop.util.bloom.* classes: - -/** - * - * Copyright (c) 2005, European Commission project OneLab under contract - * 034819 (http://www.one-lab.org) - * All rights reserved. - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the distribution. - * - Neither the name of the University Catholique de Louvain - UCL - * nor the names of its contributors may be used to endorse or - * promote products derived from this software without specific prior - * written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ \ No newline at end of file diff --git a/solr/licenses/hadoop-minikdc-NOTICE.txt b/solr/licenses/hadoop-minikdc-NOTICE.txt deleted file mode 100644 index c56a5e4eac1..00000000000 --- a/solr/licenses/hadoop-minikdc-NOTICE.txt +++ /dev/null @@ -1,2 +0,0 @@ -This product includes software developed by The Apache Software -Foundation (http://www.apache.org/). \ No newline at end of file diff --git a/solr/licenses/hamcrest-core-3.0.jar.sha1 b/solr/licenses/hamcrest-core-3.0.jar.sha1 deleted file mode 100644 index 0801ceea637..00000000000 --- a/solr/licenses/hamcrest-core-3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -254f69f4b0ca22198acfc19fcdd5f96140431c3e diff --git a/solr/licenses/jtokkit-1.1.0.jar.sha1 b/solr/licenses/jtokkit-1.1.0.jar.sha1 new file mode 100644 index 00000000000..a86a8ae6804 --- /dev/null +++ b/solr/licenses/jtokkit-1.1.0.jar.sha1 @@ -0,0 +1 @@ +b7370f801db3eb8c7c6a2c2c06231909ac6de0b0 diff --git a/solr/licenses/jtokkit-LICENSE-MIT.txt b/solr/licenses/jtokkit-LICENSE-MIT.txt new file mode 100644 index 00000000000..9409cee80de --- /dev/null +++ b/solr/licenses/jtokkit-LICENSE-MIT.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Knuddels, Philip Müller + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/solr/licenses/kerb-admin-2.0.3.jar.sha1 b/solr/licenses/kerb-admin-2.0.3.jar.sha1 deleted file mode 100644 index b92eec897b9..00000000000 --- a/solr/licenses/kerb-admin-2.0.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e239e34968f59b4c7f75cffff08c6a140b3afa44 diff --git a/solr/licenses/kerb-admin-NOTICE.txt b/solr/licenses/kerb-admin-NOTICE.txt deleted file mode 100644 index 373b85deba1..00000000000 --- a/solr/licenses/kerb-admin-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache Kerby -Copyright 2015-2016 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). diff --git a/solr/licenses/kerb-client-2.0.3.jar.sha1 b/solr/licenses/kerb-client-2.0.3.jar.sha1 deleted file mode 100644 index 1dcf56f6480..00000000000 --- a/solr/licenses/kerb-client-2.0.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ae6bb140ed545771efc13973f5ad665766a2e1d4 diff --git a/solr/licenses/kerb-client-NOTICE.txt b/solr/licenses/kerb-client-NOTICE.txt deleted file mode 100644 index 373b85deba1..00000000000 --- a/solr/licenses/kerb-client-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache Kerby -Copyright 2015-2016 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). diff --git a/solr/licenses/kerb-common-2.0.3.jar.sha1 b/solr/licenses/kerb-common-2.0.3.jar.sha1 deleted file mode 100644 index d15be096033..00000000000 --- a/solr/licenses/kerb-common-2.0.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -32fde271604661ef07ec61b6654d381dfeaff5b1 diff --git a/solr/licenses/kerb-common-NOTICE.txt b/solr/licenses/kerb-common-NOTICE.txt deleted file mode 100644 index 373b85deba1..00000000000 --- a/solr/licenses/kerb-common-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache Kerby -Copyright 2015-2016 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). diff --git a/solr/licenses/kerb-core-2.0.3.jar.sha1 b/solr/licenses/kerb-core-2.0.3.jar.sha1 deleted file mode 100644 index 827df82f24c..00000000000 --- a/solr/licenses/kerb-core-2.0.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2ed4ee03585bb1749f3d5bf4951208cb36839e82 diff --git a/solr/licenses/kerb-core-NOTICE.txt b/solr/licenses/kerb-core-NOTICE.txt deleted file mode 100644 index 373b85deba1..00000000000 --- a/solr/licenses/kerb-core-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache Kerby -Copyright 2015-2016 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). diff --git a/solr/licenses/kerb-crypto-2.0.3.jar.sha1 b/solr/licenses/kerb-crypto-2.0.3.jar.sha1 deleted file mode 100644 index 7d143e946e5..00000000000 --- a/solr/licenses/kerb-crypto-2.0.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a19d9c3fbcf52abd2f804f2d88f8bbf4195883a8 diff --git a/solr/licenses/kerb-crypto-LICENSE-ASL.txt b/solr/licenses/kerb-crypto-LICENSE-ASL.txt deleted file mode 100644 index ad410e11302..00000000000 --- a/solr/licenses/kerb-crypto-LICENSE-ASL.txt +++ /dev/null @@ -1,201 +0,0 @@ -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file diff --git a/solr/licenses/kerb-crypto-NOTICE.txt b/solr/licenses/kerb-crypto-NOTICE.txt deleted file mode 100644 index 373b85deba1..00000000000 --- a/solr/licenses/kerb-crypto-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache Kerby -Copyright 2015-2016 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). diff --git a/solr/licenses/kerb-identity-2.0.3.jar.sha1 b/solr/licenses/kerb-identity-2.0.3.jar.sha1 deleted file mode 100644 index 07ef6116423..00000000000 --- a/solr/licenses/kerb-identity-2.0.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0704d04804ff500f160b99ecf83258187e31d5ef diff --git a/solr/licenses/kerb-identity-LICENSE-ASL.txt b/solr/licenses/kerb-identity-LICENSE-ASL.txt deleted file mode 100644 index ad410e11302..00000000000 --- a/solr/licenses/kerb-identity-LICENSE-ASL.txt +++ /dev/null @@ -1,201 +0,0 @@ -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file diff --git a/solr/licenses/kerb-identity-NOTICE.txt b/solr/licenses/kerb-identity-NOTICE.txt deleted file mode 100644 index 373b85deba1..00000000000 --- a/solr/licenses/kerb-identity-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache Kerby -Copyright 2015-2016 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). diff --git a/solr/licenses/kerb-server-2.0.3.jar.sha1 b/solr/licenses/kerb-server-2.0.3.jar.sha1 deleted file mode 100644 index 9dee689f1e5..00000000000 --- a/solr/licenses/kerb-server-2.0.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7e2ea712dd36f8039a437d61379a60a388e1b2d2 diff --git a/solr/licenses/kerb-server-LICENSE-ASL.txt b/solr/licenses/kerb-server-LICENSE-ASL.txt deleted file mode 100644 index ad410e11302..00000000000 --- a/solr/licenses/kerb-server-LICENSE-ASL.txt +++ /dev/null @@ -1,201 +0,0 @@ -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file diff --git a/solr/licenses/kerb-server-NOTICE.txt b/solr/licenses/kerb-server-NOTICE.txt deleted file mode 100644 index 373b85deba1..00000000000 --- a/solr/licenses/kerb-server-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache Kerby -Copyright 2015-2016 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). diff --git a/solr/licenses/kerb-simplekdc-2.0.3.jar.sha1 b/solr/licenses/kerb-simplekdc-2.0.3.jar.sha1 deleted file mode 100644 index a7a9cd9888f..00000000000 --- a/solr/licenses/kerb-simplekdc-2.0.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d267abb85f5aa3223e66b978c5b5a148d1fa5304 diff --git a/solr/licenses/kerb-simplekdc-LICENSE-ASL.txt b/solr/licenses/kerb-simplekdc-LICENSE-ASL.txt deleted file mode 100644 index ad410e11302..00000000000 --- a/solr/licenses/kerb-simplekdc-LICENSE-ASL.txt +++ /dev/null @@ -1,201 +0,0 @@ -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file diff --git a/solr/licenses/kerb-simplekdc-NOTICE.txt b/solr/licenses/kerb-simplekdc-NOTICE.txt deleted file mode 100644 index 373b85deba1..00000000000 --- a/solr/licenses/kerb-simplekdc-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache Kerby -Copyright 2015-2016 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). diff --git a/solr/licenses/kerb-util-2.0.3.jar.sha1 b/solr/licenses/kerb-util-2.0.3.jar.sha1 deleted file mode 100644 index c69614b23ab..00000000000 --- a/solr/licenses/kerb-util-2.0.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dda5fbcbd1e8d5de40edcf6161faf94b915152ce diff --git a/solr/licenses/kerb-util-LICENSE-ASL.txt b/solr/licenses/kerb-util-LICENSE-ASL.txt deleted file mode 100644 index ad410e11302..00000000000 --- a/solr/licenses/kerb-util-LICENSE-ASL.txt +++ /dev/null @@ -1,201 +0,0 @@ -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file diff --git a/solr/licenses/kerb-util-NOTICE.txt b/solr/licenses/kerb-util-NOTICE.txt deleted file mode 100644 index 373b85deba1..00000000000 --- a/solr/licenses/kerb-util-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache Kerby -Copyright 2015-2016 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). diff --git a/solr/licenses/kerby-asn1-2.0.3.jar.sha1 b/solr/licenses/kerby-asn1-2.0.3.jar.sha1 deleted file mode 100644 index c2f07295a8e..00000000000 --- a/solr/licenses/kerby-asn1-2.0.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -71e4005a005c8c4476b959befdc1ad668fbbe758 diff --git a/solr/licenses/kerby-asn1-LICENSE-ASL.txt b/solr/licenses/kerby-asn1-LICENSE-ASL.txt deleted file mode 100644 index ad410e11302..00000000000 --- a/solr/licenses/kerby-asn1-LICENSE-ASL.txt +++ /dev/null @@ -1,201 +0,0 @@ -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file diff --git a/solr/licenses/kerby-asn1-NOTICE.txt b/solr/licenses/kerby-asn1-NOTICE.txt deleted file mode 100644 index 373b85deba1..00000000000 --- a/solr/licenses/kerby-asn1-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache Kerby -Copyright 2015-2016 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). diff --git a/solr/licenses/kerby-config-2.0.3.jar.sha1 b/solr/licenses/kerby-config-2.0.3.jar.sha1 deleted file mode 100644 index 71e9be416bd..00000000000 --- a/solr/licenses/kerby-config-2.0.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -322b25b1fb7f57f6987c54c5ef32887036108d61 diff --git a/solr/licenses/kerby-config-LICENSE-ASL.txt b/solr/licenses/kerby-config-LICENSE-ASL.txt deleted file mode 100644 index ad410e11302..00000000000 --- a/solr/licenses/kerby-config-LICENSE-ASL.txt +++ /dev/null @@ -1,201 +0,0 @@ -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file diff --git a/solr/licenses/kerby-config-NOTICE.txt b/solr/licenses/kerby-config-NOTICE.txt deleted file mode 100644 index 373b85deba1..00000000000 --- a/solr/licenses/kerby-config-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache Kerby -Copyright 2015-2016 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). diff --git a/solr/licenses/kerby-pkix-2.0.3.jar.sha1 b/solr/licenses/kerby-pkix-2.0.3.jar.sha1 deleted file mode 100644 index 5fc668a5341..00000000000 --- a/solr/licenses/kerby-pkix-2.0.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8301fe828f3b931617df990f77da636eacd9212d diff --git a/solr/licenses/kerby-pkix-LICENSE-ASL.txt b/solr/licenses/kerby-pkix-LICENSE-ASL.txt deleted file mode 100644 index ad410e11302..00000000000 --- a/solr/licenses/kerby-pkix-LICENSE-ASL.txt +++ /dev/null @@ -1,201 +0,0 @@ -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file diff --git a/solr/licenses/kerby-pkix-NOTICE.txt b/solr/licenses/kerby-pkix-NOTICE.txt deleted file mode 100644 index 373b85deba1..00000000000 --- a/solr/licenses/kerby-pkix-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache Kerby -Copyright 2015-2016 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). diff --git a/solr/licenses/kerby-util-2.0.3.jar.sha1 b/solr/licenses/kerby-util-2.0.3.jar.sha1 deleted file mode 100644 index 6d3ff35323d..00000000000 --- a/solr/licenses/kerby-util-2.0.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f1e80545460c37fca6b2c18f02fa32f6040928f0 diff --git a/solr/licenses/kerby-util-LICENSE-ASL.txt b/solr/licenses/kerby-util-LICENSE-ASL.txt deleted file mode 100644 index ad410e11302..00000000000 --- a/solr/licenses/kerby-util-LICENSE-ASL.txt +++ /dev/null @@ -1,201 +0,0 @@ -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file diff --git a/solr/licenses/kerby-util-NOTICE.txt b/solr/licenses/kerby-util-NOTICE.txt deleted file mode 100644 index 373b85deba1..00000000000 --- a/solr/licenses/kerby-util-NOTICE.txt +++ /dev/null @@ -1,5 +0,0 @@ -Apache Kerby -Copyright 2015-2016 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). diff --git a/solr/licenses/langchain4j-0.35.0.jar.sha1 b/solr/licenses/langchain4j-0.35.0.jar.sha1 new file mode 100644 index 00000000000..2199fe110d3 --- /dev/null +++ b/solr/licenses/langchain4j-0.35.0.jar.sha1 @@ -0,0 +1 @@ +f739eae706dcfd090c56c2a470e1b6bd8d60e175 diff --git a/solr/licenses/kerb-admin-LICENSE-ASL.txt b/solr/licenses/langchain4j-LICENSE-ASL.txt similarity index 99% rename from solr/licenses/kerb-admin-LICENSE-ASL.txt rename to solr/licenses/langchain4j-LICENSE-ASL.txt index ad410e11302..a12e3073373 100644 --- a/solr/licenses/kerb-admin-LICENSE-ASL.txt +++ b/solr/licenses/langchain4j-LICENSE-ASL.txt @@ -1,4 +1,5 @@ Apache License + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -178,7 +179,7 @@ Apache License APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -186,7 +187,7 @@ Apache License same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/solr/licenses/langchain4j-NOTICE.txt b/solr/licenses/langchain4j-NOTICE.txt new file mode 100644 index 00000000000..be32f8a2f46 --- /dev/null +++ b/solr/licenses/langchain4j-NOTICE.txt @@ -0,0 +1,13 @@ +The goal of LangChain4j is to simplify integrating LLMs into Java applications. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/solr/licenses/langchain4j-cohere-0.35.0.jar.sha1 b/solr/licenses/langchain4j-cohere-0.35.0.jar.sha1 new file mode 100644 index 00000000000..a2a66c77142 --- /dev/null +++ b/solr/licenses/langchain4j-cohere-0.35.0.jar.sha1 @@ -0,0 +1 @@ +737ebd43f49e1820b17b18b6e3154bf94e0b2853 diff --git a/solr/licenses/langchain4j-core-0.35.0.jar.sha1 b/solr/licenses/langchain4j-core-0.35.0.jar.sha1 new file mode 100644 index 00000000000..04931abbf53 --- /dev/null +++ b/solr/licenses/langchain4j-core-0.35.0.jar.sha1 @@ -0,0 +1 @@ +1ac1bf8b9aa2864ac3fcae7d46a5d9f80a582b30 diff --git a/solr/licenses/langchain4j-hugging-face-0.35.0.jar.sha1 b/solr/licenses/langchain4j-hugging-face-0.35.0.jar.sha1 new file mode 100644 index 00000000000..789b74e63dc --- /dev/null +++ b/solr/licenses/langchain4j-hugging-face-0.35.0.jar.sha1 @@ -0,0 +1 @@ +4eb72d70b9e26473d3e02f9199ccdf3a9bc24cbf diff --git a/solr/licenses/langchain4j-mistral-ai-0.35.0.jar.sha1 b/solr/licenses/langchain4j-mistral-ai-0.35.0.jar.sha1 new file mode 100644 index 00000000000..f37eb1b33db --- /dev/null +++ b/solr/licenses/langchain4j-mistral-ai-0.35.0.jar.sha1 @@ -0,0 +1 @@ +4f1165e6ee19430dcadc6cb260d798a40fd49ef0 diff --git a/solr/licenses/langchain4j-open-ai-0.35.0.jar.sha1 b/solr/licenses/langchain4j-open-ai-0.35.0.jar.sha1 new file mode 100644 index 00000000000..35cdfb38385 --- /dev/null +++ b/solr/licenses/langchain4j-open-ai-0.35.0.jar.sha1 @@ -0,0 +1 @@ +d77645377c517e18fd5e0ca2ca56d4a38e21495a diff --git a/solr/licenses/netty-LICENSE-ASL.txt b/solr/licenses/netty-LICENSE-ASL.txt index d6456956733..62589edd12a 100644 --- a/solr/licenses/netty-LICENSE-ASL.txt +++ b/solr/licenses/netty-LICENSE-ASL.txt @@ -1,7 +1,7 @@ Apache License Version 2.0, January 2004 - http://www.apache.org/licenses/ + https://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION @@ -193,7 +193,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/solr/licenses/netty-NOTICE.txt b/solr/licenses/netty-NOTICE.txt index f973663670b..953277d6594 100644 --- a/solr/licenses/netty-NOTICE.txt +++ b/solr/licenses/netty-NOTICE.txt @@ -4,7 +4,7 @@ Please visit the Netty web site for more information: - * http://netty.io/ + * https://netty.io/ Copyright 2014 The Netty Project @@ -12,7 +12,7 @@ The Netty Project licenses this file to you under the Apache License, version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT @@ -56,7 +56,7 @@ facade for Java, which can be obtained at: * LICENSE: * license/LICENSE.slf4j.txt (MIT License) * HOMEPAGE: - * http://www.slf4j.org/ + * https://www.slf4j.org/ This product contains a modified portion of 'Apache Harmony', an open source Java SE, which can be obtained at: @@ -66,7 +66,7 @@ Java SE, which can be obtained at: * LICENSE: * license/LICENSE.harmony.txt (Apache License 2.0) * HOMEPAGE: - * http://archive.apache.org/dist/harmony/ + * https://archive.apache.org/dist/harmony/ This product contains a modified portion of 'jbzip2', a Java bzip2 compression and decompression library written by Matthew J. Francis. It can be obtained at: @@ -125,6 +125,14 @@ and decompression library, which can be obtained at: * HOMEPAGE: * https://github.com/jponge/lzma-java +This product optionally depends on 'zstd-jni', a zstd-jni Java compression +and decompression library, which can be obtained at: + + * LICENSE: + * license/LICENSE.zstd-jni.txt (BSD) + * HOMEPAGE: + * https://github.com/luben/zstd-jni + This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression and decompression library written by William Kinney. It can be obtained at: @@ -148,7 +156,7 @@ equivalent functionality. It can be obtained at: * LICENSE: * license/LICENSE.bouncycastle.txt (MIT License) * HOMEPAGE: - * http://www.bouncycastle.org/ + * https://www.bouncycastle.org/ This product optionally depends on 'Snappy', a compression library produced by Google Inc, which can be obtained at: @@ -162,9 +170,9 @@ This product optionally depends on 'JBoss Marshalling', an alternative Java serialization API, which can be obtained at: * LICENSE: - * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1) + * license/LICENSE.jboss-marshalling.txt (Apache License 2.0) * HOMEPAGE: - * http://www.jboss.org/jbossmarshalling + * https://github.com/jboss-remoting/jboss-marshalling This product optionally depends on 'Caliper', Google's micro- benchmarking framework, which can be obtained at: @@ -180,7 +188,7 @@ framework, which can be obtained at: * LICENSE: * license/LICENSE.commons-logging.txt (Apache License 2.0) * HOMEPAGE: - * http://commons.apache.org/logging/ + * https://commons.apache.org/logging/ This product optionally depends on 'Apache Log4J', a logging framework, which can be obtained at: @@ -188,7 +196,7 @@ can be obtained at: * LICENSE: * license/LICENSE.log4j.txt (Apache License 2.0) * HOMEPAGE: - * http://logging.apache.org/log4j/ + * https://logging.apache.org/log4j/ This product optionally depends on 'Aalto XML', an ultra-high performance non-blocking XML processor, which can be obtained at: @@ -196,7 +204,7 @@ non-blocking XML processor, which can be obtained at: * LICENSE: * license/LICENSE.aalto-xml.txt (Apache License 2.0) * HOMEPAGE: - * http://wiki.fasterxml.com/AaltoHome + * https://wiki.fasterxml.com/AaltoHome This product contains a modified version of 'HPACK', a Java implementation of the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: @@ -206,6 +214,22 @@ the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: * HOMEPAGE: * https://github.com/twitter/hpack +This product contains a modified version of 'HPACK', a Java implementation of +the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: + + * LICENSE: + * license/LICENSE.hyper-hpack.txt (MIT License) + * HOMEPAGE: + * https://github.com/python-hyper/hpack/ + +This product contains a modified version of 'HPACK', a Java implementation of +the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: + + * LICENSE: + * license/LICENSE.nghttp2-hpack.txt (MIT License) + * HOMEPAGE: + * https://github.com/nghttp2/nghttp2/ + This product contains a modified portion of 'Apache Commons Lang', a Java library provides utilities for the java.lang API, which can be obtained at: @@ -221,3 +245,20 @@ This product contains the Maven wrapper scripts from 'Maven Wrapper', that provi * license/LICENSE.mvn-wrapper.txt (Apache License 2.0) * HOMEPAGE: * https://github.com/takari/maven-wrapper + +This product contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. +This private header is also used by Apple's open source + mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). + + * LICENSE: + * license/LICENSE.dnsinfo.txt (Apple Public Source License 2.0) + * HOMEPAGE: + * https://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h + +This product optionally depends on 'Brotli4j', Brotli compression and +decompression for Java., which can be obtained at: + + * LICENSE: + * license/LICENSE.brotli4j.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/hyperxpro/Brotli4j diff --git a/solr/licenses/netty-buffer-4.1.112.Final.jar.sha1 b/solr/licenses/netty-buffer-4.1.112.Final.jar.sha1 deleted file mode 100644 index 0ac205b0823..00000000000 --- a/solr/licenses/netty-buffer-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bdc12df04bb6858890b8aa108060b5b365a26102 diff --git a/solr/licenses/netty-buffer-4.1.114.Final.jar.sha1 b/solr/licenses/netty-buffer-4.1.114.Final.jar.sha1 new file mode 100644 index 00000000000..720e556e6d8 --- /dev/null +++ b/solr/licenses/netty-buffer-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +f1d77d15c0b781cd9395a2a956262766fd0c7602 diff --git a/solr/licenses/netty-codec-4.1.112.Final.jar.sha1 b/solr/licenses/netty-codec-4.1.112.Final.jar.sha1 deleted file mode 100644 index 55360968342..00000000000 --- a/solr/licenses/netty-codec-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c87f2ec3d9a97bd2b793d16817abb2bab93a7fc3 diff --git a/solr/licenses/netty-codec-4.1.114.Final.jar.sha1 b/solr/licenses/netty-codec-4.1.114.Final.jar.sha1 new file mode 100644 index 00000000000..84b726e18a0 --- /dev/null +++ b/solr/licenses/netty-codec-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +5a49dfa2828d64bf756f670e63259115332744cf diff --git a/solr/licenses/netty-codec-http-4.1.112.Final.jar.sha1 b/solr/licenses/netty-codec-http-4.1.112.Final.jar.sha1 deleted file mode 100644 index c5a4048f7eb..00000000000 --- a/solr/licenses/netty-codec-http-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -81af1040bfa977f98dd0e1bd9639513ea862ca04 diff --git a/solr/licenses/netty-codec-http-4.1.114.Final.jar.sha1 b/solr/licenses/netty-codec-http-4.1.114.Final.jar.sha1 new file mode 100644 index 00000000000..c7c0248da60 --- /dev/null +++ b/solr/licenses/netty-codec-http-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +fbce5a53884275662e68aaad70f88bf7e5d04164 diff --git a/solr/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 b/solr/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 deleted file mode 100644 index d8ff515db81..00000000000 --- a/solr/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7fa28b510f0f16f4d5d7188b86bef59e048f62f9 diff --git a/solr/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 b/solr/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 new file mode 100644 index 00000000000..47c75f00256 --- /dev/null +++ b/solr/licenses/netty-codec-http2-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +19ae07fdf99142a70338f8cda70a3d2edbc8e80a diff --git a/solr/licenses/netty-codec-socks-4.1.112.Final.jar.sha1 b/solr/licenses/netty-codec-socks-4.1.112.Final.jar.sha1 deleted file mode 100644 index 0b3d39a0cee..00000000000 --- a/solr/licenses/netty-codec-socks-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9aed7e78c467d06a47a45b5b27466380a6427e2f diff --git a/solr/licenses/netty-codec-socks-4.1.114.Final.jar.sha1 b/solr/licenses/netty-codec-socks-4.1.114.Final.jar.sha1 new file mode 100644 index 00000000000..decacfe281e --- /dev/null +++ b/solr/licenses/netty-codec-socks-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +42b1159cac3d196f6bdbd528e29f0fab9dbaae06 diff --git a/solr/licenses/netty-common-4.1.112.Final.jar.sha1 b/solr/licenses/netty-common-4.1.112.Final.jar.sha1 deleted file mode 100644 index 1659204d787..00000000000 --- a/solr/licenses/netty-common-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b2798069092a981a832b7510d0462ee9efb7a80e diff --git a/solr/licenses/netty-common-4.1.114.Final.jar.sha1 b/solr/licenses/netty-common-4.1.114.Final.jar.sha1 new file mode 100644 index 00000000000..e4304152793 --- /dev/null +++ b/solr/licenses/netty-common-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +862712e292b162c8ccaa7847a6a54df8178f77e5 diff --git a/solr/licenses/netty-handler-4.1.112.Final.jar.sha1 b/solr/licenses/netty-handler-4.1.112.Final.jar.sha1 deleted file mode 100644 index 43be3612a0d..00000000000 --- a/solr/licenses/netty-handler-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3d5e2d5bcc6baeeb8c13a230980c6132a778e036 diff --git a/solr/licenses/netty-handler-4.1.114.Final.jar.sha1 b/solr/licenses/netty-handler-4.1.114.Final.jar.sha1 new file mode 100644 index 00000000000..3a44d3b3d57 --- /dev/null +++ b/solr/licenses/netty-handler-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +e56fbde4b9aa628eed15a5dbfbeb97877db88146 diff --git a/solr/licenses/netty-handler-proxy-4.1.112.Final.jar.sha1 b/solr/licenses/netty-handler-proxy-4.1.112.Final.jar.sha1 deleted file mode 100644 index d85e58a7af2..00000000000 --- a/solr/licenses/netty-handler-proxy-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b23c87a85451b3b0e7c3e8e89698cea6831a8418 diff --git a/solr/licenses/netty-handler-proxy-4.1.114.Final.jar.sha1 b/solr/licenses/netty-handler-proxy-4.1.114.Final.jar.sha1 new file mode 100644 index 00000000000..49fbddcc7db --- /dev/null +++ b/solr/licenses/netty-handler-proxy-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +a01071edffb4812009312b461ce5f160cdec9b75 diff --git a/solr/licenses/netty-resolver-4.1.112.Final.jar.sha1 b/solr/licenses/netty-resolver-4.1.112.Final.jar.sha1 deleted file mode 100644 index 95eb0338e52..00000000000 --- a/solr/licenses/netty-resolver-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -58a631d9d44c4ed7cc0dcc9cffa6641da9374d72 diff --git a/solr/licenses/netty-resolver-4.1.114.Final.jar.sha1 b/solr/licenses/netty-resolver-4.1.114.Final.jar.sha1 new file mode 100644 index 00000000000..c1135b835f7 --- /dev/null +++ b/solr/licenses/netty-resolver-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +10b23784b23d6a948930f52ba82874f1291b5873 diff --git a/solr/licenses/netty-tcnative-boringssl-static-2.0.61.Final.jar.sha1 b/solr/licenses/netty-tcnative-boringssl-static-2.0.61.Final.jar.sha1 deleted file mode 100644 index 1c3e5f7d66f..00000000000 --- a/solr/licenses/netty-tcnative-boringssl-static-2.0.61.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -944722ba2883fe5825a0c6b38dc727d7151a9371 diff --git a/solr/licenses/netty-tcnative-boringssl-static-2.0.66.Final.jar.sha1 b/solr/licenses/netty-tcnative-boringssl-static-2.0.66.Final.jar.sha1 new file mode 100644 index 00000000000..04aa0028843 --- /dev/null +++ b/solr/licenses/netty-tcnative-boringssl-static-2.0.66.Final.jar.sha1 @@ -0,0 +1 @@ +7b0abf028c27ce6ad0c17a443a13f175fb4eda27 diff --git a/solr/licenses/netty-tcnative-classes-2.0.61.Final.jar.sha1 b/solr/licenses/netty-tcnative-classes-2.0.61.Final.jar.sha1 deleted file mode 100644 index ca1be9444d7..00000000000 --- a/solr/licenses/netty-tcnative-classes-2.0.61.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4c6ae851ed97921bc6c6b64e019c2b039f49131a diff --git a/solr/licenses/netty-tcnative-classes-2.0.66.Final.jar.sha1 b/solr/licenses/netty-tcnative-classes-2.0.66.Final.jar.sha1 new file mode 100644 index 00000000000..77a6debf6b2 --- /dev/null +++ b/solr/licenses/netty-tcnative-classes-2.0.66.Final.jar.sha1 @@ -0,0 +1 @@ +9588bd2f891157538a78d86c945aa34bf9308dda diff --git a/solr/licenses/netty-transport-4.1.112.Final.jar.sha1 b/solr/licenses/netty-transport-4.1.112.Final.jar.sha1 deleted file mode 100644 index 5edb433d5f4..00000000000 --- a/solr/licenses/netty-transport-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -77cd136dd3843f5e7cbcf68c824975d745c49ddb diff --git a/solr/licenses/netty-transport-4.1.114.Final.jar.sha1 b/solr/licenses/netty-transport-4.1.114.Final.jar.sha1 new file mode 100644 index 00000000000..f74dfc80c43 --- /dev/null +++ b/solr/licenses/netty-transport-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +e0225a575f487904be8517092cbd74e01913533c diff --git a/solr/licenses/netty-transport-classes-epoll-4.1.112.Final.jar.sha1 b/solr/licenses/netty-transport-classes-epoll-4.1.112.Final.jar.sha1 deleted file mode 100644 index 1364819aa58..00000000000 --- a/solr/licenses/netty-transport-classes-epoll-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -67e590356eb53c20aaabd67f61ae66f628e62e3d diff --git a/solr/licenses/netty-transport-classes-epoll-4.1.114.Final.jar.sha1 b/solr/licenses/netty-transport-classes-epoll-4.1.114.Final.jar.sha1 new file mode 100644 index 00000000000..c9046686be2 --- /dev/null +++ b/solr/licenses/netty-transport-classes-epoll-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +f442c794e6fe89e6974f058bf393353e01fb927d diff --git a/solr/licenses/netty-transport-native-epoll-4.1.112.Final-linux-x86_64.jar.sha1 b/solr/licenses/netty-transport-native-epoll-4.1.112.Final-linux-x86_64.jar.sha1 deleted file mode 100644 index 9995f7b1048..00000000000 --- a/solr/licenses/netty-transport-native-epoll-4.1.112.Final-linux-x86_64.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -caed6f2ae7aebcef50098a64f55918cb8550d2d0 diff --git a/solr/licenses/netty-transport-native-epoll-4.1.114.Final-linux-x86_64.jar.sha1 b/solr/licenses/netty-transport-native-epoll-4.1.114.Final-linux-x86_64.jar.sha1 new file mode 100644 index 00000000000..672f28c70a2 --- /dev/null +++ b/solr/licenses/netty-transport-native-epoll-4.1.114.Final-linux-x86_64.jar.sha1 @@ -0,0 +1 @@ +43268d2bef66e72e5a7956045a3caf8395f49ae6 diff --git a/solr/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 b/solr/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 deleted file mode 100644 index 265f56097e8..00000000000 --- a/solr/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b50ff619cdcdc48e748cba3405c9988529f28f60 diff --git a/solr/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 b/solr/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 new file mode 100644 index 00000000000..9b40cb77ea2 --- /dev/null +++ b/solr/licenses/netty-transport-native-unix-common-4.1.114.Final.jar.sha1 @@ -0,0 +1 @@ +d1171bb99411f282068f49d780cedf8c9adeabfd diff --git a/solr/licenses/okhttp-sse-4.12.0.jar.sha1 b/solr/licenses/okhttp-sse-4.12.0.jar.sha1 new file mode 100644 index 00000000000..60d422fb2c6 --- /dev/null +++ b/solr/licenses/okhttp-sse-4.12.0.jar.sha1 @@ -0,0 +1 @@ +eca9c68c54ae7fd18d465beba65d80a44e9667e4 diff --git a/solr/licenses/openai4j-0.22.0.jar.sha1 b/solr/licenses/openai4j-0.22.0.jar.sha1 new file mode 100644 index 00000000000..506a2fd66b4 --- /dev/null +++ b/solr/licenses/openai4j-0.22.0.jar.sha1 @@ -0,0 +1 @@ +a9c8a0cdbf43cd31a08ee7ce957ab84991b67427 diff --git a/solr/licenses/kerb-core-LICENSE-ASL.txt b/solr/licenses/openai4j-LICENSE-ASL.txt similarity index 99% rename from solr/licenses/kerb-core-LICENSE-ASL.txt rename to solr/licenses/openai4j-LICENSE-ASL.txt index ad410e11302..f49a4e16e68 100644 --- a/solr/licenses/kerb-core-LICENSE-ASL.txt +++ b/solr/licenses/openai4j-LICENSE-ASL.txt @@ -1,4 +1,4 @@ -Apache License + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -178,7 +178,7 @@ Apache License APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -186,7 +186,7 @@ Apache License same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/solr/licenses/openai4j-NOTICE.txt b/solr/licenses/openai4j-NOTICE.txt new file mode 100644 index 00000000000..60508a114b6 --- /dev/null +++ b/solr/licenses/openai4j-NOTICE.txt @@ -0,0 +1,11 @@ +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/solr/licenses/retrofit-2.9.0.jar.sha1 b/solr/licenses/retrofit-2.9.0.jar.sha1 new file mode 100644 index 00000000000..8939819d52c --- /dev/null +++ b/solr/licenses/retrofit-2.9.0.jar.sha1 @@ -0,0 +1 @@ +d8fdfbd5da952141a665a403348b74538efc05ff diff --git a/solr/licenses/kerb-common-LICENSE-ASL.txt b/solr/licenses/retrofit-LICENSE-ASL.txt similarity index 99% rename from solr/licenses/kerb-common-LICENSE-ASL.txt rename to solr/licenses/retrofit-LICENSE-ASL.txt index ad410e11302..7a4a3ea2424 100644 --- a/solr/licenses/kerb-common-LICENSE-ASL.txt +++ b/solr/licenses/retrofit-LICENSE-ASL.txt @@ -1,4 +1,5 @@ -Apache License + + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -178,7 +179,7 @@ Apache License APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -186,7 +187,7 @@ Apache License same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/solr/licenses/retrofit-NOTICE.txt b/solr/licenses/retrofit-NOTICE.txt new file mode 100644 index 00000000000..25fdd5926ad --- /dev/null +++ b/solr/licenses/retrofit-NOTICE.txt @@ -0,0 +1,13 @@ +Copyright 2013 Square, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/solr/modules/analysis-extras/build.gradle b/solr/modules/analysis-extras/build.gradle index 09398b4ac5b..ad1b8714915 100644 --- a/solr/modules/analysis-extras/build.gradle +++ b/solr/modules/analysis-extras/build.gradle @@ -24,25 +24,25 @@ dependencies { implementation project(':solr:solrj') - implementation 'com.ibm.icu:icu4j' - implementation 'org.apache.lucene:lucene-analysis-icu' - runtimeOnly 'org.apache.lucene:lucene-analysis-morfologik' - implementation 'org.apache.lucene:lucene-analysis-opennlp' - runtimeOnly 'org.apache.lucene:lucene-analysis-smartcn' - runtimeOnly 'org.apache.lucene:lucene-analysis-stempel' - implementation 'org.apache.lucene:lucene-core' + implementation libs.ibm.icu.icu4j + implementation libs.apache.lucene.analysis.icu + runtimeOnly libs.apache.lucene.analysis.morfologik + implementation libs.apache.lucene.analysis.opennlp + runtimeOnly libs.apache.lucene.analysis.smartcn + runtimeOnly libs.apache.lucene.analysis.stempel + implementation libs.apache.lucene.core // NOTE: Need to stay on same version of opennlp-tools as lucene-analysis-opennlp - implementation 'org.apache.opennlp:opennlp-tools' - implementation 'org.slf4j:slf4j-api' + implementation libs.apache.opennlp.tools + implementation libs.slf4j.api testImplementation project(':solr:test-framework') - testImplementation 'org.apache.lucene:lucene-analysis-common' - testImplementation 'commons-io:commons-io' - testImplementation 'junit:junit' - testImplementation('org.mockito:mockito-core', { + testImplementation libs.apache.lucene.analysis.common + testImplementation libs.commonsio.commonsio + testImplementation libs.junit.junit + testImplementation(libs.mockito.core, { exclude group: "net.bytebuddy", module: "byte-buddy-agent" }) - testRuntimeOnly('org.mockito:mockito-subclass', { + testRuntimeOnly(libs.mockito.subclass, { exclude group: "net.bytebuddy", module: "byte-buddy-agent" }) } diff --git a/solr/modules/analysis-extras/src/java/org/apache/solr/update/processor/OpenNLPExtractNamedEntitiesUpdateProcessorFactory.java b/solr/modules/analysis-extras/src/java/org/apache/solr/update/processor/OpenNLPExtractNamedEntitiesUpdateProcessorFactory.java index 9df258cd303..338eeea0e5e 100644 --- a/solr/modules/analysis-extras/src/java/org/apache/solr/update/processor/OpenNLPExtractNamedEntitiesUpdateProcessorFactory.java +++ b/solr/modules/analysis-extras/src/java/org/apache/solr/update/processor/OpenNLPExtractNamedEntitiesUpdateProcessorFactory.java @@ -394,11 +394,10 @@ private void initSourceSelectorSyntax(NamedList args) { throw new SolrException( SERVER_ERROR, "Init param '" + SOURCE_PARAM + "' child 'exclude' can not be null"); } - if (!(excObj instanceof NamedList)) { + if (!(excObj instanceof NamedList exc)) { throw new SolrException( SERVER_ERROR, "Init param '" + SOURCE_PARAM + "' child 'exclude' must be "); } - NamedList exc = (NamedList) excObj; srcExclusions.add(parseSelectorParams(exc)); if (0 < exc.size()) { throw new SolrException( @@ -445,8 +444,7 @@ private void initSourceSelectorSyntax(NamedList args) { + "for OpenNLPExtractNamedEntitiesUpdateProcessor for further details."); } - if (d instanceof NamedList) { - NamedList destList = (NamedList) d; + if (d instanceof NamedList destList) { Object patt = destList.remove(PATTERN_PARAM); Object replacement = destList.remove(REPLACEMENT_PARAM); diff --git a/solr/modules/analysis-extras/src/test/org/apache/solr/analysis/TestFoldingMultitermExtrasQuery.java b/solr/modules/analysis-extras/src/test/org/apache/solr/analysis/TestFoldingMultitermExtrasQuery.java index f6138cdde1a..e5b066263a5 100644 --- a/solr/modules/analysis-extras/src/test/org/apache/solr/analysis/TestFoldingMultitermExtrasQuery.java +++ b/solr/modules/analysis-extras/src/test/org/apache/solr/analysis/TestFoldingMultitermExtrasQuery.java @@ -16,8 +16,8 @@ */ package org.apache.solr.analysis; -import java.io.File; -import org.apache.commons.io.FileUtils; +import java.nio.file.Path; +import org.apache.commons.io.file.PathUtils; import org.apache.solr.SolrTestCaseJ4; import org.junit.BeforeClass; import org.junit.Test; @@ -32,9 +32,12 @@ public String getCoreName() { @BeforeClass public static void beforeTests() throws Exception { - File testHome = createTempDir().toFile(); - FileUtils.copyDirectory(getFile("analysis-extras/solr"), testHome); - initCore("solrconfig-icucollate.xml", "schema-folding-extra.xml", testHome.getAbsolutePath()); + Path testHome = createTempDir(); + PathUtils.copyDirectory(getFile("analysis-extras/solr"), testHome); + initCore( + "solrconfig-icucollate.xml", + "schema-folding-extra.xml", + testHome.toAbsolutePath().toString()); int idx = 1; // ICUFoldingFilterFactory diff --git a/solr/modules/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java b/solr/modules/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java index 7193d9f706a..d66f959c799 100644 --- a/solr/modules/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java +++ b/solr/modules/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java @@ -66,10 +66,10 @@ public static String setupSolrHome() throws Exception { // copy over configuration files Files.copy( - getFile("analysis-extras/solr/collection1/conf/solrconfig-icucollate.xml").toPath(), + getFile("analysis-extras/solr/collection1/conf/solrconfig-icucollate.xml"), confDir.resolve("solrconfig.xml")); Files.copy( - getFile("analysis-extras/solr/collection1/conf/schema-icucollate.xml").toPath(), + getFile("analysis-extras/solr/collection1/conf/schema-icucollate.xml"), confDir.resolve("schema.xml")); // generate custom collation rules (DIN 5007-2), saving to customrules.dat diff --git a/solr/modules/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationFieldDocValues.java b/solr/modules/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationFieldDocValues.java index 2b14bd8a5c6..37c85d9392b 100644 --- a/solr/modules/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationFieldDocValues.java +++ b/solr/modules/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationFieldDocValues.java @@ -63,10 +63,10 @@ public static String setupSolrHome() throws Exception { // copy over configuration files Files.copy( - getFile("analysis-extras/solr/collection1/conf/solrconfig-icucollate.xml").toPath(), + getFile("analysis-extras/solr/collection1/conf/solrconfig-icucollate.xml"), confDir.resolve("solrconfig.xml")); Files.copy( - getFile("analysis-extras/solr/collection1/conf/schema-icucollate-dv.xml").toPath(), + getFile("analysis-extras/solr/collection1/conf/schema-icucollate-dv.xml"), confDir.resolve("schema.xml")); // generate custom collation rules (DIN 5007-2), saving to customrules.dat diff --git a/solr/modules/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationFieldOptions.java b/solr/modules/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationFieldOptions.java index 63f8b00b0d3..62f1cbea550 100644 --- a/solr/modules/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationFieldOptions.java +++ b/solr/modules/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationFieldOptions.java @@ -16,8 +16,8 @@ */ package org.apache.solr.schema; -import java.io.File; -import org.apache.commons.io.FileUtils; +import java.nio.file.Path; +import org.apache.commons.io.file.PathUtils; import org.apache.solr.SolrTestCaseJ4; import org.junit.BeforeClass; @@ -25,10 +25,12 @@ public class TestICUCollationFieldOptions extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() throws Exception { - File testHome = createTempDir().toFile(); - FileUtils.copyDirectory(getFile("analysis-extras/solr"), testHome); + Path testHome = createTempDir(); + PathUtils.copyDirectory(getFile("analysis-extras/solr"), testHome); initCore( - "solrconfig-icucollate.xml", "schema-icucollateoptions.xml", testHome.getAbsolutePath()); + "solrconfig-icucollate.xml", + "schema-icucollateoptions.xml", + testHome.toAbsolutePath().toString()); // add some docs assertU(adoc("id", "1", "text", "foo-bar")); assertU(adoc("id", "2", "text", "foo bar")); diff --git a/solr/modules/analysis-extras/src/test/org/apache/solr/update/processor/TestOpenNLPExtractNamedEntitiesUpdateProcessorFactory.java b/solr/modules/analysis-extras/src/test/org/apache/solr/update/processor/TestOpenNLPExtractNamedEntitiesUpdateProcessorFactory.java index 46b219dedde..8144bba6cd5 100644 --- a/solr/modules/analysis-extras/src/test/org/apache/solr/update/processor/TestOpenNLPExtractNamedEntitiesUpdateProcessorFactory.java +++ b/solr/modules/analysis-extras/src/test/org/apache/solr/update/processor/TestOpenNLPExtractNamedEntitiesUpdateProcessorFactory.java @@ -17,9 +17,9 @@ package org.apache.solr.update.processor; -import java.io.File; +import java.nio.file.Path; import java.util.List; -import org.apache.commons.io.FileUtils; +import org.apache.commons.io.file.PathUtils; import org.apache.solr.common.SolrInputDocument; import org.junit.BeforeClass; import org.junit.Test; @@ -28,10 +28,12 @@ public class TestOpenNLPExtractNamedEntitiesUpdateProcessorFactory extends Updat @BeforeClass public static void beforeClass() throws Exception { - File testHome = createTempDir().toFile(); - FileUtils.copyDirectory(getFile("analysis-extras/solr"), testHome); + Path testHome = createTempDir(); + PathUtils.copyDirectory(getFile("analysis-extras/solr"), testHome); initCore( - "solrconfig-opennlp-extract.xml", "schema-opennlp-extract.xml", testHome.getAbsolutePath()); + "solrconfig-opennlp-extract.xml", + "schema-opennlp-extract.xml", + testHome.toAbsolutePath().toString()); } @Test diff --git a/solr/modules/clustering/build.gradle b/solr/modules/clustering/build.gradle index fa0811dda7d..a1546b6b09f 100644 --- a/solr/modules/clustering/build.gradle +++ b/solr/modules/clustering/build.gradle @@ -23,15 +23,15 @@ dependencies { implementation project(':solr:core') implementation project(':solr:solrj') - implementation 'org.apache.lucene:lucene-core' + implementation libs.apache.lucene.core - implementation 'org.carrot2:carrot2-core' - implementation 'org.slf4j:slf4j-api' + implementation libs.carrot2.core + implementation libs.slf4j.api testImplementation project(':solr:test-framework') - testImplementation 'com.carrotsearch.randomizedtesting:randomizedtesting-runner' - testImplementation 'junit:junit' - testImplementation 'org.hamcrest:hamcrest' + testImplementation libs.carrotsearch.randomizedtesting.runner + testImplementation libs.junit.junit + testImplementation libs.hamcrest.hamcrest - testImplementation 'commons-io:commons-io' + testImplementation libs.commonsio.commonsio } diff --git a/solr/modules/clustering/src/test/org/apache/solr/handler/clustering/ClusteringComponentDistributedTest.java b/solr/modules/clustering/src/test/org/apache/solr/handler/clustering/ClusteringComponentDistributedTest.java index e427e8840f9..003276312e3 100644 --- a/solr/modules/clustering/src/test/org/apache/solr/handler/clustering/ClusteringComponentDistributedTest.java +++ b/solr/modules/clustering/src/test/org/apache/solr/handler/clustering/ClusteringComponentDistributedTest.java @@ -36,7 +36,7 @@ public class ClusteringComponentDistributedTest extends BaseDistributedSearchTes @Override public String getSolrHome() { - return getFile("clustering/solr/collection1").getParent(); + return getFile("clustering/solr/collection1").getParent().toString(); } @Before diff --git a/solr/modules/clustering/src/test/org/apache/solr/handler/clustering/ClusteringComponentTest.java b/solr/modules/clustering/src/test/org/apache/solr/handler/clustering/ClusteringComponentTest.java index 32422d78c3d..6ff1c63a859 100644 --- a/solr/modules/clustering/src/test/org/apache/solr/handler/clustering/ClusteringComponentTest.java +++ b/solr/modules/clustering/src/test/org/apache/solr/handler/clustering/ClusteringComponentTest.java @@ -17,10 +17,10 @@ package org.apache.solr.handler.clustering; import com.carrotsearch.randomizedtesting.RandomizedContext; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -30,7 +30,7 @@ import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; -import org.apache.commons.io.FileUtils; +import org.apache.commons.io.file.PathUtils; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.response.ClusteringResponse; import org.apache.solr.common.SolrDocument; @@ -55,9 +55,9 @@ public class ClusteringComponentTest extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() throws Exception { - File testHome = createTempDir().toFile(); - FileUtils.copyDirectory(getFile("clustering/solr"), testHome); - initCore("solrconfig.xml", "schema.xml", testHome.getAbsolutePath()); + Path testHome = createTempDir(); + PathUtils.copyDirectory(getFile("clustering/solr"), testHome); + initCore("solrconfig.xml", "schema.xml", testHome.toAbsolutePath().toString()); String[] languages = { "English", "French", "German", "Unknown", diff --git a/solr/modules/cross-dc/build.gradle b/solr/modules/cross-dc/build.gradle index dbd9c00c1bf..459cd46c34a 100644 --- a/solr/modules/cross-dc/build.gradle +++ b/solr/modules/cross-dc/build.gradle @@ -32,22 +32,26 @@ dependencies { implementation project(':solr:solrj') implementation project(':solr:solrj-zookeeper') - implementation 'org.slf4j:slf4j-api' - implementation 'org.apache.kafka:kafka-clients' - implementation 'com.google.guava:guava' - implementation 'io.dropwizard.metrics:metrics-core' - implementation 'org.apache.httpcomponents:httpclient' - implementation 'org.apache.zookeeper:zookeeper' - implementation 'org.apache.zookeeper:zookeeper-jute' + implementation libs.slf4j.api + implementation libs.apache.kafka.clients + implementation libs.google.guava + implementation libs.dropwizard.metrics.core + implementation libs.apache.httpcomponents.httpclient + implementation libs.apache.zookeeper.zookeeper + implementation libs.apache.zookeeper.jute testImplementation project(':solr:test-framework') - testImplementation 'org.apache.lucene:lucene-test-framework' - testImplementation 'junit:junit' - testImplementation 'commons-io:commons-io' - testImplementation 'org.mockito:mockito-core' - testImplementation 'com.carrotsearch.randomizedtesting:randomizedtesting-runner' + testImplementation libs.apache.lucene.testframework + testImplementation libs.junit.junit + testImplementation libs.commonsio.commonsio + testImplementation libs.mockito.core + testImplementation libs.carrotsearch.randomizedtesting.runner - testImplementation "org.apache.kafka:kafka-clients:3.7.1:test" + testImplementation(libs.apache.kafka.clients) { + artifact { + classifier = "test" + } + } } // Make sure that the cloud-minimal.zip artifact is up-to-date with cloud-minimal/conf diff --git a/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/common/MirroredSolrRequest.java b/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/common/MirroredSolrRequest.java index 125c8a46a10..53e2351cf47 100644 --- a/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/common/MirroredSolrRequest.java +++ b/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/common/MirroredSolrRequest.java @@ -267,8 +267,7 @@ public String toString() { sb.append(type.toString()); sb.append(", method=" + solrRequest.getMethod()); sb.append(", params=" + solrRequest.getParams()); - if (solrRequest instanceof UpdateRequest) { - UpdateRequest req = (UpdateRequest) solrRequest; + if (solrRequest instanceof UpdateRequest req) { sb.append(", add=" + (req.getDocuments() != null ? req.getDocuments().size() : "0")); sb.append(", del=" + (req.getDeleteByIdMap() != null ? req.getDeleteByIdMap().size() : "0")); sb.append(", dbq=" + (req.getDeleteQuery() != null ? req.getDeleteQuery().size() : "0")); diff --git a/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/common/MirroredSolrRequestSerializer.java b/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/common/MirroredSolrRequestSerializer.java index f661a2689e8..667b94ba5b5 100644 --- a/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/common/MirroredSolrRequestSerializer.java +++ b/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/common/MirroredSolrRequestSerializer.java @@ -184,14 +184,11 @@ public byte[] serialize(String topic, MirroredSolrRequest request) { map.put("submitTimeNanos", request.getSubmitTimeNanos()); map.put("params", solrRequest.getParams()); map.put("type", request.getType().toString()); - if (solrRequest instanceof UpdateRequest) { - UpdateRequest update = (UpdateRequest) solrRequest; + if (solrRequest instanceof UpdateRequest update) { map.put("docs", update.getDocuments()); map.put("deletes", update.getDeleteById()); map.put("deleteQuery", update.getDeleteQuery()); - } else if (solrRequest instanceof MirroredSolrRequest.MirroredConfigSetRequest) { - MirroredSolrRequest.MirroredConfigSetRequest config = - (MirroredSolrRequest.MirroredConfigSetRequest) solrRequest; + } else if (solrRequest instanceof MirroredSolrRequest.MirroredConfigSetRequest config) { map.put("method", config.getMethod().toString()); if (config.getContentStreams() != null) { List> streamsList = new ArrayList<>(); diff --git a/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/CrossDCProducerSolrStandaloneTest.java b/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/CrossDCProducerSolrStandaloneTest.java index d8d91832efa..c35adbc17ed 100644 --- a/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/CrossDCProducerSolrStandaloneTest.java +++ b/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/CrossDCProducerSolrStandaloneTest.java @@ -120,12 +120,12 @@ private static void populateCoreDirectory( Files.createFile(coreDirectory.resolve("core.properties")); // Copy "schema.xml" from the source location to the "conf" subdirectory - File sourceSchemaFile = getFile(Path.of(sourceLocation, "schema.xml").toString()); + File sourceSchemaFile = getFile(Path.of(sourceLocation, "schema.xml").toString()).toFile(); Path targetSchemaPath = subHome.resolve("schema.xml"); Files.copy(sourceSchemaFile.toPath(), targetSchemaPath, StandardCopyOption.REPLACE_EXISTING); // Copy solr config file from the source location to the "conf" subdirectory - File sourceConfigFile = getFile(Path.of(sourceLocation, solrConfigName).toString()); + File sourceConfigFile = getFile(Path.of(sourceLocation, solrConfigName).toString()).toFile(); Path targetConfigPath = subHome.resolve("solrconfig.xml"); Files.copy(sourceConfigFile.toPath(), targetConfigPath, StandardCopyOption.REPLACE_EXISTING); } diff --git a/solr/modules/extraction/build.gradle b/solr/modules/extraction/build.gradle index 952ebee0424..0202cd75910 100644 --- a/solr/modules/extraction/build.gradle +++ b/solr/modules/extraction/build.gradle @@ -23,22 +23,22 @@ dependencies { implementation project(':solr:core') implementation project(':solr:solrj') - implementation 'org.apache.lucene:lucene-core' - implementation 'org.slf4j:slf4j-api' + implementation libs.apache.lucene.core + implementation libs.slf4j.api - implementation 'org.apache.poi:poi' - implementation 'org.apache.poi:poi-ooxml' - implementation 'org.apache.tika:tika-core' - implementation ('org.apache.tika:tika-parsers', { + implementation libs.apache.poi.poi + implementation libs.apache.poi.ooxml + implementation libs.apache.tika.core + implementation (libs.apache.tika.parsers, { exclude group: 'org.apache.cxf', module: 'cxf-rt-rs-client' exclude group: 'org.quartz-scheduler', module: 'quartz' exclude group: 'xml-apis', module: 'xml-apis' }) - implementation ('xerces:xercesImpl', { + implementation (libs.xerces.impl, { exclude group: 'xml-apis', module: 'xml-apis' }) testImplementation project(':solr:test-framework') - testImplementation 'org.apache.lucene:lucene-test-framework' - testImplementation 'junit:junit' + testImplementation libs.apache.lucene.testframework + testImplementation libs.junit.junit } diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java index d68a86351ab..60a995b4d5d 100644 --- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java +++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java @@ -264,8 +264,7 @@ public void writeSolrDocument(String name, SolrDocument doc, ReturnFields return } else { // normalize to first value - if (val instanceof Collection) { - Collection values = (Collection) val; + if (val instanceof Collection values) { val = values.iterator().next(); } writeVal(xlField.name, val); @@ -285,8 +284,7 @@ public void writeArray(String name, Iterator val, boolean raw) throws IOExcep StringBuilder output = new StringBuilder(); while (val.hasNext()) { Object v = val.next(); - if (v instanceof IndexableField) { - IndexableField f = (IndexableField) v; + if (v instanceof IndexableField f) { if (v instanceof Date) { output.append(((Date) val).toInstant().toString()).append("; "); } else { diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java index ede60a108d3..7ad7ef1dc70 100644 --- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java +++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java @@ -52,7 +52,8 @@ public static void beforeClass() throws Exception { false); } - initCore("solrconfig.xml", "schema.xml", getFile("extraction/solr").getAbsolutePath()); + initCore( + "solrconfig.xml", "schema.xml", getFile("extraction/solr").toAbsolutePath().toString()); } @Override diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java index 1c95280cac6..4aa5f9eb09f 100644 --- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java +++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java @@ -47,7 +47,8 @@ public class TestXLSXResponseWriter extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() throws Exception { System.setProperty("enable.update.log", "false"); - initCore("solrconfig.xml", "schema.xml", getFile("extraction/solr").getAbsolutePath()); + initCore( + "solrconfig.xml", "schema.xml", getFile("extraction/solr").toAbsolutePath().toString()); createIndex(); // find a reference to the default response writer so we can redirect its output later SolrCore testCore = h.getCore(); diff --git a/solr/modules/gcs-repository/build.gradle b/solr/modules/gcs-repository/build.gradle index 5eec865fc9d..7a5b9372dc7 100644 --- a/solr/modules/gcs-repository/build.gradle +++ b/solr/modules/gcs-repository/build.gradle @@ -22,23 +22,24 @@ description = 'GCS Backup Repository' dependencies { api project(':solr:core') implementation project(':solr:solrj') - - implementation 'org.apache.lucene:lucene-core' - implementation 'org.slf4j:slf4j-api' - implementation platform(group: 'com.google.cloud', name: 'google-cloud-bom') - implementation 'com.google.api:gax' - implementation 'com.google.auth:google-auth-library-oauth2-http' - implementation 'com.google.auth:google-auth-library-credentials' - implementation 'com.google.cloud:google-cloud-core' - implementation 'com.google.cloud:google-cloud-core-http' - implementation 'com.google.cloud:google-cloud-storage' - implementation 'org.threeten:threetenbp' + implementation libs.apache.lucene.core + implementation libs.slf4j.api + + implementation platform(libs.google.cloud.bom) + implementation platform(libs.grpc.bom) + implementation libs.google.api.gax + implementation libs.google.auth.oauth2http + implementation libs.google.auth.credentials + implementation libs.google.cloud.core + implementation libs.google.cloud.corehttp + implementation libs.google.cloud.storage + implementation libs.threeten.bp testImplementation project(':solr:test-framework') - testImplementation 'org.apache.lucene:lucene-test-framework' - testImplementation 'com.carrotsearch.randomizedtesting:randomizedtesting-runner' - testImplementation 'junit:junit' + testImplementation libs.apache.lucene.testframework + testImplementation libs.carrotsearch.randomizedtesting.runner + testImplementation libs.junit.junit - testImplementation 'com.google.cloud:google-cloud-nio' + testImplementation libs.google.cloud.nio } diff --git a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSIncrementalBackupTest.java b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSIncrementalBackupTest.java index 4b145c601b4..e8300147322 100644 --- a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSIncrementalBackupTest.java +++ b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSIncrementalBackupTest.java @@ -73,7 +73,7 @@ public class GCSIncrementalBackupTest extends AbstractIncrementalBackupTest { public static void setupClass() throws Exception { configureCluster(NUM_NODES) // nodes - .addConfig("conf1", getFile("conf/solrconfig.xml").getParentFile().toPath()) + .addConfig("conf1", getFile("conf/solrconfig.xml").getParent()) .withSolrXml(SOLR_XML) .configure(); } diff --git a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSInstallShardTest.java b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSInstallShardTest.java index 3d0d30262ed..ecb08fa0192 100644 --- a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSInstallShardTest.java +++ b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSInstallShardTest.java @@ -52,7 +52,7 @@ public class GCSInstallShardTest extends AbstractInstallShardTest { public static void setupClass() throws Exception { configureCluster(1) // nodes - .addConfig("conf1", getFile("conf/solrconfig.xml").getParentFile().toPath()) + .addConfig("conf1", getFile("conf/solrconfig.xml").getParent()) .withSolrXml(SOLR_XML) .configure(); diff --git a/solr/modules/hadoop-auth/README.md b/solr/modules/hadoop-auth/README.md deleted file mode 100644 index 4771efbe9fd..00000000000 --- a/solr/modules/hadoop-auth/README.md +++ /dev/null @@ -1,46 +0,0 @@ - - -Apache Solr Hadoop Authentication Module -=============================== - -Introduction ------------- -This module implements the support for Hadoop Authentication in Apache Solr. - -Building --------- -The Hadoop Authentication module uses the same Gradle build as the core Solr components. - -To build the module, you can use - -``` -./gradlew :solr:modules:hadoop-auth:assemble -``` - -The resulting module will be placed to the libs directory, for example: -`solr/modules/hdfs/build/libs/solr-hadoop-auth-9.0.0-SNAPSHOT.jar` - -To execute the module tests: - -``` -./gradlew :solr:modules:hadoop-auth:test -``` - -Usage ------ -Please refer to the 'Hadoop Authentication Plugin' and 'Kerberos Authentication Plugin' sections of the reference guide: https://solr.apache.org/guide/solr/latest/deployment-guide/hadoop-authentication-plugin.html and https://solr.apache.org/guide/solr/latest/deployment-guide/kerberos-authentication-plugin.html diff --git a/solr/modules/hadoop-auth/build.gradle b/solr/modules/hadoop-auth/build.gradle deleted file mode 100644 index a038264f1f7..00000000000 --- a/solr/modules/hadoop-auth/build.gradle +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -apply plugin: 'java-library' - -description = 'Hadoop Authentication Module' - -dependencies { - // Spotbugs Annotations are only needed for old findbugs - // annotation usage like in Zookeeper during compilation time. - // It is not included in the release so exclude from checks. - testCompileOnly 'com.github.spotbugs:spotbugs-annotations' - permitUnusedDeclared 'com.github.spotbugs:spotbugs-annotations' - // Exclude these from jar validation and license checks. - configurations.jarValidation { - exclude group: "com.github.spotbugs", module: "spotbugs-annotations" - } - - implementation project(':solr:core') - implementation project(':solr:solrj') - implementation project(':solr:solrj-zookeeper') - - implementation 'org.slf4j:slf4j-api' - - api 'org.eclipse.jetty.toolchain:jetty-servlet-api' - - implementation 'com.fasterxml.jackson.core:jackson-core' - implementation 'com.google.guava:guava' - implementation 'io.dropwizard.metrics:metrics-core' - implementation 'org.apache.httpcomponents:httpclient' - implementation 'org.apache.httpcomponents:httpcore' - - implementation 'org.eclipse.jetty:jetty-client' - - // ZooKeeper & Curator - implementation('org.apache.zookeeper:zookeeper', { - exclude group: "org.apache.yetus", module: "audience-annotations" - }) - // required for instantiating a Zookeeper server (for embedding ZK or running tests) - runtimeOnly 'org.xerial.snappy:snappy-java' - - implementation('org.apache.curator:curator-framework', { - exclude group: 'org.apache.zookeeper', module: 'zookeeper' - }) - runtimeOnly('org.apache.curator:curator-recipes', { - exclude group: 'org.apache.zookeeper', module: 'zookeeper' - }) - - // Hadoop auth framework - implementation 'org.apache.hadoop:hadoop-annotations' - permitUnusedDeclared 'org.apache.hadoop:hadoop-annotations' - implementation ('org.apache.hadoop:hadoop-auth') { transitive = false } - implementation ('org.apache.hadoop:hadoop-common') { transitive = false } - // transitive of hadoop-common; used by Kerberos auth - runtimeOnly 'org.apache.hadoop.thirdparty:hadoop-shaded-guava' - runtimeOnly 'commons-collections:commons-collections' - runtimeOnly 'com.google.re2j:re2j' - runtimeOnly 'org.apache.commons:commons-configuration2' - runtimeOnly 'org.apache.kerby:kerb-core' - runtimeOnly 'org.apache.kerby:kerb-util' - - testImplementation project(':solr:test-framework') - testImplementation 'org.apache.lucene:lucene-test-framework' - testImplementation 'com.carrotsearch.randomizedtesting:randomizedtesting-runner' - testImplementation 'junit:junit' - testImplementation 'org.hamcrest:hamcrest' - - testImplementation('org.mockito:mockito-core', { - exclude group: "net.bytebuddy", module: "byte-buddy-agent" - }) - testRuntimeOnly('org.mockito:mockito-subclass', { - exclude group: "net.bytebuddy", module: "byte-buddy-agent" - }) - - testImplementation 'commons-io:commons-io' - - testImplementation 'org.apache.lucene:lucene-core' - - testImplementation project(':solr:solrj') - - // classes like solr.ICUCollationField, used by TestSolrCloudWithSecureImpersonation for example. - testRuntimeOnly project(':solr:modules:analysis-extras') - - // Hadoop MiniKdc Dependencies (for Kerberos auth tests) - testImplementation ('org.apache.hadoop:hadoop-minikdc', { - exclude group:'org.apache.kerby', module:'kerby-xdr' - exclude group:'org.apache.kerby', module:'token-provider' - exclude group:'org.slf4j', module:'slf4j-reload4j' - exclude group:'org.jline', module:'jline' - exclude group:'com.jcraft', module:'jsch' - }) - - // Zookeeper dependency - some tests like HdfsCloudBackupRestore need this - testImplementation('org.apache.curator:curator-client', { - exclude group: 'org.apache.zookeeper', module: 'zookeeper' - }) - testImplementation('org.apache.zookeeper:zookeeper', { - exclude group: "org.apache.yetus", module: "audience-annotations" - }) - testImplementation ('org.apache.zookeeper:zookeeper-jute') { - exclude group: 'org.apache.yetus', module: 'audience-annotations' - } - // required for instantiating a Zookeeper server in tests or embedded - testRuntimeOnly ('org.xerial.snappy:snappy-java') -} - - -// Copy all the test resource files from core to the build/resources/test directory -// of the Hadoop Authentication module so we can avoid duplication of the test -// resource files like schemas and SolrConfigs. This can be improved later by making -// the test classes load the resources from core directories directly. -tasks.register('copySolrCoreTestResources', Copy) { - from(project(':solr:core').sourceSets.test.resources.srcDirs) { - exclude '**/*.java' - } - into sourceSets.test.output.resourcesDir -} - -tasks.named('processTestResources').configure { - dependsOn copySolrCoreTestResources -} diff --git a/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/AttributeOnlyServletContext.java b/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/AttributeOnlyServletContext.java deleted file mode 100644 index e6ca2b57073..00000000000 --- a/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/AttributeOnlyServletContext.java +++ /dev/null @@ -1,318 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security.hadoop; - -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.Collections; -import java.util.Enumeration; -import java.util.EventListener; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import javax.servlet.Filter; -import javax.servlet.FilterRegistration; -import javax.servlet.FilterRegistration.Dynamic; -import javax.servlet.RequestDispatcher; -import javax.servlet.Servlet; -import javax.servlet.ServletContext; -import javax.servlet.ServletException; -import javax.servlet.ServletRegistration; -import javax.servlet.SessionCookieConfig; -import javax.servlet.SessionTrackingMode; -import javax.servlet.descriptor.JspConfigDescriptor; - -/** A concrete implementation of {@linkplain ServletContext} which support only attributes. */ -class AttributeOnlyServletContext implements ServletContext { - private Map attributes = new HashMap<>(); - - @Override - public void setSessionTrackingModes(Set sessionTrackingModes) {} - - @Override - public boolean setInitParameter(String name, String value) { - return false; - } - - @Override - public void setAttribute(String name, Object object) { - attributes.put(name, object); - } - - @Override - public void removeAttribute(String name) { - attributes.remove(name); - } - - @Override - public void log(String message, Throwable throwable) {} - - @Override - public void log(Exception exception, String msg) {} - - @Override - public void log(String msg) {} - - @Override - public String getVirtualServerName() { - return null; - } - - @Override - public int getSessionTimeout() { - return 0; - } - - @Override - public void setSessionTimeout(int sessionTimeout) {} - - @Override - public String getRequestCharacterEncoding() { - return null; - } - - @Override - public void setRequestCharacterEncoding(String encoding) {} - - @Override - public String getResponseCharacterEncoding() { - return null; - } - - @Override - public void setResponseCharacterEncoding(String encoding) {} - - @Override - public SessionCookieConfig getSessionCookieConfig() { - return null; - } - - @Override - public Enumeration getServlets() { - return null; - } - - @Override - public Map getServletRegistrations() { - return null; - } - - @Override - public ServletRegistration getServletRegistration(String servletName) { - return null; - } - - @Override - public Enumeration getServletNames() { - return null; - } - - @Override - public String getServletContextName() { - return null; - } - - @Override - public Servlet getServlet(String name) throws ServletException { - return null; - } - - @Override - public String getServerInfo() { - return null; - } - - @Override - public Set getResourcePaths(String path) { - return null; - } - - @Override - public InputStream getResourceAsStream(String path) { - return null; - } - - @Override - public URL getResource(String path) throws MalformedURLException { - return null; - } - - @Override - public RequestDispatcher getRequestDispatcher(String path) { - return null; - } - - @Override - public String getRealPath(String path) { - return null; - } - - @Override - public RequestDispatcher getNamedDispatcher(String name) { - return null; - } - - @Override - public int getMinorVersion() { - return 0; - } - - @Override - public String getMimeType(String file) { - return null; - } - - @Override - public int getMajorVersion() { - return 0; - } - - @Override - public JspConfigDescriptor getJspConfigDescriptor() { - return null; - } - - @Override - public Enumeration getInitParameterNames() { - return null; - } - - @Override - public String getInitParameter(String name) { - return null; - } - - @Override - public Map getFilterRegistrations() { - return null; - } - - @Override - public FilterRegistration getFilterRegistration(String filterName) { - return null; - } - - @Override - public Set getEffectiveSessionTrackingModes() { - return null; - } - - @Override - public int getEffectiveMinorVersion() { - return 0; - } - - @Override - public int getEffectiveMajorVersion() { - return 0; - } - - @Override - public Set getDefaultSessionTrackingModes() { - return null; - } - - @Override - public String getContextPath() { - return null; - } - - @Override - public ServletContext getContext(String uripath) { - return null; - } - - @Override - public ClassLoader getClassLoader() { - return null; - } - - @Override - public Enumeration getAttributeNames() { - return Collections.enumeration(attributes.keySet()); - } - - @Override - public Object getAttribute(String name) { - return attributes.get(name); - } - - @Override - public void declareRoles(String... roleNames) {} - - @Override - public T createServlet(Class clazz) throws ServletException { - return null; - } - - @Override - public T createListener(Class clazz) throws ServletException { - return null; - } - - @Override - public T createFilter(Class clazz) throws ServletException { - return null; - } - - @Override - public javax.servlet.ServletRegistration.Dynamic addServlet( - String servletName, Class servletClass) { - return null; - } - - @Override - public ServletRegistration.Dynamic addJspFile(String servletName, String jspFile) { - return null; - } - - @Override - public javax.servlet.ServletRegistration.Dynamic addServlet(String servletName, Servlet servlet) { - return null; - } - - @Override - public javax.servlet.ServletRegistration.Dynamic addServlet( - String servletName, String className) { - return null; - } - - @Override - public void addListener(Class listenerClass) {} - - @Override - public void addListener(T t) {} - - @Override - public void addListener(String className) {} - - @Override - public Dynamic addFilter(String filterName, Class filterClass) { - return null; - } - - @Override - public Dynamic addFilter(String filterName, Filter filter) { - return null; - } - - @Override - public Dynamic addFilter(String filterName, String className) { - return null; - } -} diff --git a/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/ConfigurableInternodeAuthHadoopPlugin.java b/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/ConfigurableInternodeAuthHadoopPlugin.java deleted file mode 100644 index 35f59105012..00000000000 --- a/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/ConfigurableInternodeAuthHadoopPlugin.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security.hadoop; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.net.URISyntaxException; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.BiConsumer; -import org.apache.http.HttpRequest; -import org.apache.http.client.methods.HttpRequestWrapper; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.protocol.HttpContext; -import org.apache.solr.client.solrj.impl.Http2SolrClient; -import org.apache.solr.client.solrj.impl.HttpClientBuilderFactory; -import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.request.SolrRequestInfo; -import org.apache.solr.security.HttpClientBuilderPlugin; -import org.apache.solr.servlet.SolrDispatchFilter; -import org.eclipse.jetty.client.api.Request; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This class extends {@linkplain HadoopAuthPlugin} by enabling configuration of authentication - * mechanism for Solr internal communication. - */ -public class ConfigurableInternodeAuthHadoopPlugin extends HadoopAuthPlugin - implements HttpClientBuilderPlugin { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - /** - * A property specifying the {@linkplain HttpClientBuilderFactory} used for the Solr internal - * communication. - */ - private static final String HTTPCLIENT_BUILDER_FACTORY = "clientBuilderFactory"; - - private static final String DO_AS = "doAs"; - - private HttpClientBuilderFactory factory = null; - - public ConfigurableInternodeAuthHadoopPlugin(CoreContainer coreContainer) { - super(coreContainer); - } - - @Override - public void init(Map pluginConfig) { - super.init(pluginConfig); - - String httpClientBuilderFactory = - (String) - Objects.requireNonNull( - pluginConfig.get(HTTPCLIENT_BUILDER_FACTORY), - "Please specify clientBuilderFactory to be used for Solr internal communication."); - factory = - this.coreContainer - .getResourceLoader() - .newInstance(httpClientBuilderFactory, HttpClientBuilderFactory.class); - } - - @Override - public void setup(Http2SolrClient client) { - factory.setup(client); - } - - @Override - public SolrHttpClientBuilder getHttpClientBuilder(SolrHttpClientBuilder builder) { - return factory.getHttpClientBuilder(builder); - } - - @Override - public void close() throws IOException { - super.close(); - - if (factory != null) { - factory.close(); - } - } - - @Override - public boolean interceptInternodeRequest(HttpRequest httpRequest, HttpContext httpContext) { - if (!(httpRequest instanceof HttpRequestWrapper)) { - log.warn("Unable to add doAs to forwarded/distributed request - unknown request type"); - return false; - } - AtomicBoolean success = new AtomicBoolean(false); - return intercept( - (key, value) -> { - HttpRequestWrapper request = (HttpRequestWrapper) httpRequest; - URIBuilder uriBuilder = new URIBuilder(request.getURI()); - uriBuilder.setParameter(key, value); - try { - request.setURI(uriBuilder.build()); - success.set(true); - } catch (URISyntaxException e) { - log.warn("Unable to add doAs to forwarded/distributed request - bad URI"); - } - }) - && success.get(); - } - - @Override - protected boolean interceptInternodeRequest(Request request) { - return intercept(request::param); - } - - private boolean intercept(BiConsumer setParam) { - SolrRequestInfo info = SolrRequestInfo.getRequestInfo(); - if (info != null - && (info.getAction() == SolrDispatchFilter.Action.FORWARD - || info.getAction() == SolrDispatchFilter.Action.REMOTEQUERY)) { - if (info.getUserPrincipal() != null) { - String name = info.getUserPrincipal().getName(); - log.debug("Setting doAs={} to forwarded/remote request", name); - setParam.accept(DO_AS, name); - return true; - } - } - return false; - } -} diff --git a/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/DelegationTokenKerberosFilter.java b/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/DelegationTokenKerberosFilter.java deleted file mode 100644 index 318ad72e7e7..00000000000 --- a/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/DelegationTokenKerberosFilter.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security.hadoop; - -import static org.apache.hadoop.security.token.delegation.ZKDelegationTokenSecretManager.ZK_DTSM_ZNODE_WORKING_PATH; -import static org.apache.hadoop.security.token.delegation.ZKDelegationTokenSecretManager.ZK_DTSM_ZNODE_WORKING_PATH_DEAFULT; - -import java.io.IOException; -import java.util.Enumeration; -import java.util.Objects; -import javax.servlet.FilterChain; -import javax.servlet.FilterConfig; -import javax.servlet.ServletException; -import javax.servlet.ServletRequest; -import javax.servlet.ServletResponse; -import javax.servlet.http.HttpServletRequest; -import org.apache.curator.framework.CuratorFramework; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authentication.server.AuthenticationHandler; -import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter; -import org.apache.hadoop.security.token.delegation.web.HttpUserGroupInformation; -import org.apache.solr.common.cloud.SecurityAwareZkACLProvider; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; - -/** - * This is an authentication filter based on Hadoop's {@link DelegationTokenAuthenticationFilter}. - * The Kerberos plugin can be configured to use delegation tokens, which allow an application to - * reuse the authentication of an end-user or another application. - */ -public class DelegationTokenKerberosFilter extends DelegationTokenAuthenticationFilter { - @Override - public void init(FilterConfig conf) throws ServletException { - if (conf != null && "zookeeper".equals(conf.getInitParameter("signer.secret.provider"))) { - SolrZkClient zkClient = - (SolrZkClient) - conf.getServletContext().getAttribute(KerberosPlugin.DELEGATION_TOKEN_ZK_CLIENT); - try { - conf.getServletContext() - .setAttribute( - "signer.secret.provider.zookeeper.curator.client", - getCuratorClientInternal(conf, zkClient)); - } catch (InterruptedException | KeeperException e) { - throw new ServletException(e); - } - } - super.init(conf); - } - - /** - * Return the ProxyUser Configuration. FilterConfig properties beginning with - * "solr.impersonator.user.name" will be added to the configuration. - */ - @Override - protected Configuration getProxyuserConfiguration(FilterConfig filterConf) { - Configuration conf = new Configuration(false); - - Enumeration names = filterConf.getInitParameterNames(); - while (names.hasMoreElements()) { - String name = names.nextElement(); - if (name.startsWith(KerberosPlugin.IMPERSONATOR_PREFIX)) { - String value = filterConf.getInitParameter(name); - conf.set( - PROXYUSER_PREFIX + "." + name.substring(KerberosPlugin.IMPERSONATOR_PREFIX.length()), - value); - conf.set(name, value); - } - } - return conf; - } - - @Override - public void doFilter(ServletRequest request, ServletResponse response, FilterChain filterChain) - throws IOException, ServletException { - // include Impersonator User Name in case someone (e.g. logger) wants it - FilterChain filterChainWrapper = - new FilterChain() { - @Override - public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse) - throws IOException, ServletException { - HttpServletRequest httpRequest = (HttpServletRequest) servletRequest; - - UserGroupInformation ugi = HttpUserGroupInformation.get(); - if (ugi != null - && ugi.getAuthenticationMethod() - == UserGroupInformation.AuthenticationMethod.PROXY) { - UserGroupInformation realUserUgi = ugi.getRealUser(); - if (realUserUgi != null) { - httpRequest.setAttribute( - KerberosPlugin.IMPERSONATOR_USER_NAME, realUserUgi.getShortUserName()); - } - } - filterChain.doFilter(servletRequest, servletResponse); - } - }; - - super.doFilter(request, response, filterChainWrapper); - } - - @Override - protected void initializeAuthHandler(String authHandlerClassName, FilterConfig filterConfig) - throws ServletException { - // set the internal authentication handler in order to record whether the request should - // continue - super.initializeAuthHandler(authHandlerClassName, filterConfig); - AuthenticationHandler authHandler = getAuthenticationHandler(); - super.initializeAuthHandler( - RequestContinuesRecorderAuthenticationHandler.class.getName(), filterConfig); - RequestContinuesRecorderAuthenticationHandler newAuthHandler = - (RequestContinuesRecorderAuthenticationHandler) getAuthenticationHandler(); - newAuthHandler.setAuthHandler(authHandler); - } - - private CuratorFramework getCuratorClientInternal(FilterConfig conf, SolrZkClient zkClient) - throws KeeperException, InterruptedException { - // There is a race condition where the znodeWorking path used by ZKDelegationTokenSecretManager - // can be created by multiple nodes, but Hadoop doesn't handle this well. This explicitly - // creates it up front and handles if the znode already exists. This relates to HADOOP-18452 - // but didn't solve the underlying issue of the race condition. - - // If namespace parents are implicitly created, they won't have ACLs. - // So, let's explicitly create them. - CuratorFramework curatorFramework = getCuratorClient(zkClient); - CuratorFramework nullNsFw = curatorFramework.usingNamespace(null); - try { - String znodeWorkingPath = - '/' - + Objects.requireNonNullElse( - conf.getInitParameter(ZK_DTSM_ZNODE_WORKING_PATH), - ZK_DTSM_ZNODE_WORKING_PATH_DEAFULT) - + "/ZKDTSMRoot"; - nullNsFw.create().creatingParentContainersIfNeeded().forPath(znodeWorkingPath); - } catch (Exception ignore) { - } - - return curatorFramework; - } - - protected CuratorFramework getCuratorClient(SolrZkClient zkClient) - throws InterruptedException, KeeperException { - // Create /security znode upfront. Without this, the curator framework creates this directory - // path - // without the appropriate ACL configuration. This issue is possibly related to HADOOP-11973 - try { - zkClient.makePath( - SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH, CreateMode.PERSISTENT, true); - } catch (KeeperException.NodeExistsException ex) { - // ignore? - } - - // Note - Curator complains if the namespace starts with / - String namespace = zkClient.getAbsolutePath(SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH); - namespace = namespace.startsWith("/") ? namespace.substring(1) : namespace; - - return zkClient.getCuratorFramework().usingNamespace(namespace); - } -} diff --git a/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/HadoopAuthFilter.java b/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/HadoopAuthFilter.java deleted file mode 100644 index 25bd363d3ad..00000000000 --- a/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/HadoopAuthFilter.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security.hadoop; - -import static org.apache.hadoop.security.token.delegation.ZKDelegationTokenSecretManager.ZK_DTSM_ZNODE_WORKING_PATH; -import static org.apache.hadoop.security.token.delegation.ZKDelegationTokenSecretManager.ZK_DTSM_ZNODE_WORKING_PATH_DEAFULT; - -import java.io.IOException; -import java.util.Objects; -import javax.servlet.FilterChain; -import javax.servlet.FilterConfig; -import javax.servlet.ServletException; -import javax.servlet.ServletRequest; -import javax.servlet.ServletResponse; -import javax.servlet.http.HttpServletRequest; -import org.apache.curator.framework.CuratorFramework; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authentication.server.AuthenticationHandler; -import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter; -import org.apache.hadoop.security.token.delegation.web.HttpUserGroupInformation; -import org.apache.solr.common.cloud.SecurityAwareZkACLProvider; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; - -/** - * This is an authentication filter based on Hadoop's {@link DelegationTokenAuthenticationFilter}. - */ -public class HadoopAuthFilter extends DelegationTokenAuthenticationFilter { - /** - * This property defines the configuration parameter storing the Solr zookeeper client ref in the - * servlet filter config. - */ - static final String DELEGATION_TOKEN_ZK_CLIENT = "solr.kerberos.delegation.token.zk.client"; - - @Override - public void init(FilterConfig conf) throws ServletException { - if (conf != null && "zookeeper".equals(conf.getInitParameter("signer.secret.provider"))) { - SolrZkClient zkClient = - (SolrZkClient) conf.getServletContext().getAttribute(DELEGATION_TOKEN_ZK_CLIENT); - try { - conf.getServletContext() - .setAttribute( - "signer.secret.provider.zookeeper.curator.client", - getCuratorClientInternal(conf, zkClient)); - } catch (KeeperException | InterruptedException e) { - throw new ServletException(e); - } - } - super.init(conf); - } - - @Override - public void doFilter(ServletRequest request, ServletResponse response, FilterChain filterChain) - throws IOException, ServletException { - // include Impersonator User Name in case someone (e.g. logger) wants it - FilterChain filterChainWrapper = - new FilterChain() { - @Override - public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse) - throws IOException, ServletException { - HttpServletRequest httpRequest = (HttpServletRequest) servletRequest; - - UserGroupInformation ugi = HttpUserGroupInformation.get(); - if (ugi != null - && ugi.getAuthenticationMethod() - == UserGroupInformation.AuthenticationMethod.PROXY) { - UserGroupInformation realUserUgi = ugi.getRealUser(); - if (realUserUgi != null) { - httpRequest.setAttribute( - KerberosPlugin.IMPERSONATOR_USER_NAME, realUserUgi.getShortUserName()); - } - } - filterChain.doFilter(servletRequest, servletResponse); - } - }; - - super.doFilter(request, response, filterChainWrapper); - } - - @Override - public void destroy() { - super.destroy(); - } - - @Override - protected void initializeAuthHandler(String authHandlerClassName, FilterConfig filterConfig) - throws ServletException { - // set the internal authentication handler in order to record whether the request should - // continue - super.initializeAuthHandler(authHandlerClassName, filterConfig); - AuthenticationHandler authHandler = getAuthenticationHandler(); - super.initializeAuthHandler( - RequestContinuesRecorderAuthenticationHandler.class.getName(), filterConfig); - RequestContinuesRecorderAuthenticationHandler newAuthHandler = - (RequestContinuesRecorderAuthenticationHandler) getAuthenticationHandler(); - newAuthHandler.setAuthHandler(authHandler); - } - - private CuratorFramework getCuratorClientInternal(FilterConfig conf, SolrZkClient zkClient) - throws KeeperException, InterruptedException { - // There is a race condition where the znodeWorking path used by ZKDelegationTokenSecretManager - // can be created by multiple nodes, but Hadoop doesn't handle this well. This explicitly - // creates it up front and handles if the znode already exists. This relates to HADOOP-18452 - // but didn't solve the underlying issue of the race condition. - - // If namespace parents are implicitly created, they won't have ACLs. - // So, let's explicitly create them. - CuratorFramework curatorFramework = getCuratorClient(zkClient); - CuratorFramework nullNsFw = curatorFramework.usingNamespace(null); - try { - String znodeWorkingPath = - '/' - + Objects.requireNonNullElse( - conf.getInitParameter(ZK_DTSM_ZNODE_WORKING_PATH), - ZK_DTSM_ZNODE_WORKING_PATH_DEAFULT) - + "/ZKDTSMRoot"; - nullNsFw.create().creatingParentContainersIfNeeded().forPath(znodeWorkingPath); - } catch (Exception ignore) { - } - - return curatorFramework; - } - - protected CuratorFramework getCuratorClient(SolrZkClient zkClient) - throws InterruptedException, KeeperException { - // Create /security znode upfront. Without this, the curator framework creates this directory - // path - // without the appropriate ACL configuration. This issue is possibly related to HADOOP-11973 - try { - zkClient.makePath( - SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH, CreateMode.PERSISTENT, true); - } catch (KeeperException.NodeExistsException ex) { - // ignore? - } - - // Note - Curator complains if the namespace starts with / - String namespace = zkClient.getAbsolutePath(SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH); - namespace = namespace.startsWith("/") ? namespace.substring(1) : namespace; - - return zkClient.getCuratorFramework().usingNamespace(namespace); - } -} diff --git a/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/HadoopAuthPlugin.java b/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/HadoopAuthPlugin.java deleted file mode 100644 index b103ee6919e..00000000000 --- a/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/HadoopAuthPlugin.java +++ /dev/null @@ -1,300 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security.hadoop; - -import static org.apache.solr.security.hadoop.HadoopAuthFilter.DELEGATION_TOKEN_ZK_CLIENT; -import static org.apache.solr.security.hadoop.RequestContinuesRecorderAuthenticationHandler.REQUEST_CONTINUES_ATTR; - -import com.fasterxml.jackson.core.JsonGenerator; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.Collection; -import java.util.Collections; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import javax.servlet.FilterChain; -import javax.servlet.FilterConfig; -import javax.servlet.ServletContext; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import org.apache.hadoop.security.authentication.server.AuthenticationFilter; -import org.apache.hadoop.security.authentication.util.ZKSignerSecretProvider; -import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler; -import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder; -import org.apache.solr.cloud.ZkController; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.util.EnvUtils; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.security.AuthenticationPlugin; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This class implements a generic plugin which can use authentication schemes exposed by the Hadoop - * framework. This plugin supports following features - integration with authentication mechanisms - * (e.g. kerberos) - Delegation token support - Proxy users (or secure impersonation) support - * - *

    This plugin enables defining configuration parameters required by the underlying Hadoop - * authentication mechanism. These configuration parameters can either be specified as a Java system - * property or the default value can be specified as part of the plugin configuration. - * - *

    The proxy users are configured by specifying relevant Hadoop configuration parameters. Please - * note that the delegation token support must be enabled for using the proxy users support. - * - *

    Note - this class does not support configuring authentication mechanism for Solr internal - * communication. For this purpose {@linkplain ConfigurableInternodeAuthHadoopPlugin} should be - * used. If this plugin is used in the SolrCloud mode, it will use PKI based authentication - * mechanism for Solr internal communication. - */ -public class HadoopAuthPlugin extends AuthenticationPlugin { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - /** A property specifying the type of authentication scheme to be configured. */ - private static final String HADOOP_AUTH_TYPE = "type"; - - /** - * A property specifies the value of the prefix to be used to define Java system property for - * configuring the authentication mechanism. The name of the Java system property is defined by - * appending the configuration parmeter namne to this prefix value e.g. if prefix is 'solr' then - * the Java system property 'solr.kerberos.principal' defines the value of configuration parameter - * 'kerberos.principal'. - */ - private static final String SYSPROP_PREFIX_PROPERTY = "sysPropPrefix"; - - /** - * A property specifying the configuration parameters required by the authentication scheme - * defined by {@linkplain #HADOOP_AUTH_TYPE} property. - */ - private static final String AUTH_CONFIG_NAMES_PROPERTY = "authConfigs"; - - /** - * A property specifying the default values for the configuration parameters specified by the - * {@linkplain #AUTH_CONFIG_NAMES_PROPERTY} property. The default values are specified as a - * collection of key-value pairs (i.e. property-name : default_value). - */ - private static final String DEFAULT_AUTH_CONFIGS_PROPERTY = "defaultConfigs"; - - /** A property which enable (or disable) the delegation tokens functionality. */ - private static final String DELEGATION_TOKEN_ENABLED_PROPERTY = "enableDelegationToken"; - - /** A property which enables initialization of kerberos before connecting to Zookeeper. */ - private static final String INIT_KERBEROS_ZK = "initKerberosZk"; - - /** - * A property which configures proxy users for the underlying Hadoop authentication mechanism. - * This configuration is expressed as a collection of key-value pairs (i.e. property-name : - * value). - */ - public static final String PROXY_USER_CONFIGS = "proxyUserConfigs"; - - /** - * This parameter is used to debug the authentication related issues during development. This - * should not be used in production. - */ - private static final boolean TRACE_HTTP = Boolean.getBoolean("hadoopauth.tracehttp"); - - private AuthenticationFilter authFilter; - protected final CoreContainer coreContainer; - private boolean delegationTokenEnabled; - - public HadoopAuthPlugin(CoreContainer coreContainer) { - this.coreContainer = coreContainer; - } - - @Override - public void init(Map pluginConfig) { - try { - delegationTokenEnabled = - Boolean.parseBoolean((String) pluginConfig.get(DELEGATION_TOKEN_ENABLED_PROPERTY)); - authFilter = delegationTokenEnabled ? new HadoopAuthFilter() : new AuthenticationFilter(); - - // Initialize kerberos before initializing curator instance. - boolean initKerberosZk = - Boolean.parseBoolean((String) pluginConfig.getOrDefault(INIT_KERBEROS_ZK, "false")); - if (initKerberosZk) { - (new Krb5HttpClientBuilder()).getBuilder(); - } - - FilterConfig conf = getInitFilterConfig(pluginConfig); - authFilter.init(conf); - - } catch (ServletException e) { - log.error("Error initializing {}", getClass().getSimpleName(), e); - throw new SolrException( - ErrorCode.SERVER_ERROR, "Error initializing " + getClass().getName() + ": " + e); - } - } - - @SuppressWarnings("unchecked") - protected FilterConfig getInitFilterConfig(Map pluginConfig) { - Map params = new HashMap<>(); - - String type = (String) Objects.requireNonNull(pluginConfig.get(HADOOP_AUTH_TYPE)); - params.put(HADOOP_AUTH_TYPE, type); - - String sysPropPrefix = (String) pluginConfig.getOrDefault(SYSPROP_PREFIX_PROPERTY, "solr."); - Collection authConfigNames = - (Collection) - pluginConfig.getOrDefault(AUTH_CONFIG_NAMES_PROPERTY, Collections.emptyList()); - Map authConfigDefaults = - (Map) - pluginConfig.getOrDefault(DEFAULT_AUTH_CONFIGS_PROPERTY, Collections.emptyMap()); - Map proxyUserConfigs = - (Map) pluginConfig.getOrDefault(PROXY_USER_CONFIGS, Collections.emptyMap()); - - for (String configName : authConfigNames) { - String systemProperty = sysPropPrefix + configName; - String defaultConfigVal = authConfigDefaults.get(configName); - String configVal = EnvUtils.getProperty(systemProperty, defaultConfigVal); - if (configVal != null) { - params.put(configName, configVal); - } - } - if (delegationTokenEnabled) { - // This is the only kind we support right now anyway - params.putIfAbsent( - "delegation-token.token-kind", KerberosPlugin.DELEGATION_TOKEN_TYPE_DEFAULT); - } - - // Configure proxy user settings. - params.putAll(proxyUserConfigs); - - // Needed to work around HADOOP-13346 - params.put( - DelegationTokenAuthenticationHandler.JSON_MAPPER_PREFIX - + JsonGenerator.Feature.AUTO_CLOSE_TARGET, - "false"); - - final ServletContext servletContext = new AttributeOnlyServletContext(); - if (log.isInfoEnabled()) { - log.info("Params: {}", params); - } - - ZkController controller = coreContainer.getZkController(); - if (controller != null) { - servletContext.setAttribute(DELEGATION_TOKEN_ZK_CLIENT, controller.getZkClient()); - params.put(ZKSignerSecretProvider.DISCONNECT_FROM_ZOOKEEPER_ON_SHUTDOWN, "false"); - } - - FilterConfig conf = - new FilterConfig() { - @Override - public ServletContext getServletContext() { - return servletContext; - } - - @Override - public Enumeration getInitParameterNames() { - return Collections.enumeration(params.keySet()); - } - - @Override - public String getInitParameter(String param) { - return params.get(param); - } - - @Override - public String getFilterName() { - return "HadoopAuthFilter"; - } - }; - - return conf; - } - - @Override - public boolean doAuthenticate( - HttpServletRequest request, HttpServletResponse response, FilterChain filterChain) - throws Exception { - if (TRACE_HTTP) { - log.info("----------HTTP Request---------"); - if (log.isInfoEnabled()) { - log.info("{} : {}", request.getMethod(), request.getRequestURI()); - log.info("Query : {}", request.getQueryString()); // nowarn - } - log.info("Headers :"); - Enumeration headers = request.getHeaderNames(); - while (headers.hasMoreElements()) { - String name = headers.nextElement(); - Enumeration hvals = request.getHeaders(name); - while (hvals.hasMoreElements()) { - if (log.isInfoEnabled()) { - log.info("{} : {}", name, hvals.nextElement()); - } - } - } - log.info("-------------------------------"); - } - - authFilter.doFilter(request, response, filterChain); - - switch (response.getStatus()) { - case HttpServletResponse.SC_UNAUTHORIZED: - // Cannot tell whether the 401 is due to wrong or missing credentials - numWrongCredentials.inc(); - break; - - case HttpServletResponse.SC_FORBIDDEN: - // Are there other status codes which should also translate to error? - numErrors.mark(); - break; - default: - if (response.getStatus() >= 200 && response.getStatus() <= 299) { - numAuthenticated.inc(); - } else { - numErrors.mark(); - } - } - - if (TRACE_HTTP) { - log.info("----------HTTP Response---------"); - if (log.isInfoEnabled()) { - log.info("Status : {}", response.getStatus()); - } - log.info("Headers :"); - for (String name : response.getHeaderNames()) { - for (String value : response.getHeaders(name)) { - log.info("{} : {}", name, value); - } - } - log.info("-------------------------------"); - } - - if (authFilter instanceof HadoopAuthFilter) { // delegation token mgmt. - String requestContinuesAttr = (String) request.getAttribute(REQUEST_CONTINUES_ATTR); - if (requestContinuesAttr == null) { - log.warn("Could not find {}", REQUEST_CONTINUES_ATTR); - return false; - } else { - return Boolean.parseBoolean(requestContinuesAttr); - } - } - - return true; - } - - @Override - public void close() throws IOException { - if (authFilter != null) { - authFilter.destroy(); - } - } -} diff --git a/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/KerberosFilter.java b/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/KerberosFilter.java deleted file mode 100644 index 436342bd484..00000000000 --- a/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/KerberosFilter.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security.hadoop; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.security.Principal; -import javax.servlet.FilterChain; -import javax.servlet.FilterConfig; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletRequestWrapper; -import javax.servlet.http.HttpServletResponse; -import org.apache.hadoop.security.authentication.server.AuthenticationFilter; -import org.apache.hadoop.security.authentication.server.AuthenticationHandler; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.security.AuthorizationPlugin; -import org.apache.solr.security.PermissionNameProvider; -import org.apache.solr.security.RuleBasedAuthorizationPlugin; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class KerberosFilter extends AuthenticationFilter { - private final CoreContainer coreContainer; - - public KerberosFilter(CoreContainer coreContainer) { - this.coreContainer = coreContainer; - } - - @Override - public void init(FilterConfig conf) throws ServletException { - super.init(conf); - } - - @Override - protected void initializeAuthHandler(String authHandlerClassName, FilterConfig filterConfig) - throws ServletException { - // set the internal authentication handler in order to record whether the request should - // continue - super.initializeAuthHandler(authHandlerClassName, filterConfig); - AuthenticationHandler authHandler = getAuthenticationHandler(); - super.initializeAuthHandler( - RequestContinuesRecorderAuthenticationHandler.class.getName(), filterConfig); - RequestContinuesRecorderAuthenticationHandler newAuthHandler = - (RequestContinuesRecorderAuthenticationHandler) getAuthenticationHandler(); - newAuthHandler.setAuthHandler(authHandler); - } - - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - @Override - protected void doFilter( - FilterChain filterChain, HttpServletRequest request, HttpServletResponse response) - throws IOException, ServletException { - request = substituteOriginalUserRequest(request); - - super.doFilter(filterChain, request, response); - } - - /** - * If principal is an admin user, i.e. has ALL permissions (e.g. request coming from Solr node), - * and "originalUserPrincipal" is specified, then set originalUserPrincipal as the principal. This - * is the case in forwarded/remote requests through KerberosPlugin. This is needed because the - * original node that received this request did not perform any authorization, and hence we are - * the first ones to authorize the request (and we need the original user principal to do so). - * - * @return Substituted request, if applicable, or the original request - */ - private HttpServletRequest substituteOriginalUserRequest(HttpServletRequest request) { - final HttpServletRequest originalRequest = request; - AuthorizationPlugin authzPlugin = coreContainer.getAuthorizationPlugin(); - if (authzPlugin instanceof RuleBasedAuthorizationPlugin) { - RuleBasedAuthorizationPlugin ruleBased = (RuleBasedAuthorizationPlugin) authzPlugin; - if (request.getHeader(KerberosPlugin.ORIGINAL_USER_PRINCIPAL_HEADER) != null - && ruleBased.doesUserHavePermission( - request.getUserPrincipal(), PermissionNameProvider.Name.ALL)) { - request = - new HttpServletRequestWrapper(request) { - @Override - public Principal getUserPrincipal() { - String originalUserPrincipal = - originalRequest.getHeader(KerberosPlugin.ORIGINAL_USER_PRINCIPAL_HEADER); - if (log.isInfoEnabled()) { - log.info( - "Substituting user principal from {} to {}.", - originalRequest.getUserPrincipal(), - originalUserPrincipal); - } - return new Principal() { - @Override - public String getName() { - return originalUserPrincipal; - } - - @Override - public String toString() { - return originalUserPrincipal; - } - }; - } - }; - } - } - return request; - } -} diff --git a/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/KerberosPlugin.java b/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/KerberosPlugin.java deleted file mode 100644 index 66b9b90dee2..00000000000 --- a/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/KerberosPlugin.java +++ /dev/null @@ -1,347 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security.hadoop; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.google.common.annotations.VisibleForTesting; -import java.lang.invoke.MethodHandles; -import java.util.Collections; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.Map; -import java.util.function.BiConsumer; -import javax.servlet.Filter; -import javax.servlet.FilterChain; -import javax.servlet.FilterConfig; -import javax.servlet.ServletContext; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import org.apache.hadoop.security.authentication.util.KerberosName; -import org.apache.hadoop.security.authentication.util.ZKSignerSecretProvider; -import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler; -import org.apache.http.HttpRequest; -import org.apache.http.protocol.HttpContext; -import org.apache.solr.client.solrj.impl.Http2SolrClient; -import org.apache.solr.client.solrj.impl.HttpListenerFactory; -import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder; -import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder; -import org.apache.solr.cloud.ZkController; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.cloud.SecurityAwareZkACLProvider; -import org.apache.solr.common.util.EnvUtils; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.request.SolrRequestInfo; -import org.apache.solr.security.AuthenticationPlugin; -import org.apache.solr.security.HttpClientBuilderPlugin; -import org.apache.solr.servlet.SolrDispatchFilter; -import org.eclipse.jetty.client.api.Request; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class KerberosPlugin extends AuthenticationPlugin implements HttpClientBuilderPlugin { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - Krb5HttpClientBuilder kerberosBuilder = new Krb5HttpClientBuilder(); - private Filter kerberosFilter; - - public static final String NAME_RULES_PARAM = "solr.kerberos.name.rules"; - public static final String NAME_RULES_MECHANISM_PARAM = "solr.kerberos.name.rules.mechanism"; - public static final String COOKIE_DOMAIN_PARAM = "solr.kerberos.cookie.domain"; - public static final String COOKIE_PATH_PARAM = "solr.kerberos.cookie.path"; - public static final String PRINCIPAL_PARAM = "solr.kerberos.principal"; - public static final String KEYTAB_PARAM = "solr.kerberos.keytab"; - public static final String TOKEN_VALID_PARAM = "solr.kerberos.token.valid"; - public static final String COOKIE_PORT_AWARE_PARAM = "solr.kerberos.cookie.portaware"; - public static final String IMPERSONATOR_PREFIX = "solr.kerberos.impersonator.user."; - public static final String DELEGATION_TOKEN_ENABLED = "solr.kerberos.delegation.token.enabled"; - public static final String DELEGATION_TOKEN_KIND = "solr.kerberos.delegation.token.kind"; - public static final String DELEGATION_TOKEN_VALIDITY = "solr.kerberos.delegation.token.validity"; - public static final String DELEGATION_TOKEN_SECRET_PROVIDER = - "solr.kerberos.delegation.token.signer.secret.provider"; - public static final String DELEGATION_TOKEN_SECRET_PROVIDER_ZK_PATH = - "solr.kerberos.delegation.token.signer.secret.provider.zookeper.path"; - public static final String DELEGATION_TOKEN_SECRET_MANAGER_ZNODE_WORKING_PATH = - "solr.kerberos.delegation.token.secret.manager.znode.working.path"; - - public static final String DELEGATION_TOKEN_TYPE_DEFAULT = "solr-dt"; - public static final String IMPERSONATOR_DO_AS_HTTP_PARAM = "doAs"; - public static final String IMPERSONATOR_USER_NAME = "solr.impersonator.user.name"; - - public static final String ORIGINAL_USER_PRINCIPAL_HEADER = "originalUserPrincipal"; - - static final String DELEGATION_TOKEN_ZK_CLIENT = "solr.kerberos.delegation.token.zk.client"; - - private final CoreContainer coreContainer; - - public KerberosPlugin(CoreContainer coreContainer) { - this.coreContainer = coreContainer; - } - - @Override - public void init(Map pluginConfig) { - try { - FilterConfig conf = getInitFilterConfig(pluginConfig, false); - kerberosFilter.init(conf); - } catch (ServletException e) { - throw new SolrException( - ErrorCode.SERVER_ERROR, "Error initializing kerberos authentication plugin", e); - } - } - - @VisibleForTesting - protected FilterConfig getInitFilterConfig( - Map pluginConfig, boolean skipKerberosChecking) { - Map params = new HashMap<>(); - params.put("type", "kerberos"); - putParam(params, "kerberos.name.rules", NAME_RULES_PARAM, "DEFAULT"); - putParam( - params, - "kerberos.name.rules.mechanism", - NAME_RULES_MECHANISM_PARAM, - KerberosName.DEFAULT_MECHANISM); - putParam(params, "token.valid", TOKEN_VALID_PARAM, "30"); - putParam(params, "cookie.path", COOKIE_PATH_PARAM, "/"); - if (!skipKerberosChecking) { - putParam(params, "kerberos.principal", PRINCIPAL_PARAM, null); - putParam(params, "kerberos.keytab", KEYTAB_PARAM, null); - } else { - putParamOptional(params, "kerberos.principal", PRINCIPAL_PARAM); - putParamOptional(params, "kerberos.keytab", KEYTAB_PARAM); - } - - boolean delegationTokenEnabled = Boolean.getBoolean(DELEGATION_TOKEN_ENABLED); - ZkController controller = coreContainer.getZkController(); - - if (delegationTokenEnabled) { - putParam( - params, - "delegation-token.token-kind", - DELEGATION_TOKEN_KIND, - DELEGATION_TOKEN_TYPE_DEFAULT); - if (coreContainer.isZooKeeperAware()) { - putParam(params, "signer.secret.provider", DELEGATION_TOKEN_SECRET_PROVIDER, "zookeeper"); - if ("zookeeper".equals(params.get("signer.secret.provider"))) { - String zkHost = controller.getZkServerAddress(); - putParam(params, "token.validity", DELEGATION_TOKEN_VALIDITY, "36000"); - params.put("zk-dt-secret-manager.enable", "true"); - - String chrootPath = zkHost.contains("/") ? zkHost.substring(zkHost.indexOf('/')) : ""; - String znodeWorkingPath = - chrootPath + SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH + "/zkdtsm"; - // Note - Curator complains if the znodeWorkingPath starts with / - znodeWorkingPath = - znodeWorkingPath.startsWith("/") ? znodeWorkingPath.substring(1) : znodeWorkingPath; - putParam( - params, - "zk-dt-secret-manager.znodeWorkingPath", - DELEGATION_TOKEN_SECRET_MANAGER_ZNODE_WORKING_PATH, - znodeWorkingPath); - putParam( - params, - "signer.secret.provider.zookeeper.path", - DELEGATION_TOKEN_SECRET_PROVIDER_ZK_PATH, - "/token"); - // ensure krb5 is setup properly before running curator - getHttpClientBuilder(SolrHttpClientBuilder.create()); - - params.put(ZKSignerSecretProvider.DISCONNECT_FROM_ZOOKEEPER_ON_SHUTDOWN, "false"); - } - } else { - log.info( - "CoreContainer is not ZooKeeperAware, not setting ZK-related delegation token properties"); - } - } - - // Special handling for the "cookie.domain" based on whether port should be - // appended to the domain. Useful for situations where multiple solr nodes are - // on the same host. - String usePortStr = EnvUtils.getProperty(COOKIE_PORT_AWARE_PARAM, null); - boolean needPortAwareCookies = (usePortStr == null) ? false : Boolean.parseBoolean(usePortStr); - - if (!needPortAwareCookies || !coreContainer.isZooKeeperAware()) { - putParam(params, "cookie.domain", COOKIE_DOMAIN_PARAM, null); - } else { // we need port aware cookies and we are in SolrCloud mode. - String host = EnvUtils.getProperty(COOKIE_DOMAIN_PARAM, null); - if (host == null) { - throw new SolrException( - ErrorCode.SERVER_ERROR, "Missing required parameter '" + COOKIE_DOMAIN_PARAM + "'."); - } - int port = controller.getHostPort(); - params.put("cookie.domain", host + ":" + port); - } - - // check impersonator config - EnvUtils.getProperties() - .forEach( - (key, value) -> { - if (key.startsWith(IMPERSONATOR_PREFIX)) { - if (!delegationTokenEnabled) { - throw new SolrException( - ErrorCode.SERVER_ERROR, - "Impersonator configuration requires delegation tokens to be enabled: " - + key); - } - params.put(key, value); - } - }); - - // Needed to work around HADOOP-13346 - params.put( - DelegationTokenAuthenticationHandler.JSON_MAPPER_PREFIX - + JsonGenerator.Feature.AUTO_CLOSE_TARGET, - "false"); - - final ServletContext servletContext = new AttributeOnlyServletContext(); - if (controller != null) { - servletContext.setAttribute(DELEGATION_TOKEN_ZK_CLIENT, controller.getZkClient()); - } - if (delegationTokenEnabled) { - kerberosFilter = new DelegationTokenKerberosFilter(); - // pass an attribute-enabled context in order to pass the zkClient - // and because the filter may pass a curator instance. - } else { - kerberosFilter = new KerberosFilter(coreContainer); - } - log.info("Params: {}", params); - - FilterConfig conf = - new FilterConfig() { - @Override - public ServletContext getServletContext() { - return servletContext; - } - - @Override - public Enumeration getInitParameterNames() { - return Collections.enumeration(params.keySet()); - } - - @Override - public String getInitParameter(String param) { - return params.get(param); - } - - @Override - public String getFilterName() { - return "KerberosFilter"; - } - }; - - return conf; - } - - private void putParam( - Map params, - String internalParamName, - String externalParamName, - String defaultValue) { - String value = EnvUtils.getProperty(externalParamName, defaultValue); - if (value == null) { - throw new SolrException( - ErrorCode.SERVER_ERROR, "Missing required parameter '" + externalParamName + "'."); - } - params.put(internalParamName, value); - } - - private void putParamOptional( - Map params, String internalParamName, String externalParamName) { - String value = EnvUtils.getProperty(externalParamName); - if (value != null) { - params.put(internalParamName, value); - } - } - - @Override - public boolean doAuthenticate(HttpServletRequest req, HttpServletResponse rsp, FilterChain chain) - throws Exception { - log.debug("Request to authenticate using kerberos: {}", req); - kerberosFilter.doFilter(req, rsp, chain); - - String requestContinuesAttr = - (String) - req.getAttribute(RequestContinuesRecorderAuthenticationHandler.REQUEST_CONTINUES_ATTR); - if (requestContinuesAttr == null) { - log.warn( - "Could not find {}", - RequestContinuesRecorderAuthenticationHandler.REQUEST_CONTINUES_ATTR); - return false; - } else { - return Boolean.parseBoolean(requestContinuesAttr); - } - } - - @Override - public boolean interceptInternodeRequest(HttpRequest httpRequest, HttpContext httpContext) { - return intercept(httpRequest::setHeader); - } - - @Override - protected boolean interceptInternodeRequest(Request request) { - return intercept(request::header); - } - - private boolean intercept(BiConsumer header) { - SolrRequestInfo info = SolrRequestInfo.getRequestInfo(); - if (info != null - && (info.getAction() == SolrDispatchFilter.Action.FORWARD - || info.getAction() == SolrDispatchFilter.Action.REMOTEQUERY)) { - if (info.getUserPrincipal() != null) { - if (log.isInfoEnabled()) { - log.info("Setting original user principal: {}", info.getUserPrincipal().getName()); - } - header.accept(ORIGINAL_USER_PRINCIPAL_HEADER, info.getUserPrincipal().getName()); - return true; - } - } - return false; - } - - @Override - public SolrHttpClientBuilder getHttpClientBuilder(SolrHttpClientBuilder builder) { - return kerberosBuilder.getBuilder(builder); - } - - @Override - public void setup(Http2SolrClient client) { - final HttpListenerFactory.RequestResponseListener listener = - new HttpListenerFactory.RequestResponseListener() { - @Override - public void onQueued(Request request) { - interceptInternodeRequest(request); - } - }; - client.addListenerFactory(() -> listener); - - kerberosBuilder.setup(client); - } - - @Override - public void close() { - kerberosFilter.destroy(); - kerberosBuilder.close(); - } - - protected Filter getKerberosFilter() { - return kerberosFilter; - } - - protected void setKerberosFilter(Filter kerberosFilter) { - this.kerberosFilter = kerberosFilter; - } -} diff --git a/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/RequestContinuesRecorderAuthenticationHandler.java b/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/RequestContinuesRecorderAuthenticationHandler.java deleted file mode 100644 index 2d18ddbf605..00000000000 --- a/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/RequestContinuesRecorderAuthenticationHandler.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security.hadoop; - -import java.io.IOException; -import java.util.Properties; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.apache.hadoop.security.authentication.server.AuthenticationHandler; -import org.apache.hadoop.security.authentication.server.AuthenticationToken; - -/** - * {@link AuthenticationHandler} that delegates to another {@link AuthenticationHandler} and records - * the response of managementOperation (which indicates whether the request should continue or not). - */ -public class RequestContinuesRecorderAuthenticationHandler implements AuthenticationHandler { - // filled in by Plugin/Filter - static final String REQUEST_CONTINUES_ATTR = - "org.apache.solr.security.authentication.requestcontinues"; - - private AuthenticationHandler authHandler; - - public void setAuthHandler(AuthenticationHandler authHandler) { - this.authHandler = authHandler; - } - - @Override - public String getType() { - return authHandler.getType(); - } - - @Override - public void init(Properties config) throws ServletException { - // authHandler has already been init'ed, nothing to do here - } - - @Override - public void destroy() { - authHandler.destroy(); - } - - @Override - public boolean managementOperation( - AuthenticationToken token, HttpServletRequest request, HttpServletResponse response) - throws IOException, AuthenticationException { - boolean result = authHandler.managementOperation(token, request, response); - request.setAttribute( - RequestContinuesRecorderAuthenticationHandler.REQUEST_CONTINUES_ATTR, - Boolean.toString(result)); - return result; - } - - @Override - public AuthenticationToken authenticate(HttpServletRequest request, HttpServletResponse response) - throws IOException, AuthenticationException { - return authHandler.authenticate(request, response); - } -} diff --git a/solr/modules/hadoop-auth/src/java/org/apache/solr/util/configuration/providers/hadoop/HadoopSSLCredentialProvider.java b/solr/modules/hadoop-auth/src/java/org/apache/solr/util/configuration/providers/hadoop/HadoopSSLCredentialProvider.java deleted file mode 100644 index b1926dad454..00000000000 --- a/solr/modules/hadoop-auth/src/java/org/apache/solr/util/configuration/providers/hadoop/HadoopSSLCredentialProvider.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.util.configuration.providers.hadoop; - -import static org.apache.hadoop.security.alias.CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.EnumMap; -import org.apache.hadoop.conf.Configuration; -import org.apache.solr.common.util.EnvUtils; -import org.apache.solr.common.util.StrUtils; -import org.apache.solr.util.configuration.providers.AbstractSSLCredentialProvider; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** System property based SSL configuration provider */ -public class HadoopSSLCredentialProvider extends AbstractSSLCredentialProvider { - - private Configuration hadoopConfigurationProvider; - - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public HadoopSSLCredentialProvider() { - this(new Configuration()); - } - - public HadoopSSLCredentialProvider(Configuration hadoopConfigurationProvider) { - if (StrUtils.isNullOrEmpty(EnvUtils.getProperty(CREDENTIAL_PROVIDER_PATH))) { - throw new RuntimeException( - "Cannot initialize Hadoop configuration provider without credential provider path. Use " - + CREDENTIAL_PROVIDER_PATH - + " system property to configure."); - } - this.hadoopConfigurationProvider = hadoopConfigurationProvider; - hadoopConfigurationProvider.set( - CREDENTIAL_PROVIDER_PATH, EnvUtils.getProperty(CREDENTIAL_PROVIDER_PATH)); - } - - @Override - protected EnumMap getCredentialKeyMap() { - return DEFAULT_CREDENTIAL_KEY_MAP; - } - - @Override - protected String getCredential(String keystoreKey) { - try { - char[] password = hadoopConfigurationProvider.getPassword(keystoreKey); - return password == null ? null : String.valueOf(password); - } catch (IOException e) { - log.error("Could not read password from Hadoop Credential Store: {}", keystoreKey, e); - return null; - } - } -} diff --git a/solr/modules/hadoop-auth/src/test-files/krb5-template.conf b/solr/modules/hadoop-auth/src/test-files/krb5-template.conf deleted file mode 100644 index 4fdd43718c4..00000000000 --- a/solr/modules/hadoop-auth/src/test-files/krb5-template.conf +++ /dev/null @@ -1,12 +0,0 @@ -[libdefaults] - kdc_realm = _REALM_ - default_realm = _REALM_ - udp_preference_limit = _UDP_LIMIT_ - allow_weak_crypto = true - dns_canonicalize_hostname = false - #_KDC_TCP_PORT_ - #_KDC_UDP_PORT_ -[realms] - _REALM_ = { - kdc = localhost:_KDC_PORT_ - } diff --git a/solr/modules/hadoop-auth/src/test-files/solr/security/hadoop_kerberos_authz_config.json b/solr/modules/hadoop-auth/src/test-files/solr/security/hadoop_kerberos_authz_config.json deleted file mode 100644 index 8799fed5fd1..00000000000 --- a/solr/modules/hadoop-auth/src/test-files/solr/security/hadoop_kerberos_authz_config.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "authentication": { - "class": "solr.ConfigurableInternodeAuthHadoopPlugin", - "sysPropPrefix": "solr.", - "type": "kerberos", - "clientBuilderFactory": "org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder", - "enableDelegationToken": "true", - "initKerberosZk": "true", - "authConfigs": [ - "kerberos.principal", - "kerberos.keytab", - "kerberos.name.rules" - ], - "defaultConfigs": { - }, - "proxyUserConfigs": { - "proxyuser.solr.hosts": "*", - "proxyuser.solr.groups": "*" - } - }, - "authorization":{ - "class":"solr.RuleBasedAuthorizationPlugin", - "useShortName": "true", - "permissions":[ - { - "name": "read", - "role": "alt_role", - "collection": "public" - }, - { - "name":"read", - "role":"admin" - }, - { - "name":"update", - "role":"admin" - } - ], - "user-role": { - "solr":"admin", - "solr_alt": "alt_role" - } - } -} diff --git a/solr/modules/hadoop-auth/src/test-files/solr/security/hadoop_kerberos_config.json b/solr/modules/hadoop-auth/src/test-files/solr/security/hadoop_kerberos_config.json deleted file mode 100644 index a25b7061175..00000000000 --- a/solr/modules/hadoop-auth/src/test-files/solr/security/hadoop_kerberos_config.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "authentication": { - "class": "solr.ConfigurableInternodeAuthHadoopPlugin", - "sysPropPrefix": "solr.", - "type": "kerberos", - "clientBuilderFactory": "org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder", - "initKerberosZk": "true", - "authConfigs": [ - "kerberos.principal", - "kerberos.keytab", - "kerberos.name.rules" - ], - "defaultConfigs": { - } - } -} diff --git a/solr/modules/hadoop-auth/src/test-files/solr/security/hadoop_simple_auth_with_delegation.json b/solr/modules/hadoop-auth/src/test-files/solr/security/hadoop_simple_auth_with_delegation.json deleted file mode 100644 index c06c58a05ee..00000000000 --- a/solr/modules/hadoop-auth/src/test-files/solr/security/hadoop_simple_auth_with_delegation.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "authentication": { - "class": "solr.HadoopAuthPlugin", - "sysPropPrefix": "solr.", - "type": "simple", - "enableDelegationToken":"true", - "authConfigs": [ - "delegation-token.token-kind", - "delegation-token.update-interval.sec", - "delegation-token.max-lifetime.sec", - "delegation-token.renewal-interval.sec", - "delegation-token.removal-scan-interval.sec", - "cookie.domain", - "signer.secret.provider", - "zk-dt-secret-manager.enable", - "zk-dt-secret-manager.znodeWorkingPath", - "signer.secret.provider.zookeeper.path" - ], - "defaultConfigs": { - "delegation-token.token-kind": "solr-dt", - "signer.secret.provider": "zookeeper", - "zk-dt-secret-manager.enable": "true", - "token.validity": "36000", - "zk-dt-secret-manager.znodeWorkingPath": "solr/security/zkdtsm", - "signer.secret.provider.zookeeper.path": "/token", - "cookie.domain": "127.0.0.1" - } - } -} diff --git a/solr/modules/hadoop-auth/src/test/org/apache/solr/client/solrj/impl/Krb5HttpClientUtils.java b/solr/modules/hadoop-auth/src/test/org/apache/solr/client/solrj/impl/Krb5HttpClientUtils.java deleted file mode 100644 index 8f78c5035fb..00000000000 --- a/solr/modules/hadoop-auth/src/test/org/apache/solr/client/solrj/impl/Krb5HttpClientUtils.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.client.solrj.impl; - -import java.lang.invoke.MethodHandles; -import java.net.URI; -import javax.security.auth.login.AppConfigurationEntry; -import javax.security.auth.login.Configuration; -import org.apache.solr.common.util.EnvUtils; -import org.eclipse.jetty.client.HttpAuthenticationStore; -import org.eclipse.jetty.client.WWWAuthenticationProtocolHandler; -import org.eclipse.jetty.client.util.SPNEGOAuthentication; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * All of this is a clone of Krb5HttpClientBuilder to hardcode the user-principal for a unit test - */ -public class Krb5HttpClientUtils { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private static Configuration jaasConfig = new Krb5HttpClientBuilder.SolrJaasConfiguration(); - - public static void setup(Http2SolrClient http2Client, String principalName) { - HttpAuthenticationStore authenticationStore = new HttpAuthenticationStore(); - authenticationStore.addAuthentication(createSPNEGOAuthentication(principalName)); - http2Client - .getProtocolHandlers() - .put(new WWWAuthenticationProtocolHandler(http2Client.getHttpClient())); - http2Client.setAuthenticationStore(authenticationStore); - } - - private static SPNEGOAuthentication createSPNEGOAuthentication(String principalName) { - SPNEGOAuthentication authentication = - new SPNEGOAuthentication(null) { - - @Override - public boolean matches(String type, URI uri, String realm) { - return this.getType().equals(type); - } - }; - String clientAppName = EnvUtils.getProperty("solr.kerberos.jaas.appname", "Client"); - AppConfigurationEntry[] entries = jaasConfig.getAppConfigurationEntry(clientAppName); - if (entries == null) { - log.warn( - "Could not find login configuration entry for {}. SPNego authentication may not be successful.", - (Object) clientAppName); - return authentication; - } - if (entries.length != 1) { - log.warn("Multiple login modules are specified in the configuration file"); - return authentication; - } - - Krb5HttpClientBuilder.setAuthenticationOptions( - authentication, entries[0].getOptions(), principalName); - return authentication; - } -} diff --git a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/HadoopTestUtil.java b/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/HadoopTestUtil.java deleted file mode 100644 index 2b1955b55f9..00000000000 --- a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/HadoopTestUtil.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.security.hadoop; - -import java.lang.invoke.MethodHandles; -import org.apache.hadoop.io.nativeio.NativeIO; -import org.apache.lucene.util.Constants; -import org.apache.solr.SolrTestCase; -import org.apache.solr.common.util.EnvUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class HadoopTestUtil { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public static void checkAssumptions() { - ensureHadoopHomeNotSet(); - checkHadoopWindows(); - } - - /** - * If Hadoop home is set via environment variable HADOOP_HOME or Java system property - * hadoop.home.dir, the behavior of test is undefined. Ensure that these are not set before - * starting. It is not possible to easily unset environment variables so better to bail out early - * instead of trying to test. - */ - protected static void ensureHadoopHomeNotSet() { - if (System.getenv("HADOOP_HOME") != null) { - SolrTestCase.fail("Ensure that HADOOP_HOME environment variable is not set."); - } - if (EnvUtils.getProperty("hadoop.home.dir") != null) { - SolrTestCase.fail("Ensure that \"hadoop.home.dir\" Java property is not set."); - } - } - - /** Hadoop integration tests fail on Windows without Hadoop NativeIO */ - protected static void checkHadoopWindows() { - SolrTestCase.assumeTrue( - "Hadoop does not work on Windows without Hadoop NativeIO", - !Constants.WINDOWS || NativeIO.isAvailable()); - } -} diff --git a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/HttpParamDelegationTokenPlugin.java b/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/HttpParamDelegationTokenPlugin.java deleted file mode 100644 index 58212f8b8ba..00000000000 --- a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/HttpParamDelegationTokenPlugin.java +++ /dev/null @@ -1,297 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security.hadoop; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.security.Principal; -import java.util.ArrayList; -import java.util.Enumeration; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Properties; -import javax.servlet.Filter; -import javax.servlet.FilterChain; -import javax.servlet.FilterConfig; -import javax.servlet.ServletContext; -import javax.servlet.ServletException; -import javax.servlet.ServletRequest; -import javax.servlet.ServletResponse; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletRequestWrapper; -import javax.servlet.http.HttpServletResponse; -import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.apache.hadoop.security.authentication.server.AuthenticationFilter; -import org.apache.hadoop.security.authentication.server.AuthenticationHandler; -import org.apache.hadoop.security.authentication.server.AuthenticationToken; -import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler; -import org.apache.http.HttpException; -import org.apache.http.HttpRequest; -import org.apache.http.HttpRequestInterceptor; -import org.apache.http.NameValuePair; -import org.apache.http.client.utils.URLEncodedUtils; -import org.apache.http.protocol.HttpContext; -import org.apache.solr.client.solrj.impl.Http2SolrClient; -import org.apache.solr.client.solrj.impl.HttpClientUtil; -import org.apache.solr.client.solrj.impl.HttpListenerFactory; -import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.util.ExecutorUtil; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.request.SolrRequestInfo; -import org.eclipse.jetty.client.api.Request; - -/** - * AuthenticationHandler that supports delegation tokens and simple authentication via the "user" - * http parameter - */ -public class HttpParamDelegationTokenPlugin extends KerberosPlugin { - public static final String USER_PARAM = "user"; // http parameter for user authentication - public static final String REMOTE_HOST_PARAM = - "remoteHost"; // http parameter for indicating remote host - public static final String REMOTE_ADDRESS_PARAM = - "remoteAddress"; // http parameter for indicating remote address - public static final String INTERNAL_REQUEST_HEADER = - "internalRequest"; // http header for indicating internal request - - boolean isSolrThread() { - return ExecutorUtil.isSolrServerThread(); - } - - private final HttpRequestInterceptor interceptor = - new HttpRequestInterceptor() { - @Override - public void process(HttpRequest httpRequest, HttpContext httpContext) - throws HttpException, IOException { - getPrincipal().ifPresent(usr -> httpRequest.setHeader(INTERNAL_REQUEST_HEADER, usr)); - } - }; - - public HttpParamDelegationTokenPlugin(CoreContainer coreContainer) { - super(coreContainer); - } - - @Override - public void init(Map pluginConfig) { - try { - final FilterConfig initConf = getInitFilterConfig(pluginConfig, true); - - FilterConfig conf = - new FilterConfig() { - @Override - public ServletContext getServletContext() { - return initConf.getServletContext(); - } - - @Override - public Enumeration getInitParameterNames() { - return initConf.getInitParameterNames(); - } - - @Override - public String getInitParameter(String param) { - if (AuthenticationFilter.AUTH_TYPE.equals(param)) { - return HttpParamDelegationTokenAuthenticationHandler.class.getName(); - } - return initConf.getInitParameter(param); - } - - @Override - public String getFilterName() { - return "HttpParamFilter"; - } - }; - Filter kerberosFilter = new HttpParamToRequestFilter(); - kerberosFilter.init(conf); - setKerberosFilter(kerberosFilter); - } catch (ServletException e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Error initializing kerberos authentication plugin: " + e); - } - } - - private Optional getPrincipal() { - SolrRequestInfo reqInfo = SolrRequestInfo.getRequestInfo(); - String usr; - if (reqInfo != null) { - Principal principal = reqInfo.getUserPrincipal(); - if (principal == null) { - // this had a request but not authenticated - // so we don't not need to set a principal - return Optional.empty(); - } else { - usr = principal.getName(); - } - } else { - if (!isSolrThread()) { - // if this is not running inside a Solr threadpool (as in testcases) - // then no need to add any header - return Optional.empty(); - } - // this request seems to be originated from Solr itself - usr = "$"; // special name to denote the user is the node itself - } - return Optional.of(usr); - } - - @Override - public void setup(Http2SolrClient client) { - final HttpListenerFactory.RequestResponseListener listener = - new HttpListenerFactory.RequestResponseListener() { - @Override - public void onQueued(Request request) { - getPrincipal().ifPresent(usr -> request.header(INTERNAL_REQUEST_HEADER, usr)); - } - }; - client.addListenerFactory(() -> listener); - } - - @Override - public SolrHttpClientBuilder getHttpClientBuilder(SolrHttpClientBuilder builder) { - HttpClientUtil.addRequestInterceptor(interceptor); - builder = super.getHttpClientBuilder(builder); - return builder; - } - - @Override - public void close() { - HttpClientUtil.removeRequestInterceptor(interceptor); - super.close(); - } - - private static String getHttpParam(HttpServletRequest request, String param) { - List pairs = - URLEncodedUtils.parse(request.getQueryString(), StandardCharsets.UTF_8); - for (NameValuePair nvp : pairs) { - if (param.equals(nvp.getName())) { - return nvp.getValue(); - } - } - return null; - } - - public static class HttpParamDelegationTokenAuthenticationHandler - extends DelegationTokenAuthenticationHandler { - - public HttpParamDelegationTokenAuthenticationHandler() { - super(new HttpParamAuthenticationHandler()); - } - - @Override - public void init(Properties config) throws ServletException { - Properties conf = new Properties(); - for (Map.Entry entry : config.entrySet()) { - conf.setProperty((String) entry.getKey(), (String) entry.getValue()); - } - conf.setProperty(TOKEN_KIND, KerberosPlugin.DELEGATION_TOKEN_TYPE_DEFAULT); - super.init(conf); - } - - private static class HttpParamAuthenticationHandler implements AuthenticationHandler { - @Override - public String getType() { - return "dummy"; - } - - @Override - public void init(Properties config) throws ServletException {} - - @Override - public void destroy() {} - - @Override - public boolean managementOperation( - AuthenticationToken token, HttpServletRequest request, HttpServletResponse response) - throws IOException, AuthenticationException { - return false; - } - - @Override - public AuthenticationToken authenticate( - HttpServletRequest request, HttpServletResponse response) - throws IOException, AuthenticationException { - AuthenticationToken token = null; - String userName = getHttpParam(request, USER_PARAM); - if (userName == null) { - // check if this is an internal request - userName = request.getHeader(INTERNAL_REQUEST_HEADER); - } - if (userName != null) { - return new AuthenticationToken(userName, userName, "test"); - } else { - response.setStatus(HttpServletResponse.SC_UNAUTHORIZED); - response.setHeader("WWW-Authenticate", "dummy"); - } - return token; - } - } - } - - /** Filter that converts http params to HttpServletRequest params */ - private static class HttpParamToRequestFilter extends DelegationTokenKerberosFilter { - @Override - public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) - throws IOException, ServletException { - final HttpServletRequest httpRequest = (HttpServletRequest) request; - final HttpServletRequestWrapper requestWrapper = - new HttpServletRequestWrapper(httpRequest) { - @Override - public String getRemoteHost() { - String param = getHttpParam(httpRequest, REMOTE_HOST_PARAM); - return param != null ? param : httpRequest.getRemoteHost(); - } - - @Override - public String getRemoteAddr() { - String param = getHttpParam(httpRequest, REMOTE_ADDRESS_PARAM); - return param != null ? param : httpRequest.getRemoteAddr(); - } - }; - - super.doFilter(requestWrapper, response, chain); - } - - @Override - protected void doFilter( - FilterChain filterChain, HttpServletRequest request, HttpServletResponse response) - throws IOException, ServletException { - // remove the filter-specific authentication information, so it doesn't get accidentally - // forwarded. - List newPairs = new ArrayList<>(); - List pairs = - URLEncodedUtils.parse(request.getQueryString(), StandardCharsets.UTF_8); - for (NameValuePair nvp : pairs) { - if (!USER_PARAM.equals(nvp.getName())) { - newPairs.add(nvp); - } else { - request.setAttribute(USER_PARAM, nvp.getValue()); - } - } - final String queryStringNoUser = URLEncodedUtils.format(newPairs, StandardCharsets.UTF_8); - HttpServletRequest requestWrapper = - new HttpServletRequestWrapper(request) { - @Override - public String getQueryString() { - return queryStringNoUser; - } - }; - super.doFilter(filterChain, requestWrapper, response); - } - } -} diff --git a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/ImpersonationUtil.java b/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/ImpersonationUtil.java deleted file mode 100644 index d8ef9bd0c24..00000000000 --- a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/ImpersonationUtil.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security.hadoop; - -import java.util.List; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.authentication.client.PseudoAuthenticator; -import org.apache.lucene.util.Constants; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.response.CollectionAdminResponse; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.EnvUtils; - -/** - * This class implements utility functions required to test the secure impersonation feature for - * {@linkplain HadoopAuthPlugin} - */ -public class ImpersonationUtil { - - static String getUsersFirstGroup() throws Exception { - String group = "*"; // accept any group if a group can't be found - if (!Constants.WINDOWS) { // does not work on Windows! - org.apache.hadoop.security.Groups hGroups = - new org.apache.hadoop.security.Groups(new Configuration()); - try { - List g = hGroups.getGroups(EnvUtils.getProperty("user.name")); - if (g != null && g.size() > 0) { - group = g.get(0); - } - } catch (NullPointerException npe) { - // if user/group doesn't exist on test box - } - } - return group; - } - - static SolrRequest getProxyRequest(String user, String doAs) { - return new CollectionAdminRequest.List() { - @Override - public SolrParams getParams() { - ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); - params.set(PseudoAuthenticator.USER_NAME, user); - params.set(KerberosPlugin.IMPERSONATOR_DO_AS_HTTP_PARAM, doAs); - return params; - } - }; - } - - static String getExpectedGroupExMsg(String user, String doAs) { - return "User: " + user + " is not allowed to impersonate " + doAs; - } - - static String getExpectedHostExMsg(String user) { - return "Unauthorized connection for super-user: " + user; - } -} diff --git a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/ImpersonatorCollectionsHandler.java b/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/ImpersonatorCollectionsHandler.java deleted file mode 100644 index 4295b2f2d96..00000000000 --- a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/ImpersonatorCollectionsHandler.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security.hadoop; - -import java.util.concurrent.atomic.AtomicBoolean; -import javax.servlet.http.HttpServletRequest; -import org.apache.hadoop.security.authentication.client.PseudoAuthenticator; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.handler.admin.CollectionsHandler; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; -import org.junit.Assert; - -/** - * This class extends {@linkplain CollectionsHandler} and implements extra validations for verifying - * proxy users support in {@linkplain HadoopAuthPlugin} - */ -public class ImpersonatorCollectionsHandler extends CollectionsHandler { - static AtomicBoolean called = new AtomicBoolean(false); - - public ImpersonatorCollectionsHandler() { - super(); - } - - public ImpersonatorCollectionsHandler(final CoreContainer coreContainer) { - super(coreContainer); - } - - @Override - public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { - called.set(true); - super.handleRequestBody(req, rsp); - String doAs = req.getParams().get(KerberosPlugin.IMPERSONATOR_DO_AS_HTTP_PARAM); - if (doAs != null) { - HttpServletRequest httpRequest = (HttpServletRequest) req.getContext().get("httpRequest"); - Assert.assertNotNull(httpRequest); - String user = req.getParams().get(PseudoAuthenticator.USER_NAME); - Assert.assertNotNull(user); - Assert.assertEquals(user, httpRequest.getAttribute(KerberosPlugin.IMPERSONATOR_USER_NAME)); - } - } -} diff --git a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/KerberosTestServices.java b/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/KerberosTestServices.java deleted file mode 100644 index 9d205f0f5f1..00000000000 --- a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/KerberosTestServices.java +++ /dev/null @@ -1,299 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security.hadoop; - -import java.io.File; -import java.lang.invoke.MethodHandles; -import java.net.BindException; -import java.nio.file.Path; -import java.util.HashMap; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.Properties; -import javax.security.auth.login.AppConfigurationEntry; -import javax.security.auth.login.Configuration; -import javax.security.auth.login.LoginContext; -import javax.security.auth.login.LoginException; -import org.apache.commons.io.file.PathUtils; -import org.apache.hadoop.minikdc.MiniKdc; -import org.apache.lucene.tests.util.LuceneTestCase; -import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder; -import org.apache.solr.client.solrj.util.Constants; -import org.apache.solr.common.util.EnvUtils; -import org.junit.Assume; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class KerberosTestServices { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private volatile MiniKdc kdc; - private final JaasConfiguration jaasConfiguration; - private final Configuration savedConfig; - private final boolean debug; - - private final File workDir; - - public static void assumeMiniKdcWorksWithDefaultLocale() throws Exception { - final String principal = "server"; - - Path kdcDir = LuceneTestCase.createTempDir().resolve("miniKdc"); - File keytabFile = kdcDir.resolve("keytabs").toFile(); - - Properties conf = MiniKdc.createConf(); - conf.setProperty("kdc.port", "0"); - MiniKdc kdc = new MiniKdc(conf, kdcDir.toFile()); - kdc.start(); - kdc.createPrincipal(keytabFile, principal); - - AppConfigurationEntry appConfigEntry = - new AppConfigurationEntry( - KerberosTestServices.krb5LoginModuleName, - AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, - Map.of( - "principal", - principal, - "storeKey", - "true", - "useKeyTab", - "true", - "useTicketCache", - "false", - "refreshKrb5Config", - "true", - "keyTab", - keytabFile.getAbsolutePath(), - "keytab", - keytabFile.getAbsolutePath())); - Configuration configuration = - new Configuration() { - @Override - public AppConfigurationEntry[] getAppConfigurationEntry(String name) { - return new AppConfigurationEntry[] {appConfigEntry}; - } - }; - - try { - new LoginContext("Server", null, null, configuration).login(); - } catch (LoginException e) { - Assume.assumeNoException(Locale.getDefault() + " appears to be incompatible with MiniKdc", e); - } finally { - kdc.stop(); - } - } - - private KerberosTestServices( - File workDir, JaasConfiguration jaasConfiguration, Configuration savedConfig, boolean debug) { - this.jaasConfiguration = jaasConfiguration; - this.savedConfig = savedConfig; - this.workDir = workDir; - this.debug = debug; - } - - public MiniKdc getKdc() { - return kdc; - } - - public void start() throws Exception { - assumeMiniKdcWorksWithDefaultLocale(); - - // There is time lag between selecting a port and trying to bind with it. It's possible that - // another service captures the port in between which'll result in BindException. - boolean bindException; - int numTries = 0; - do { - try { - bindException = false; - - kdc = getKdc(workDir, debug); - kdc.start(); - } catch (BindException e) { - PathUtils.deleteDirectory(workDir.toPath()); // clean directory - numTries++; - if (numTries == 3) { - log.error("Failed setting up MiniKDC. Tried {} times.", numTries); - throw e; - } - log.error("BindException encountered when setting up MiniKdc. Trying again."); - bindException = true; - } - } while (bindException); - - Configuration.setConfiguration(jaasConfiguration); - Krb5HttpClientBuilder.regenerateJaasConfiguration(); - } - - public void stop() { - if (kdc != null) kdc.stop(); - Configuration.setConfiguration(savedConfig); - Krb5HttpClientBuilder.regenerateJaasConfiguration(); - } - - public static Builder builder() { - return new Builder(); - } - - /** - * Returns a MiniKdc that can be used for creating kerberos principals and keytabs. Caller is - * responsible for starting/stopping the kdc. - */ - private static MiniKdc getKdc(File workDir, boolean debug) throws Exception { - Properties conf = MiniKdc.createConf(); - conf.setProperty("kdc.port", "0"); - conf.setProperty("debug", String.valueOf(debug)); - return new MiniKdc(conf, workDir); - } - - /** - * Programmatic version of a jaas.conf file suitable for connecting to a SASL-configured - * zookeeper. - */ - private static class JaasConfiguration extends Configuration { - - private static AppConfigurationEntry[] clientEntry; - private static AppConfigurationEntry[] serverEntry; - private String clientAppName = "Client", serverAppName = "Server"; - - /** - * Add an entry to the jaas configuration with the passed in name, principal, and keytab. The - * other necessary options will be set for you. - * - * @param clientPrincipal The principal of the client - * @param clientKeytab The location of the keytab with the clientPrincipal - * @param serverPrincipal The principal of the server - * @param serverKeytab The location of the keytab with the serverPrincipal - */ - public JaasConfiguration( - String clientPrincipal, File clientKeytab, String serverPrincipal, File serverKeytab) { - Map clientOptions = new HashMap<>(); - clientOptions.put("principal", clientPrincipal); - clientOptions.put("keyTab", clientKeytab.getAbsolutePath()); - clientOptions.put("useKeyTab", "true"); - clientOptions.put("storeKey", "true"); - clientOptions.put("useTicketCache", "false"); - clientOptions.put("refreshKrb5Config", "true"); - String jaasProp = EnvUtils.getProperty("solr.jaas.debug"); - if (jaasProp != null && "true".equalsIgnoreCase(jaasProp)) { - clientOptions.put("debug", "true"); - } - clientEntry = - new AppConfigurationEntry[] { - new AppConfigurationEntry( - krb5LoginModuleName, - AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, - clientOptions) - }; - if (serverPrincipal != null && serverKeytab != null) { - Map serverOptions = new HashMap<>(clientOptions); - serverOptions.put("principal", serverPrincipal); - serverOptions.put("keytab", serverKeytab.getAbsolutePath()); - serverEntry = - new AppConfigurationEntry[] { - new AppConfigurationEntry( - krb5LoginModuleName, - AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, - serverOptions) - }; - } - } - - /** - * Add an entry to the jaas configuration with the passed in principal and keytab, along with - * the app name. - * - * @param principal The principal - * @param keytab The keytab containing credentials for the principal - * @param appName The app name of the configuration - */ - public JaasConfiguration(String principal, File keytab, String appName) { - this(principal, keytab, null, null); - clientAppName = appName; - serverAppName = null; - } - - @Override - public AppConfigurationEntry[] getAppConfigurationEntry(String name) { - if (name.equals(clientAppName)) { - return clientEntry; - } else if (name.equals(serverAppName)) { - return serverEntry; - } - return null; - } - } - - public static final String krb5LoginModuleName = - Constants.IS_IBM_JAVA - ? "com.ibm.security.auth.module.Krb5LoginModule" - : "com.sun.security.auth.module.Krb5LoginModule"; - - public static class Builder { - private File kdcWorkDir; - private String clientPrincipal; - private File clientKeytab; - private String serverPrincipal; - private File serverKeytab; - private String appName; - private boolean debug = false; - - public Builder() {} - - public Builder withKdc(File kdcWorkDir) { - this.kdcWorkDir = kdcWorkDir; - return this; - } - - public Builder withDebug() { - this.debug = true; - return this; - } - - public Builder withJaasConfiguration( - String clientPrincipal, File clientKeytab, String serverPrincipal, File serverKeytab) { - this.clientPrincipal = Objects.requireNonNull(clientPrincipal); - this.clientKeytab = Objects.requireNonNull(clientKeytab); - this.serverPrincipal = serverPrincipal; - this.serverKeytab = serverKeytab; - this.appName = null; - return this; - } - - public Builder withJaasConfiguration(String principal, File keytab, String appName) { - this.clientPrincipal = Objects.requireNonNull(principal); - this.clientKeytab = Objects.requireNonNull(keytab); - this.serverPrincipal = null; - this.serverKeytab = null; - this.appName = appName; - return this; - } - - public KerberosTestServices build() throws Exception { - final Configuration oldConfig = - clientPrincipal != null ? Configuration.getConfiguration() : null; - JaasConfiguration jaasConfiguration = null; - if (clientPrincipal != null) { - jaasConfiguration = - (appName == null) - ? new JaasConfiguration( - clientPrincipal, clientKeytab, serverPrincipal, serverKeytab) - : new JaasConfiguration(clientPrincipal, clientKeytab, appName); - } - return new KerberosTestServices(kdcWorkDir, jaasConfiguration, oldConfig, debug); - } - } -} diff --git a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/KerberosUtils.java b/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/KerberosUtils.java deleted file mode 100644 index 6a9734d5e44..00000000000 --- a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/KerberosUtils.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security.hadoop; - -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; - -/** A utility class which provides common functionality required to test kerberos integration. */ -public class KerberosUtils { - /** - * This method sets up Hadoop mini-kdc along with relevant Kerberos configuration files (e.g. - * jaas.conf) as well as system properties. - * - * @param baseDir The directory path which should be used by the Hadoop mini-kdc - * @return An instance of {@link KerberosTestServices} - * @throws Exception in case of errors. - */ - static KerberosTestServices setupMiniKdc(Path baseDir) throws Exception { - System.setProperty("solr.jaas.debug", "true"); - Path kdcDir = baseDir.resolve("minikdc"); - String solrClientPrincipal = "solr"; - String solrAltClientPrincipal = - "solr_alt"; // An alternate principal that can be handled differently by authz tests - File keytabFile = kdcDir.resolve("keytabs").toFile(); - KerberosTestServices tmp = - KerberosTestServices.builder() - .withKdc(kdcDir.toFile()) - .withJaasConfiguration(solrClientPrincipal, keytabFile, "SolrClient") - .build(); - String solrServerPrincipal = "HTTP/127.0.0.1"; - tmp.start(); - tmp.getKdc() - .createPrincipal( - keytabFile, solrServerPrincipal, solrAltClientPrincipal, solrClientPrincipal); - - String appName = "SolrClient"; - String jaas = - appName - + " {\n" - + " com.sun.security.auth.module.Krb5LoginModule required\n" - + " useKeyTab=true\n" - + " keyTab=\"" - + keytabFile.getAbsolutePath() - + "\"\n" - + " storeKey=true\n" - + " useTicketCache=false\n" - + " doNotPrompt=true\n" - + " debug=true\n" - + " principal=\"" - + solrClientPrincipal - + "\";\n" - + "};"; - - Path jaasFile = kdcDir.resolve("jaas-client.conf"); - Files.writeString(jaasFile, jaas); - System.setProperty("java.security.auth.login.config", jaasFile.toString()); - System.setProperty("solr.kerberos.jaas.appname", appName); - System.setProperty("solr.kerberos.cookie.domain", "127.0.0.1"); - - System.setProperty("solr.kerberos.principal", solrServerPrincipal); - System.setProperty("solr.kerberos.keytab", keytabFile.getAbsolutePath()); - // Extracts 127.0.0.1 from HTTP/127.0.0.1@EXAMPLE.COM - System.setProperty( - "solr.kerberos.name.rules", - "RULE:[1:$1@$0](.*EXAMPLE.COM)s/@.*//" - + "\nRULE:[2:$2@$0](.*EXAMPLE.COM)s/@.*//" - + "\nDEFAULT"); - - // more debugging, if needed - // System.setProperty("sun.security.jgss.debug", "true"); - // System.setProperty("sun.security.krb5.debug", "true"); - // System.setProperty("sun.security.jgss.debug", "true"); - // System.setProperty("java.security.debug", "logincontext,policy,scl,gssloginconfig"); - return tmp; - } - - /** - * This method stops the Hadoop mini-kdc instance as well as cleanup relevant Java system - * properties. - * - * @param kerberosTestServices An instance of Hadoop mini-kdc - */ - public static void cleanupMiniKdc(KerberosTestServices kerberosTestServices) { - System.clearProperty("solr.jaas.debug"); - System.clearProperty("java.security.auth.login.config"); - System.clearProperty("solr.kerberos.jaas.appname"); - System.clearProperty("solr.kerberos.cookie.domain"); - System.clearProperty("solr.kerberos.principal"); - System.clearProperty("solr.kerberos.keytab"); - System.clearProperty("solr.kerberos.name.rules"); - - // more debugging, if needed - System.clearProperty("sun.security.jgss.debug"); - System.clearProperty("sun.security.krb5.debug"); - System.clearProperty("sun.security.jgss.debug"); - System.clearProperty("java.security.debug"); - - if (kerberosTestServices != null) { - kerberosTestServices.stop(); - } - } -} diff --git a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/SaslZkACLProviderTest.java b/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/SaslZkACLProviderTest.java deleted file mode 100644 index b3601b5e7fd..00000000000 --- a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/SaslZkACLProviderTest.java +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security.hadoop; - -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; -import java.io.File; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.util.concurrent.TimeUnit; -import org.apache.curator.framework.api.ACLProvider; -import org.apache.lucene.tests.util.QuickPatchThreadsFilter; -import org.apache.lucene.util.Constants; -import org.apache.solr.SolrIgnoredThreadsFilter; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.cloud.AbstractDigestZkACLAndCredentialsProvidersTestBase; -import org.apache.solr.cloud.AbstractZkTestCase; -import org.apache.solr.cloud.ZkTestServer; -import org.apache.solr.common.cloud.DefaultZkACLProvider; -import org.apache.solr.common.cloud.SaslZkACLProvider; -import org.apache.solr.common.cloud.SecurityAwareZkACLProvider; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZkACLProvider; -import org.apache.solr.util.BadZookeeperThreadsFilter; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.client.ZKClientConfig; -import org.apache.zookeeper.client.ZooKeeperSaslClient; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@ThreadLeakFilters( - defaultFilters = true, - filters = { - SolrIgnoredThreadsFilter.class, - QuickPatchThreadsFilter.class, - BadZookeeperThreadsFilter.class - }) -public class SaslZkACLProviderTest extends SolrTestCaseJ4 { - - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static final Charset DATA_ENCODING = StandardCharsets.UTF_8; - - protected ZkTestServer zkServer; - - @BeforeClass - public static void beforeClass() { - assumeFalse( - "FIXME: SOLR-7040: This test fails under IBM J9", Constants.JAVA_VENDOR.startsWith("IBM")); - System.setProperty("solrcloud.skip.autorecovery", "true"); - System.setProperty("hostName", "127.0.0.1"); - } - - @AfterClass - public static void afterClass() { - System.clearProperty("solrcloud.skip.autorecovery"); - System.clearProperty("hostName"); - System.clearProperty(ZKClientConfig.ZOOKEEPER_SERVER_PRINCIPAL); - } - - @Override - public void setUp() throws Exception { - super.setUp(); - if (log.isInfoEnabled()) { - log.info("####SETUP_START {}", getTestName()); - } - createTempDir(); - - Path zkDir = createTempDir().resolve("zookeeper/server1/data"); - log.info("ZooKeeper dataDir:{}", zkDir); - zkServer = new SaslZkTestServer(zkDir, createTempDir().resolve("miniKdc")); - zkServer.run(); - - System.setProperty("zkHost", zkServer.getZkAddress()); - - try (SolrZkClient zkClient = - new SolrZkClientWithACLs(zkServer.getZkHost(), AbstractZkTestCase.TIMEOUT)) { - ZooKeeperSaslClient saslClient = - zkClient.getCuratorFramework().getZookeeperClient().getZooKeeper().getSaslClient(); - assumeFalse("Could not set up ZK with SASL", saslClient.isFailed()); - zkClient.makePath("/solr", false, true); - } catch (KeeperException e) { - // This fails on Linux but passes on Windows and MacOS. Why? - assumeNoException("Could not set up ZK chroot, see SOLR-15366.", e); - } - setupZNodes(); - - if (log.isInfoEnabled()) { - log.info("####SETUP_END {}", getTestName()); - } - } - - protected void setupZNodes() throws Exception { - SolrZkClient zkClient = - new SolrZkClientWithACLs(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); - try { - zkClient.create( - "/protectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); - zkClient.makePath( - "/protectedMakePathNode", - "content".getBytes(DATA_ENCODING), - CreateMode.PERSISTENT, - false); - zkClient.create( - SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH, - "content".getBytes(DATA_ENCODING), - CreateMode.PERSISTENT, - false); - } finally { - zkClient.close(); - } - - zkClient = new SolrZkClientNoACLs(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); - try { - zkClient.create( - "/unprotectedCreateNode", - "content".getBytes(DATA_ENCODING), - CreateMode.PERSISTENT, - false); - zkClient.makePath( - "/unprotectedMakePathNode", - "content".getBytes(DATA_ENCODING), - CreateMode.PERSISTENT, - false); - } finally { - zkClient.close(); - } - } - - @Override - public void tearDown() throws Exception { - System.clearProperty("zkHost"); - zkServer.shutdown(); - super.tearDown(); - } - - @Test - public void testSaslZkACLProvider() throws Exception { - // Test with Sasl enabled - SolrZkClient zkClient = - new SolrZkClientWithACLs(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); - try { - AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest( - zkClient, true, true, true, true, true, true, true, true, true, true); - } finally { - zkClient.close(); - } - - // Test without Sasl enabled - setupZNodes(); - System.setProperty("zookeeper.sasl.client", "false"); - zkClient = new SolrZkClientNoACLs(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT); - try { - AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest( - zkClient, true, true, false, false, false, false, false, false, false, false); - } finally { - zkClient.close(); - System.clearProperty("zookeeper.sasl.client"); - } - } - - /** A SolrZKClient that adds Sasl ACLs */ - private static class SolrZkClientWithACLs extends SolrZkClient { - - public SolrZkClientWithACLs(String zkServerAddress, int zkClientTimeout) { - super( - new Builder() - .withUrl(zkServerAddress) - .withTimeout(zkClientTimeout, TimeUnit.MILLISECONDS)); - } - - @Override - public ZkACLProvider createACLProvider() { - return new SaslZkACLProvider(); - } - } - - /** A SolrZKClient that doesn't add ACLs */ - private static class SolrZkClientNoACLs extends SolrZkClient { - - public SolrZkClientNoACLs(String zkServerAddress, int zkClientTimeout) { - super( - new Builder() - .withUrl(zkServerAddress) - .withTimeout(zkClientTimeout, TimeUnit.MILLISECONDS)); - } - - @Override - public ACLProvider createACLProvider() { - return new DefaultZkACLProvider(); - } - } - - /** A ZkTestServer with Sasl support */ - public static class SaslZkTestServer extends ZkTestServer { - private final Path kdcDir; - private KerberosTestServices kerberosTestServices; - - public SaslZkTestServer(Path zkDir, Path kdcDir) throws Exception { - super(zkDir); - this.kdcDir = kdcDir; - } - - @Override - public void run() throws InterruptedException, IOException { - try { - // Don't require that credentials match the entire principal string, e.g. - // can match "solr" rather than "solr/host@DOMAIN" - System.setProperty("zookeeper.kerberos.removeRealmFromPrincipal", "true"); - System.setProperty("zookeeper.kerberos.removeHostFromPrincipal", "true"); - File keytabFile = kdcDir.resolve("keytabs").toFile(); - String zkClientPrincipal = "solr"; - String zkServerPrincipal = "zookeeper/127.0.0.1"; - System.setProperty(ZKClientConfig.ZOOKEEPER_SERVER_PRINCIPAL, zkServerPrincipal); - - kerberosTestServices = - KerberosTestServices.builder() - .withKdc(kdcDir.toFile()) - .withDebug() // SOLR-15366 - .withJaasConfiguration(zkClientPrincipal, keytabFile, zkServerPrincipal, keytabFile) - .build(); - kerberosTestServices.start(); - - kerberosTestServices - .getKdc() - .createPrincipal(keytabFile, zkClientPrincipal, zkServerPrincipal); - } catch (RuntimeException rex) { - throw rex; - } catch (Exception ex) { - throw new RuntimeException(ex); - } - super.run(false); - } - - @Override - public void shutdown() throws IOException, InterruptedException { - System.clearProperty("zookeeper.authProvider.1"); - System.clearProperty("zookeeper.kerberos.removeRealmFromPrincipal"); - System.clearProperty("zookeeper.kerberos.removeHostFromPrincipal"); - super.shutdown(); - if (kerberosTestServices != null) kerberosTestServices.stop(); - } - } -} diff --git a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/TestDelegationWithHadoopAuth.java b/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/TestDelegationWithHadoopAuth.java deleted file mode 100644 index 207a91ba6d6..00000000000 --- a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/TestDelegationWithHadoopAuth.java +++ /dev/null @@ -1,403 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security.hadoop; - -import java.util.Collections; -import java.util.HashSet; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import org.apache.hadoop.security.authentication.client.PseudoAuthenticator; -import org.apache.hadoop.util.Time; -import org.apache.http.HttpStatus; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.impl.CloudLegacySolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClient; -import org.apache.solr.client.solrj.impl.LBHttpSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.DelegationTokenRequest; -import org.apache.solr.client.solrj.response.CollectionAdminResponse; -import org.apache.solr.client.solrj.response.DelegationTokenResponse; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.IOUtils; -import org.apache.solr.embedded.JettySolrRunner; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -public class TestDelegationWithHadoopAuth extends SolrCloudTestCase { - protected static final int NUM_SERVERS = 2; - protected static final String USER_1 = "foo"; - protected static final String USER_2 = "bar"; - private static HttpSolrClient primarySolrClient, secondarySolrClient; - - @BeforeClass - public static void setupClass() throws Exception { - HadoopTestUtil.checkAssumptions(); - - configureCluster(NUM_SERVERS) // nodes - .withSecurityJson( - TEST_PATH().resolve("security").resolve("hadoop_simple_auth_with_delegation.json")) - .configure(); - - JettySolrRunner runnerPrimary = cluster.getJettySolrRunners().get(0); - primarySolrClient = new HttpSolrClient.Builder(runnerPrimary.getBaseUrl().toString()).build(); - JettySolrRunner runnerSecondary = cluster.getJettySolrRunners().get(1); - secondarySolrClient = - new HttpSolrClient.Builder(runnerSecondary.getBaseUrl().toString()).build(); - } - - @AfterClass - public static void tearDownClass() throws Exception { - if (primarySolrClient != null) { - IOUtils.closeQuietly(primarySolrClient); - primarySolrClient = null; - } - - if (secondarySolrClient != null) { - IOUtils.closeQuietly(secondarySolrClient); - secondarySolrClient = null; - } - } - - private String getDelegationToken(final String renewer, final String user, SolrClient solrClient) - throws Exception { - DelegationTokenRequest.Get get = - new DelegationTokenRequest.Get(renewer) { - @Override - public SolrParams getParams() { - ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); - params.set(PseudoAuthenticator.USER_NAME, user); - return params; - } - }; - DelegationTokenResponse.Get getResponse = get.process(solrClient); - return getResponse.getDelegationToken(); - } - - private long renewDelegationToken( - final String token, final int expectedStatusCode, final String user, SolrClient client) - throws Exception { - DelegationTokenRequest.Renew renew = - new DelegationTokenRequest.Renew(token) { - @Override - public SolrParams getParams() { - ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); - params.set(PseudoAuthenticator.USER_NAME, user); - return params; - } - - @Override - public Set getQueryParams() { - Set queryParams = super.getQueryParams(); - queryParams.add(PseudoAuthenticator.USER_NAME); - return queryParams; - } - }; - try { - DelegationTokenResponse.Renew renewResponse = renew.process(client); - assertEquals(HttpStatus.SC_OK, expectedStatusCode); - return renewResponse.getExpirationTime(); - } catch (SolrClient.RemoteSolrException ex) { - assertEquals(expectedStatusCode, ex.code()); - return -1; - } - } - - private void cancelDelegationToken(String token, int expectedStatusCode, SolrClient client) - throws Exception { - DelegationTokenRequest.Cancel cancel = new DelegationTokenRequest.Cancel(token); - try { - cancel.process(client); - assertEquals(HttpStatus.SC_OK, expectedStatusCode); - } catch (SolrClient.RemoteSolrException ex) { - assertEquals(expectedStatusCode, ex.code()); - } - } - - private void doSolrRequest(String token, int expectedStatusCode, HttpSolrClient client) - throws Exception { - doSolrRequest(token, expectedStatusCode, client, 1); - } - - private void doSolrRequest( - String token, int expectedStatusCode, HttpSolrClient client, int trials) throws Exception { - int lastStatusCode = 0; - for (int i = 0; i < trials; ++i) { - lastStatusCode = getStatusCode(token, null, null, client); - if (lastStatusCode == expectedStatusCode) { - return; - } - Thread.sleep(1000); - } - assertEquals("Did not receieve excepted status code", expectedStatusCode, lastStatusCode); - } - - private SolrRequest getAdminRequest(final SolrParams params) { - return new CollectionAdminRequest.List() { - @Override - public SolrParams getParams() { - ModifiableSolrParams p = new ModifiableSolrParams(super.getParams()); - p.add(params); - return p; - } - }; - } - - private int getStatusCode(String token, final String user, final String op, HttpSolrClient client) - throws Exception { - SolrClient delegationTokenClient; - if (random().nextBoolean()) - delegationTokenClient = - new HttpSolrClient.Builder(client.getBaseURL().toString()) - .withKerberosDelegationToken(token) - .withResponseParser(client.getParser()) - .build(); - else - delegationTokenClient = - new CloudLegacySolrClient.Builder( - Collections.singletonList(cluster.getZkServer().getZkAddress()), Optional.empty()) - .withLBHttpSolrClientBuilder( - new LBHttpSolrClient.Builder() - .withResponseParser(client.getParser()) - .withSocketTimeout(30000, TimeUnit.MILLISECONDS) - .withConnectionTimeout(15000, TimeUnit.MILLISECONDS) - .withHttpSolrClientBuilder( - new HttpSolrClient.Builder().withKerberosDelegationToken(token))) - .build(); - try { - ModifiableSolrParams p = new ModifiableSolrParams(); - if (user != null) p.set(PseudoAuthenticator.USER_NAME, user); - if (op != null) p.set("op", op); - SolrRequest req = getAdminRequest(p); - if (user != null || op != null) { - Set queryParams = new HashSet<>(); - if (user != null) queryParams.add(PseudoAuthenticator.USER_NAME); - if (op != null) queryParams.add("op"); - req.setQueryParams(queryParams); - } - try { - delegationTokenClient.request(req, null); - return HttpStatus.SC_OK; - } catch (SolrClient.RemoteSolrException re) { - return re.code(); - } - } finally { - delegationTokenClient.close(); - } - } - - private void doSolrRequest(SolrClient client, SolrRequest request, int expectedStatusCode) - throws Exception { - try { - client.request(request); - assertEquals(HttpStatus.SC_OK, expectedStatusCode); - } catch (SolrClient.RemoteSolrException ex) { - assertEquals(expectedStatusCode, ex.code()); - } - } - - private void verifyTokenValid(String token) throws Exception { - // pass with token - doSolrRequest(token, HttpStatus.SC_OK, primarySolrClient); - - // fail without token - doSolrRequest(null, ErrorCode.UNAUTHORIZED.code, primarySolrClient); - - // pass with token on other server - doSolrRequest(token, HttpStatus.SC_OK, secondarySolrClient); - - // fail without token on other server - doSolrRequest(null, ErrorCode.UNAUTHORIZED.code, secondarySolrClient); - } - - /** Test basic Delegation Token get/verify */ - @Test - public void testDelegationTokenVerify() throws Exception { - // Get token - String token = getDelegationToken(null, USER_1, primarySolrClient); - assertNotNull(token); - verifyTokenValid(token); - } - - private void verifyTokenCancelled(String token, HttpSolrClient client) throws Exception { - // fail with token on both servers. If cancelToOtherURL is true, - // the request went to other url, so FORBIDDEN should be returned immediately. - // The cancelled token may take awhile to propogate to the standard url (via ZK). - // This is of course the opposite if cancelToOtherURL is false. - doSolrRequest(token, ErrorCode.FORBIDDEN.code, client, 10); - - // fail without token on both servers - doSolrRequest(null, ErrorCode.UNAUTHORIZED.code, primarySolrClient); - doSolrRequest(null, ErrorCode.UNAUTHORIZED.code, secondarySolrClient); - } - - @Test - public void testDelegationTokenCancel() throws Exception { - { - // Get token - String token = getDelegationToken(null, USER_1, primarySolrClient); - assertNotNull(token); - - // cancel token, note don't need to be authenticated to cancel (no user specified) - cancelDelegationToken(token, HttpStatus.SC_OK, primarySolrClient); - verifyTokenCancelled(token, primarySolrClient); - } - - { - // cancel token on different server from where we got it - String token = getDelegationToken(null, USER_1, primarySolrClient); - assertNotNull(token); - - cancelDelegationToken(token, HttpStatus.SC_OK, secondarySolrClient); - verifyTokenCancelled(token, secondarySolrClient); - } - } - - @Test - public void testDelegationTokenCancelFail() throws Exception { - // cancel a bogus token - cancelDelegationToken("BOGUS", ErrorCode.NOT_FOUND.code, primarySolrClient); - - { - // cancel twice, first on same server - String token = getDelegationToken(null, USER_1, primarySolrClient); - assertNotNull(token); - cancelDelegationToken(token, HttpStatus.SC_OK, primarySolrClient); - cancelDelegationToken(token, ErrorCode.NOT_FOUND.code, secondarySolrClient); - cancelDelegationToken(token, ErrorCode.NOT_FOUND.code, primarySolrClient); - } - - { - // cancel twice, first on other server - String token = getDelegationToken(null, USER_1, primarySolrClient); - assertNotNull(token); - cancelDelegationToken(token, HttpStatus.SC_OK, secondarySolrClient); - cancelDelegationToken(token, ErrorCode.NOT_FOUND.code, secondarySolrClient); - cancelDelegationToken(token, ErrorCode.NOT_FOUND.code, primarySolrClient); - } - } - - private void verifyDelegationTokenRenew(String renewer, String user) throws Exception { - { - // renew on same server - String token = getDelegationToken(renewer, user, primarySolrClient); - assertNotNull(token); - long now = Time.now(); - assertTrue(renewDelegationToken(token, HttpStatus.SC_OK, user, primarySolrClient) > now); - verifyTokenValid(token); - } - - { - // renew on different server - String token = getDelegationToken(renewer, user, primarySolrClient); - assertNotNull(token); - long now = Time.now(); - assertTrue(renewDelegationToken(token, HttpStatus.SC_OK, user, secondarySolrClient) > now); - verifyTokenValid(token); - } - } - - @Test - // commented 4-Sep-2018 - // @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // - // 2-Aug-2018 - public void testDelegationTokenRenew() throws Exception { - // test with specifying renewer - verifyDelegationTokenRenew(USER_1, USER_1); - - // test without specifying renewer - verifyDelegationTokenRenew(null, USER_1); - } - - @Test - public void testDelegationTokenRenewFail() throws Exception { - // don't set renewer and try to renew as an a different user - String token = getDelegationToken(null, USER_1, primarySolrClient); - assertNotNull(token); - renewDelegationToken(token, ErrorCode.FORBIDDEN.code, USER_2, primarySolrClient); - renewDelegationToken(token, ErrorCode.FORBIDDEN.code, USER_2, secondarySolrClient); - - // set renewer and try to renew as different user - token = getDelegationToken("renewUser", USER_1, primarySolrClient); - assertNotNull(token); - renewDelegationToken(token, ErrorCode.FORBIDDEN.code, "notRenewUser", primarySolrClient); - renewDelegationToken(token, ErrorCode.FORBIDDEN.code, "notRenewUser", secondarySolrClient); - } - - /** Test that a non-delegation-token "op" http param is handled correctly */ - @Test - public void testDelegationOtherOp() throws Exception { - assertEquals( - HttpStatus.SC_OK, getStatusCode(null, USER_1, "someSolrOperation", primarySolrClient)); - } - - @Test - public void testZNodePaths() throws Exception { - getDelegationToken(null, USER_1, primarySolrClient); - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(cluster.getZkServer().getZkAddress()) - .withTimeout(1000, TimeUnit.MILLISECONDS) - .build()) { - assertTrue(zkClient.exists("/security/zkdtsm", true)); - assertTrue(zkClient.exists("/security/token", true)); - } - } - - /** Test HttpSolrServer's delegation token support */ - @Test - public void testDelegationTokenSolrClient() throws Exception { - // Get token - String token = getDelegationToken(null, USER_1, primarySolrClient); - assertNotNull(token); - - SolrRequest request = getAdminRequest(new ModifiableSolrParams()); - - // test without token - HttpSolrClient ss = - new HttpSolrClient.Builder(primarySolrClient.getBaseURL().toString()) - .withResponseParser(primarySolrClient.getParser()) - .build(); - try { - doSolrRequest(ss, request, ErrorCode.UNAUTHORIZED.code); - } finally { - ss.close(); - } - - try (HttpSolrClient client = - new HttpSolrClient.Builder(primarySolrClient.getBaseURL()) - .withKerberosDelegationToken(token) - .withResponseParser(primarySolrClient.getParser()) - .build()) { - // test with token via property - doSolrRequest(client, request, HttpStatus.SC_OK); - - // test with param -- should throw an exception - ModifiableSolrParams tokenParam = new ModifiableSolrParams(); - tokenParam.set("delegation", "invalidToken"); - expectThrows( - IllegalArgumentException.class, - () -> doSolrRequest(client, getAdminRequest(tokenParam), ErrorCode.FORBIDDEN.code)); - } - } -} diff --git a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/TestImpersonationWithHadoopAuth.java b/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/TestImpersonationWithHadoopAuth.java deleted file mode 100644 index c940391b01e..00000000000 --- a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/TestImpersonationWithHadoopAuth.java +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security.hadoop; - -import static org.apache.solr.security.hadoop.HttpParamDelegationTokenPlugin.USER_PARAM; -import static org.apache.solr.security.hadoop.ImpersonationUtil.getExpectedGroupExMsg; -import static org.apache.solr.security.hadoop.ImpersonationUtil.getExpectedHostExMsg; -import static org.apache.solr.security.hadoop.ImpersonationUtil.getProxyRequest; - -import java.net.InetAddress; -import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.HashMap; -import java.util.Map; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.EnvUtils; -import org.apache.solr.common.util.Utils; -import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.servlet.SolrRequestParsers; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -public class TestImpersonationWithHadoopAuth extends SolrCloudTestCase { - protected static final int NUM_SERVERS = 2; - private static final boolean defaultAddRequestHeadersToContext = - SolrRequestParsers.DEFAULT.isAddRequestHeadersToContext(); - - @SuppressWarnings("unchecked") - @BeforeClass - public static void setupClass() throws Exception { - HadoopTestUtil.checkAssumptions(); - - InetAddress loopback = InetAddress.getLoopbackAddress(); - Path securityJsonPath = - TEST_PATH().resolve("security").resolve("hadoop_simple_auth_with_delegation.json"); - String securityJson = Files.readString(securityJsonPath, Charset.defaultCharset()); - - Map securityConfig = (Map) Utils.fromJSONString(securityJson); - Map authConfig = (Map) securityConfig.get("authentication"); - Map proxyUserConfigs = - (Map) - authConfig.getOrDefault(HadoopAuthPlugin.PROXY_USER_CONFIGS, new HashMap<>()); - proxyUserConfigs.put("proxyuser.noGroups.hosts", "*"); - proxyUserConfigs.put("proxyuser.anyHostAnyUser.hosts", "*"); - proxyUserConfigs.put("proxyuser.anyHostAnyUser.groups", "*"); - proxyUserConfigs.put("proxyuser.wrongHost.hosts", DEAD_HOST_1); - proxyUserConfigs.put("proxyuser.wrongHost.groups", "*"); - proxyUserConfigs.put("proxyuser.noHosts.groups", "*"); - proxyUserConfigs.put( - "proxyuser.localHostAnyGroup.hosts", - loopback.getCanonicalHostName() - + "," - + loopback.getHostName() - + "," - + loopback.getHostAddress()); - proxyUserConfigs.put("proxyuser.localHostAnyGroup.groups", "*"); - proxyUserConfigs.put("proxyuser.bogusGroup.hosts", "*"); - proxyUserConfigs.put("proxyuser.bogusGroup.groups", "__some_bogus_group"); - proxyUserConfigs.put( - "proxyuser.anyHostUsersGroup.groups", ImpersonationUtil.getUsersFirstGroup()); - proxyUserConfigs.put("proxyuser.anyHostUsersGroup.hosts", "*"); - - authConfig.put(HadoopAuthPlugin.PROXY_USER_CONFIGS, proxyUserConfigs); - - SolrRequestParsers.DEFAULT.setAddRequestHeadersToContext(true); - System.setProperty("collectionsHandler", ImpersonatorCollectionsHandler.class.getName()); - - configureCluster(NUM_SERVERS) // nodes - .withSecurityJson(Utils.toJSONString(securityConfig)) - .addConfig( - "conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) - .configure(); - } - - @AfterClass - public static void tearDownClass() throws Exception { - SolrRequestParsers.DEFAULT.setAddRequestHeadersToContext(defaultAddRequestHeadersToContext); - System.clearProperty("collectionsHandler"); - } - - private SolrClient newSolrClient() { - return new HttpSolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()) - .build(); - } - - @Test - public void testProxyNoConfigGroups() throws Exception { - try (SolrClient solrClient = newSolrClient()) { - SolrClient.RemoteSolrException ex = - expectThrows( - SolrClient.RemoteSolrException.class, - () -> solrClient.request(getProxyRequest("noGroups", "bar"))); - assertTrue( - ex.getLocalizedMessage(), - ex.getMessage().contains(getExpectedGroupExMsg("noGroups", "bar"))); - } - } - - @Test - public void testProxyWrongHost() throws Exception { - try (SolrClient solrClient = newSolrClient()) { - SolrClient.RemoteSolrException ex = - expectThrows( - SolrClient.RemoteSolrException.class, - () -> solrClient.request(getProxyRequest("wrongHost", "bar"))); - assertTrue(ex.getMessage().contains(getExpectedHostExMsg("wrongHost"))); - } - } - - @Test - public void testProxyNoConfigHosts() throws Exception { - try (SolrClient solrClient = newSolrClient()) { - SolrClient.RemoteSolrException ex = - expectThrows( - SolrClient.RemoteSolrException.class, - () -> solrClient.request(getProxyRequest("noHosts", "bar"))); - assertTrue(ex.getMessage().contains(getExpectedHostExMsg("noHosts"))); - } - } - - @Test - public void testProxyValidateAnyHostAnyUser() throws Exception { - try (SolrClient solrClient = newSolrClient()) { - solrClient.request(getProxyRequest("anyHostAnyUser", "bar")); - assertTrue(ImpersonatorCollectionsHandler.called.get()); - } - } - - @Test - public void testProxyInvalidProxyUser() throws Exception { - try (SolrClient solrClient = newSolrClient()) { - // wrong direction, should fail - SolrClient.RemoteSolrException ex = - expectThrows( - SolrClient.RemoteSolrException.class, - () -> solrClient.request(getProxyRequest("bar", "anyHostAnyUser"))); - assertTrue(ex.getMessage().contains(getExpectedGroupExMsg("bar", "anyHostAnyUser"))); - } - } - - @Test - public void testProxyValidateHost() throws Exception { - try (SolrClient solrClient = newSolrClient()) { - solrClient.request(getProxyRequest("localHostAnyGroup", "bar")); - assertTrue(ImpersonatorCollectionsHandler.called.get()); - } - } - - @Test - public void testProxyValidateGroup() throws Exception { - try (SolrClient solrClient = newSolrClient()) { - solrClient.request(getProxyRequest("anyHostUsersGroup", EnvUtils.getProperty("user.name"))); - assertTrue(ImpersonatorCollectionsHandler.called.get()); - } - } - - @Test - public void testProxyInvalidGroup() throws Exception { - try (SolrClient solrClient = newSolrClient()) { - SolrClient.RemoteSolrException ex = - expectThrows( - SolrClient.RemoteSolrException.class, - () -> solrClient.request(getProxyRequest("bogusGroup", "bar"))); - assertTrue(ex.getMessage().contains(getExpectedGroupExMsg("bogusGroup", "bar"))); - } - } - - @Test - public void testProxyNullProxyUser() throws Exception { - try (SolrClient solrClient = newSolrClient()) { - expectThrows( - SolrClient.RemoteSolrException.class, - () -> solrClient.request(getProxyRequest("", "bar"))); - } - } - - @Test - @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/HADOOP-9893") - public void testForwarding() throws Exception { - String collectionName = "forwardingCollection"; - - // create collection - CollectionAdminRequest.Create create = - CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1); - try (SolrClient solrClient = newSolrClient()) { - create.process(solrClient); - } - - // try a command to each node, one of them must be forwarded - for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { - try (SolrClient client = - new HttpSolrClient.Builder(jetty.getBaseUrl().toString()) - .withDefaultCollection(collectionName) - .build()) { - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("q", "*:*"); - params.set(USER_PARAM, "user"); - client.query(params); - } - } - } -} diff --git a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/TestRuleBasedAuthorizationWithKerberos.java b/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/TestRuleBasedAuthorizationWithKerberos.java deleted file mode 100644 index a6bdd61155c..00000000000 --- a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/TestRuleBasedAuthorizationWithKerberos.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security.hadoop; - -import org.apache.lucene.util.Constants; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.impl.Http2SolrClient; -import org.apache.solr.client.solrj.impl.Krb5HttpClientUtils; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.cloud.AbstractDistribZkTestBase; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.util.LogLevel; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -@LogLevel("org.apache.solr.security=DEBUG") -public class TestRuleBasedAuthorizationWithKerberos extends SolrCloudTestCase { - protected static final int NUM_SERVERS = 2; - protected static final int NUM_SHARDS = 1; - protected static final int REPLICATION_FACTOR = 1; - private static KerberosTestServices kerberosTestServices; - - private String collectionName; - - @BeforeClass - public static void setupClass() throws Exception { - assumeFalse("Hadoop does not work on Windows", Constants.WINDOWS); - - kerberosTestServices = KerberosUtils.setupMiniKdc(createTempDir()); - - configureCluster(NUM_SERVERS) // nodes - .withSecurityJson( - TEST_PATH().resolve("security").resolve("hadoop_kerberos_authz_config.json")) - .addConfig( - "conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) - .configure(); - } - - @AfterClass - public static void tearDownClass() { - KerberosUtils.cleanupMiniKdc(kerberosTestServices); - kerberosTestServices = null; - } - - @Before - @Override - public void setUp() throws Exception { - super.setUp(); - collectionName = getSaferTestName(); - - // create collection - CollectionAdminRequest.Create create = - CollectionAdminRequest.createCollection( - collectionName, "conf1", NUM_SHARDS, REPLICATION_FACTOR); - create.process(cluster.getSolrClient()); - cluster.waitForActiveCollection(collectionName, 1, 1); - } - - @Test - public void testReadsAltUser() throws Exception { - String authorizedColl = "public"; - - // create collection - CollectionAdminRequest.createCollection(authorizedColl, "conf1", NUM_SHARDS, REPLICATION_FACTOR) - .process(cluster.getSolrClient()); - cluster.waitForActiveCollection(authorizedColl, 1, 1); - - final SolrQuery q = new SolrQuery("*:*"); - - for (JettySolrRunner jsr : cluster.getJettySolrRunners()) { - final String baseUrl = jsr.getBaseUrl().toString(); - try (Http2SolrClient client = new Http2SolrClient.Builder(baseUrl).build()) { - Krb5HttpClientUtils.setup(client, "solr_alt"); - assertEquals(0, client.query(authorizedColl, q).getStatus()); - SolrClient.RemoteSolrException e = - assertThrows( - SolrClient.RemoteSolrException.class, () -> client.query(collectionName, q)); - assertEquals(403, e.code()); - } - } - } - - @Test - public void testCollectionCreateSearchDelete() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - - SolrInputDocument doc = new SolrInputDocument(); - doc.setField("id", "1"); - solrClient.add(collectionName, doc); - solrClient.commit(collectionName); - - SolrQuery query = new SolrQuery(); - query.setQuery("*:*"); - QueryResponse rsp = solrClient.query(collectionName, query); - assertEquals(1, rsp.getResults().getNumFound()); - - CollectionAdminRequest.Delete deleteReq = - CollectionAdminRequest.deleteCollection(collectionName); - deleteReq.process(solrClient); - AbstractDistribZkTestBase.waitForCollectionToDisappear( - collectionName, ZkStateReader.from(solrClient), true, 330); - } -} diff --git a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithDelegationTokens.java b/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithDelegationTokens.java deleted file mode 100644 index 7f2528aa7ca..00000000000 --- a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithDelegationTokens.java +++ /dev/null @@ -1,487 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security.hadoop; - -import static org.apache.solr.security.hadoop.HttpParamDelegationTokenPlugin.USER_PARAM; - -import java.lang.invoke.MethodHandles; -import java.util.Collections; -import java.util.HashSet; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import org.apache.hadoop.util.Time; -import org.apache.http.HttpStatus; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.impl.CloudLegacySolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClient; -import org.apache.solr.client.solrj.impl.LBHttpSolrClient; -import org.apache.solr.client.solrj.request.AbstractUpdateRequest.ACTION; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.DelegationTokenRequest; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.client.solrj.response.CollectionAdminResponse; -import org.apache.solr.client.solrj.response.DelegationTokenResponse; -import org.apache.solr.client.solrj.response.UpdateResponse; -import org.apache.solr.cloud.MiniSolrCloudCluster; -import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.embedded.JettyConfig; -import org.apache.solr.embedded.JettySolrRunner; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Test the delegation token support in the {@link KerberosPlugin}. */ -public class TestSolrCloudWithDelegationTokens extends SolrTestCaseJ4 { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private static final int NUM_SERVERS = 2; - private static MiniSolrCloudCluster miniCluster; - private static HttpSolrClient solrClientPrimary; - private static HttpSolrClient solrClientSecondary; - - @BeforeClass - public static void startup() throws Exception { - System.setProperty("authenticationPlugin", HttpParamDelegationTokenPlugin.class.getName()); - System.setProperty(KerberosPlugin.DELEGATION_TOKEN_ENABLED, "true"); - System.setProperty("solr.kerberos.cookie.domain", "127.0.0.1"); - - miniCluster = - new MiniSolrCloudCluster(NUM_SERVERS, createTempDir(), JettyConfig.builder().build()); - JettySolrRunner runnerPrimary = miniCluster.getJettySolrRunners().get(0); - solrClientPrimary = new HttpSolrClient.Builder(runnerPrimary.getBaseUrl().toString()).build(); - JettySolrRunner runnerSecondary = miniCluster.getJettySolrRunners().get(1); - solrClientSecondary = - new HttpSolrClient.Builder(runnerSecondary.getBaseUrl().toString()).build(); - } - - @AfterClass - public static void shutdown() throws Exception { - if (null != solrClientPrimary) { - solrClientPrimary.close(); - solrClientPrimary = null; - } - if (null != solrClientSecondary) { - solrClientSecondary.close(); - solrClientSecondary = null; - } - - if (miniCluster != null) { - miniCluster.shutdown(); - miniCluster = null; - } - System.clearProperty("authenticationPlugin"); - System.clearProperty(KerberosPlugin.DELEGATION_TOKEN_ENABLED); - System.clearProperty("solr.kerberos.cookie.domain"); - } - - private String getDelegationToken( - final String renewer, final String user, HttpSolrClient solrClient) throws Exception { - DelegationTokenRequest.Get get = - new DelegationTokenRequest.Get(renewer) { - @Override - public SolrParams getParams() { - ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); - params.set(USER_PARAM, user); - return params; - } - }; - DelegationTokenResponse.Get getResponse = get.process(solrClient); - return getResponse.getDelegationToken(); - } - - private long renewDelegationToken( - final String token, final int expectedStatusCode, final String user, HttpSolrClient client) - throws Exception { - DelegationTokenRequest.Renew renew = - new DelegationTokenRequest.Renew(token) { - @Override - public SolrParams getParams() { - ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); - params.set(USER_PARAM, user); - return params; - } - - @Override - public Set getQueryParams() { - Set queryParams = super.getQueryParams(); - queryParams.add(USER_PARAM); - return queryParams; - } - }; - try { - DelegationTokenResponse.Renew renewResponse = renew.process(client); - assertEquals(HttpStatus.SC_OK, expectedStatusCode); - return renewResponse.getExpirationTime(); - } catch (SolrClient.RemoteSolrException ex) { - assertEquals(expectedStatusCode, ex.code()); - return -1; - } - } - - private void cancelDelegationToken(String token, int expectedStatusCode, HttpSolrClient client) - throws Exception { - DelegationTokenRequest.Cancel cancel = new DelegationTokenRequest.Cancel(token); - try { - cancel.process(client); - assertEquals(HttpStatus.SC_OK, expectedStatusCode); - } catch (SolrClient.RemoteSolrException ex) { - assertEquals(expectedStatusCode, ex.code()); - } - } - - private void doSolrRequest(String token, int expectedStatusCode, HttpSolrClient client) - throws Exception { - doSolrRequest(token, expectedStatusCode, client, 1); - } - - private void doSolrRequest( - String token, int expectedStatusCode, HttpSolrClient client, int trials) throws Exception { - int lastStatusCode = 0; - for (int i = 0; i < trials; ++i) { - lastStatusCode = getStatusCode(token, null, null, client); - if (lastStatusCode == expectedStatusCode) { - return; - } - Thread.sleep(1000); - } - assertEquals("Did not receive expected status code", expectedStatusCode, lastStatusCode); - } - - private SolrRequest getAdminRequest(final SolrParams params) { - return new CollectionAdminRequest.List() { - @Override - public SolrParams getParams() { - ModifiableSolrParams p = new ModifiableSolrParams(super.getParams()); - p.add(params); - return p; - } - }; - } - - private SolrRequest getUpdateRequest(boolean commit) { - UpdateRequest request = new UpdateRequest(); - if (commit) { - request.setAction(ACTION.COMMIT, false, false); - } - SolrInputDocument doc = new SolrInputDocument(); - doc.addField("id", "dummy_id"); - request.add(doc); - return request; - } - - private int getStatusCode(String token, final String user, final String op, HttpSolrClient client) - throws Exception { - SolrClient delegationTokenClient; - if (random().nextBoolean()) - delegationTokenClient = - new HttpSolrClient.Builder(client.getBaseURL().toString()) - .withKerberosDelegationToken(token) - .withResponseParser(client.getParser()) - .build(); - else - delegationTokenClient = - new CloudLegacySolrClient.Builder( - Collections.singletonList(miniCluster.getZkServer().getZkAddress()), - Optional.empty()) - .withLBHttpSolrClientBuilder( - new LBHttpSolrClient.Builder() - .withSocketTimeout(30000, TimeUnit.MILLISECONDS) - .withConnectionTimeout(15000, TimeUnit.MILLISECONDS) - .withResponseParser(client.getParser()) - .withHttpSolrClientBuilder( - new HttpSolrClient.Builder().withKerberosDelegationToken(token))) - .build(); - try { - ModifiableSolrParams p = new ModifiableSolrParams(); - if (user != null) p.set(USER_PARAM, user); - if (op != null) p.set("op", op); - SolrRequest req = getAdminRequest(p); - if (user != null || op != null) { - Set queryParams = new HashSet<>(); - if (user != null) queryParams.add(USER_PARAM); - if (op != null) queryParams.add("op"); - req.setQueryParams(queryParams); - } - try { - delegationTokenClient.request(req, null); - return HttpStatus.SC_OK; - } catch (SolrClient.RemoteSolrException re) { - return re.code(); - } - } finally { - delegationTokenClient.close(); - } - } - - private void doSolrRequest(HttpSolrClient client, SolrRequest request, int expectedStatusCode) - throws Exception { - try { - client.request(request); - assertEquals(HttpStatus.SC_OK, expectedStatusCode); - } catch (SolrClient.RemoteSolrException ex) { - assertEquals(expectedStatusCode, ex.code()); - } - } - - private void doSolrRequest( - HttpSolrClient client, SolrRequest request, String collectionName, int expectedStatusCode) - throws Exception { - try { - client.request(request, collectionName); - assertEquals(HttpStatus.SC_OK, expectedStatusCode); - } catch (SolrClient.RemoteSolrException ex) { - assertEquals(expectedStatusCode, ex.code()); - } - } - - private void verifyTokenValid(String token) throws Exception { - // pass with token - doSolrRequest(token, HttpStatus.SC_OK, solrClientPrimary); - - // fail without token - doSolrRequest(null, ErrorCode.UNAUTHORIZED.code, solrClientPrimary); - - // pass with token on other server - doSolrRequest(token, HttpStatus.SC_OK, solrClientSecondary); - - // fail without token on other server - doSolrRequest(null, ErrorCode.UNAUTHORIZED.code, solrClientSecondary); - } - - /** Test basic Delegation Token get/verify */ - @Test - public void testDelegationTokenVerify() throws Exception { - final String user = "bar"; - - // Get token - String token = getDelegationToken(null, user, solrClientPrimary); - assertNotNull(token); - verifyTokenValid(token); - } - - private void verifyTokenCancelled(String token, HttpSolrClient client) throws Exception { - // fail with token on both servers. If cancelToOtherURL is true, - // the request went to other url, so FORBIDDEN should be returned immediately. - // The cancelled token may take awhile to propogate to the standard url (via ZK). - // This is of course the opposite if cancelToOtherURL is false. - doSolrRequest(token, ErrorCode.FORBIDDEN.code, client, 10); - - // fail without token on both servers - doSolrRequest(null, ErrorCode.UNAUTHORIZED.code, solrClientPrimary); - doSolrRequest(null, ErrorCode.UNAUTHORIZED.code, solrClientSecondary); - } - - @Test - public void testDelegationTokenCancel() throws Exception { - { - // Get token - String token = getDelegationToken(null, "user", solrClientPrimary); - assertNotNull(token); - - // cancel token, note don't need to be authenticated to cancel (no user specified) - cancelDelegationToken(token, HttpStatus.SC_OK, solrClientPrimary); - verifyTokenCancelled(token, solrClientPrimary); - } - - { - // cancel token on different server from where we got it - String token = getDelegationToken(null, "user", solrClientPrimary); - assertNotNull(token); - - cancelDelegationToken(token, HttpStatus.SC_OK, solrClientSecondary); - verifyTokenCancelled(token, solrClientSecondary); - } - } - - @Test - public void testDelegationTokenCancelFail() throws Exception { - // cancel a bogus token - cancelDelegationToken("BOGUS", ErrorCode.NOT_FOUND.code, solrClientPrimary); - - { - // cancel twice, first on same server - String token = getDelegationToken(null, "bar", solrClientPrimary); - assertNotNull(token); - cancelDelegationToken(token, HttpStatus.SC_OK, solrClientPrimary); - cancelDelegationToken(token, ErrorCode.NOT_FOUND.code, solrClientSecondary); - cancelDelegationToken(token, ErrorCode.NOT_FOUND.code, solrClientPrimary); - } - - { - // cancel twice, first on other server - String token = getDelegationToken(null, "bar", solrClientPrimary); - assertNotNull(token); - cancelDelegationToken(token, HttpStatus.SC_OK, solrClientSecondary); - cancelDelegationToken(token, ErrorCode.NOT_FOUND.code, solrClientSecondary); - cancelDelegationToken(token, ErrorCode.NOT_FOUND.code, solrClientPrimary); - } - } - - private void verifyDelegationTokenRenew(String renewer, String user) throws Exception { - { - // renew on same server - String token = getDelegationToken(renewer, user, solrClientPrimary); - assertNotNull(token); - long now = Time.now(); - assertTrue(renewDelegationToken(token, HttpStatus.SC_OK, user, solrClientPrimary) > now); - verifyTokenValid(token); - } - - { - // renew on different server - String token = getDelegationToken(renewer, user, solrClientPrimary); - assertNotNull(token); - long now = Time.now(); - assertTrue(renewDelegationToken(token, HttpStatus.SC_OK, user, solrClientSecondary) > now); - verifyTokenValid(token); - } - } - - @Test - // commented 20-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // - // added 23-Aug-2018 - public void testDelegationTokenRenew() throws Exception { - // test with specifying renewer - verifyDelegationTokenRenew("bar", "bar"); - - // test without specifying renewer - verifyDelegationTokenRenew(null, "bar"); - } - - @Test - public void testDelegationTokenRenewFail() throws Exception { - // don't set renewer and try to renew as an a different user - String token = getDelegationToken(null, "bar", solrClientPrimary); - assertNotNull(token); - renewDelegationToken(token, ErrorCode.FORBIDDEN.code, "foo", solrClientPrimary); - renewDelegationToken(token, ErrorCode.FORBIDDEN.code, "foo", solrClientSecondary); - - // set renewer and try to renew as different user - token = getDelegationToken("renewUser", "bar", solrClientPrimary); - assertNotNull(token); - renewDelegationToken(token, ErrorCode.FORBIDDEN.code, "notRenewUser", solrClientPrimary); - renewDelegationToken(token, ErrorCode.FORBIDDEN.code, "notRenewUser", solrClientSecondary); - } - - /** Test that a non-delegation-token "op" http param is handled correctly */ - @Test - public void testDelegationOtherOp() throws Exception { - assertEquals( - HttpStatus.SC_OK, getStatusCode(null, "bar", "someSolrOperation", solrClientPrimary)); - } - - @Test - public void testZNodePaths() throws Exception { - getDelegationToken(null, "bar", solrClientPrimary); - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(miniCluster.getZkServer().getZkAddress()) - .withTimeout(1000, TimeUnit.MILLISECONDS) - .build()) { - assertTrue(zkClient.exists("/security/zkdtsm", true)); - assertTrue(zkClient.exists("/security/token", true)); - } - } - - /** Test HttpSolrServer's delegation token support */ - @Test - public void testDelegationTokenSolrClient() throws Exception { - // Get token - String token = getDelegationToken(null, "bar", solrClientPrimary); - assertNotNull(token); - - SolrRequest request = getAdminRequest(new ModifiableSolrParams()); - - // test without token - final HttpSolrClient ssWoToken = - new HttpSolrClient.Builder(solrClientPrimary.getBaseURL().toString()) - .withResponseParser(solrClientPrimary.getParser()) - .build(); - try { - doSolrRequest(ssWoToken, request, ErrorCode.UNAUTHORIZED.code); - } finally { - ssWoToken.close(); - } - - final HttpSolrClient ssWToken = - new HttpSolrClient.Builder(solrClientPrimary.getBaseURL().toString()) - .withKerberosDelegationToken(token) - .withResponseParser(solrClientPrimary.getParser()) - .build(); - try { - // test with token via property - doSolrRequest(ssWToken, request, HttpStatus.SC_OK); - - // test with param -- should throw an exception - ModifiableSolrParams tokenParam = new ModifiableSolrParams(); - tokenParam.set("delegation", "invalidToken"); - expectThrows( - IllegalArgumentException.class, - () -> doSolrRequest(ssWToken, getAdminRequest(tokenParam), ErrorCode.FORBIDDEN.code)); - } finally { - ssWToken.close(); - } - } - - /** Test HttpSolrServer's delegation token support for Update Requests */ - @Test - public void testDelegationTokenSolrClientWithUpdateRequests() throws Exception { - String collectionName = "testDelegationTokensWithUpdate"; - - // Get token - String token = getDelegationToken(null, "bar", solrClientPrimary); - assertNotNull(token); - - // Tests with update request. - // Before SOLR-13921, the request without commit will fail with a NullpointerException in - // DelegationTokenHttpSolrClient.createMethod - // due to a missing null check in the createMethod. (When requesting a commit, the setAction - // method will call setParams on the - // request so there is no NPE in the createMethod.) - final HttpSolrClient scUpdateWToken = - new HttpSolrClient.Builder(solrClientPrimary.getBaseURL().toString()) - .withKerberosDelegationToken(token) - .withResponseParser(solrClientPrimary.getParser()) - .build(); - - // Create collection - CollectionAdminRequest.Create create = - CollectionAdminRequest.createCollection(collectionName, 1, 1); - create.process(scUpdateWToken); - - try { - // test update request with token via property and commit=true - SolrRequest request = getUpdateRequest(true); - doSolrRequest(scUpdateWToken, request, collectionName, HttpStatus.SC_OK); - - // test update request with token via property and commit=false - request = getUpdateRequest(false); - doSolrRequest(scUpdateWToken, request, collectionName, HttpStatus.SC_OK); - - } finally { - scUpdateWToken.close(); - } - } -} diff --git a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithHadoopAuthPlugin.java b/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithHadoopAuthPlugin.java deleted file mode 100644 index 8c6d5a6711b..00000000000 --- a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithHadoopAuthPlugin.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security.hadoop; - -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.cloud.AbstractDistribZkTestBase; -import org.apache.solr.cloud.SolrCloudAuthTestCase; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.ZkStateReader; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -public class TestSolrCloudWithHadoopAuthPlugin extends SolrCloudAuthTestCase { - protected static final int NUM_SERVERS = 1; - protected static final int NUM_SHARDS = 1; - protected static final int REPLICATION_FACTOR = 1; - private static KerberosTestServices kerberosTestServices; - - @BeforeClass - public static void setupClass() throws Exception { - HadoopTestUtil.checkAssumptions(); - - kerberosTestServices = KerberosUtils.setupMiniKdc(createTempDir()); - - configureCluster(NUM_SERVERS) // nodes - .withSecurityJson(TEST_PATH().resolve("security").resolve("hadoop_kerberos_config.json")) - .addConfig( - "conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) - .withDefaultClusterProperty("useLegacyReplicaAssignment", "false") - .configure(); - } - - @AfterClass - public static void tearDownClass() throws Exception { - KerberosUtils.cleanupMiniKdc(kerberosTestServices); - kerberosTestServices = null; - } - - @Test - public void testBasics() throws Exception { - testCollectionCreateSearchDelete(); - // sometimes run a second test e.g. to test collection create-delete-create scenario - if (random().nextBoolean()) testCollectionCreateSearchDelete(); - } - - protected void testCollectionCreateSearchDelete() throws Exception { - CloudSolrClient solrClient = cluster.getSolrClient(); - String collectionName = "testkerberoscollection"; - - // create collection - CollectionAdminRequest.Create create = - CollectionAdminRequest.createCollection( - collectionName, "conf1", NUM_SHARDS, REPLICATION_FACTOR); - create.process(solrClient); - // The metrics counter for wrong credentials here really just means - assertAuthMetricsMinimums(2, 1, 0, 1, 0, 0); - - SolrInputDocument doc = new SolrInputDocument(); - doc.setField("id", "1"); - solrClient.add(collectionName, doc); - solrClient.commit(collectionName); - assertAuthMetricsMinimums(4, 2, 0, 2, 0, 0); - - SolrQuery query = new SolrQuery(); - query.setQuery("*:*"); - QueryResponse rsp = solrClient.query(collectionName, query); - assertEquals(1, rsp.getResults().getNumFound()); - - CollectionAdminRequest.Delete deleteReq = - CollectionAdminRequest.deleteCollection(collectionName); - deleteReq.process(solrClient); - AbstractDistribZkTestBase.waitForCollectionToDisappear( - collectionName, ZkStateReader.from(solrClient), true, 330); - // cookie was used to avoid re-authentication - assertAuthMetricsMinimums(6, 4, 0, 2, 0, 0); - } -} diff --git a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithKerberosAlt.java b/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithKerberosAlt.java deleted file mode 100644 index c9a5bb42df8..00000000000 --- a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithKerberosAlt.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security.hadoop; - -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering; -import java.lang.invoke.MethodHandles; -import org.apache.lucene.tests.util.QuickPatchThreadsFilter; -import org.apache.solr.SolrIgnoredThreadsFilter; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.cloud.AbstractDistribZkTestBase; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.util.BadZookeeperThreadsFilter; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@ThreadLeakFilters( - defaultFilters = true, - filters = { - SolrIgnoredThreadsFilter.class, - QuickPatchThreadsFilter.class, - BadZookeeperThreadsFilter.class // Zookeeper login leaks TGT renewal threads - }) -@ThreadLeakLingering(linger = 10000) // minikdc has some lingering threads -public class TestSolrCloudWithKerberosAlt extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static final int numShards = 1; - private static final int numReplicas = 1; - private static final int nodeCount = numShards * numReplicas; - private static final String configName = "solrCloudCollectionConfig"; - private static final String collectionName = "testkerberoscollection"; - - private KerberosTestServices kerberosTestServices; - - @Override - public void setUp() throws Exception { - super.setUp(); - - kerberosTestServices = KerberosUtils.setupMiniKdc(createTempDir()); - - System.setProperty("authenticationPlugin", "org.apache.solr.security.hadoop.KerberosPlugin"); - boolean enableDt = random().nextBoolean(); - log.info("Enable delegation token: {}", enableDt); - System.setProperty("solr.kerberos.delegation.token.enabled", Boolean.toString(enableDt)); - - configureCluster(nodeCount).addConfig(configName, configset("cloud-minimal")).configure(); - } - - @Test - public void testBasics() throws Exception { - testCollectionCreateSearchDelete(); - // sometimes run a second test e.g. to test collection create-delete-create scenario - if (random().nextBoolean()) testCollectionCreateSearchDelete(); - } - - private void testCollectionCreateSearchDelete() throws Exception { - CloudSolrClient client = cluster.getSolrClient(); - CollectionAdminRequest.createCollection(collectionName, configName, numShards, numReplicas) - .process(client); - - cluster.waitForActiveCollection(collectionName, numShards, numShards * numReplicas); - - // modify/query collection - new UpdateRequest().add("id", "1").commit(client, collectionName); - QueryResponse rsp = client.query(collectionName, new SolrQuery("*:*")); - assertEquals(1, rsp.getResults().getNumFound()); - - // delete the collection we created earlier - CollectionAdminRequest.deleteCollection(collectionName).process(client); - - AbstractDistribZkTestBase.waitForCollectionToDisappear( - collectionName, ZkStateReader.from(client), true, 330); - } - - @Override - public void tearDown() throws Exception { - System.clearProperty("authenticationPlugin"); - System.clearProperty("solr.kerberos.delegation.token.enabled"); - - KerberosUtils.cleanupMiniKdc(kerberosTestServices); - - super.tearDown(); - } -} diff --git a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithSecureImpersonation.java b/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithSecureImpersonation.java deleted file mode 100644 index 2155965e3b6..00000000000 --- a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithSecureImpersonation.java +++ /dev/null @@ -1,353 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security.hadoop; - -import static org.apache.solr.security.hadoop.HttpParamDelegationTokenPlugin.REMOTE_ADDRESS_PARAM; -import static org.apache.solr.security.hadoop.HttpParamDelegationTokenPlugin.REMOTE_HOST_PARAM; -import static org.apache.solr.security.hadoop.HttpParamDelegationTokenPlugin.USER_PARAM; - -import java.net.InetAddress; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; -import java.util.concurrent.atomic.AtomicBoolean; -import javax.servlet.http.HttpServletRequest; -import org.apache.hadoop.conf.Configuration; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.impl.HttpSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.response.CollectionAdminResponse; -import org.apache.solr.cloud.MiniSolrCloudCluster; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.util.EnvUtils; -import org.apache.solr.common.util.IOUtils; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.embedded.JettyConfig; -import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.handler.admin.CollectionsHandler; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.servlet.SolrRequestParsers; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -public class TestSolrCloudWithSecureImpersonation extends SolrTestCaseJ4 { - private static final int NUM_SERVERS = 2; - private static MiniSolrCloudCluster miniCluster; - private static SolrClient solrClient; - - private static String getUsersFirstGroup() throws Exception { - String group = "*"; // accept any group if a group can't be found - org.apache.hadoop.security.Groups hGroups = - new org.apache.hadoop.security.Groups(new Configuration()); - try { - List g = hGroups.getGroups(EnvUtils.getProperty("user.name")); - if (g != null && g.size() > 0) { - group = g.get(0); - } - } catch (NullPointerException npe) { - // if user/group doesn't exist on test box - } - return group; - } - - private static Map getImpersonatorSettings() throws Exception { - Map filterProps = new TreeMap<>(); - filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "noGroups.hosts", "*"); - filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "anyHostAnyUser.groups", "*"); - filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "anyHostAnyUser.hosts", "*"); - filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "wrongHost.hosts", DEAD_HOST_1); - filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "wrongHost.groups", "*"); - filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "noHosts.groups", "*"); - filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "localHostAnyGroup.groups", "*"); - InetAddress loopback = InetAddress.getLoopbackAddress(); - filterProps.put( - KerberosPlugin.IMPERSONATOR_PREFIX + "localHostAnyGroup.hosts", - loopback.getCanonicalHostName() - + "," - + loopback.getHostName() - + "," - + loopback.getHostAddress()); - filterProps.put( - KerberosPlugin.IMPERSONATOR_PREFIX + "anyHostUsersGroup.groups", getUsersFirstGroup()); - filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "anyHostUsersGroup.hosts", "*"); - filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "bogusGroup.groups", "__some_bogus_group"); - filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "bogusGroup.hosts", "*"); - return filterProps; - } - - @BeforeClass - public static void startup() throws Exception { - HadoopTestUtil.checkAssumptions(); - - System.setProperty("authenticationPlugin", HttpParamDelegationTokenPlugin.class.getName()); - System.setProperty(KerberosPlugin.DELEGATION_TOKEN_ENABLED, "true"); - - System.setProperty("solr.kerberos.cookie.domain", "127.0.0.1"); - Map impSettings = getImpersonatorSettings(); - for (Map.Entry entry : impSettings.entrySet()) { - System.setProperty(entry.getKey(), entry.getValue()); - } - System.setProperty("solr.test.sys.prop1", "propone"); - System.setProperty("solr.test.sys.prop2", "proptwo"); - - SolrRequestParsers.DEFAULT.setAddRequestHeadersToContext(true); - System.setProperty("collectionsHandler", ImpersonatorCollectionsHandler.class.getName()); - - miniCluster = - new MiniSolrCloudCluster(NUM_SERVERS, createTempDir(), JettyConfig.builder().build()); - JettySolrRunner runner = miniCluster.getJettySolrRunners().get(0); - solrClient = new HttpSolrClient.Builder(runner.getBaseUrl().toString()).build(); - } - - /** Verify that impersonator info is preserved in the request */ - public static class ImpersonatorCollectionsHandler extends CollectionsHandler { - public static AtomicBoolean called = new AtomicBoolean(false); - - public ImpersonatorCollectionsHandler() { - super(); - } - - public ImpersonatorCollectionsHandler(final CoreContainer coreContainer) { - super(coreContainer); - } - - @Override - public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { - called.set(true); - super.handleRequestBody(req, rsp); - String doAs = req.getParams().get(KerberosPlugin.IMPERSONATOR_DO_AS_HTTP_PARAM); - if (doAs != null) { - HttpServletRequest httpRequest = (HttpServletRequest) req.getContext().get("httpRequest"); - assertNotNull(httpRequest); - String user = (String) httpRequest.getAttribute(USER_PARAM); - assertNotNull(user); - assertEquals(user, httpRequest.getAttribute(KerberosPlugin.IMPERSONATOR_USER_NAME)); - } - } - } - - @Before - public void clearCalledIndicator() { - ImpersonatorCollectionsHandler.called.set(false); - } - - @AfterClass - public static void shutdown() throws Exception { - if (solrClient != null) { - IOUtils.closeQuietly(solrClient); - solrClient = null; - } - try { - if (miniCluster != null) { - miniCluster.shutdown(); - } - } finally { - miniCluster = null; - System.clearProperty("authenticationPlugin"); - System.clearProperty(KerberosPlugin.DELEGATION_TOKEN_ENABLED); - System.clearProperty("solr.kerberos.cookie.domain"); - Map impSettings = getImpersonatorSettings(); - for (Map.Entry entry : impSettings.entrySet()) { - System.clearProperty(entry.getKey()); - } - System.clearProperty("solr.test.sys.prop1"); - System.clearProperty("solr.test.sys.prop2"); - System.clearProperty("collectionsHandler"); - - SolrRequestParsers.DEFAULT.setAddRequestHeadersToContext(false); - } - } - - private void create1ShardCollection(String name, String config, MiniSolrCloudCluster solrCluster) - throws Exception { - CollectionAdminResponse response; - CollectionAdminRequest.Create create = - new CollectionAdminRequest.Create(name, config, 1, 1, 0, 0) { - @Override - public SolrParams getParams() { - ModifiableSolrParams msp = new ModifiableSolrParams(super.getParams()); - msp.set(USER_PARAM, "user"); - return msp; - } - }; - response = create.process(solrCluster.getSolrClient()); - - miniCluster.waitForActiveCollection(name, 1, 1); - - if (response.getStatus() != 0 || response.getErrorMessages() != null) { - fail("Could not create collection. Response" + response.toString()); - } - } - - private SolrRequest getProxyRequest(String user, String doAs) { - return getProxyRequest(user, doAs, null); - } - - private SolrRequest getProxyRequest(String user, String doAs, String remoteHost) { - return getProxyRequest(user, doAs, remoteHost, null); - } - - private SolrRequest getProxyRequest( - String user, String doAs, String remoteHost, String remoteAddress) { - return new CollectionAdminRequest.List() { - @Override - public SolrParams getParams() { - ModifiableSolrParams params = new ModifiableSolrParams(super.getParams()); - params.set(USER_PARAM, user); - params.set(KerberosPlugin.IMPERSONATOR_DO_AS_HTTP_PARAM, doAs); - if (remoteHost != null) params.set(REMOTE_HOST_PARAM, remoteHost); - if (remoteAddress != null) params.set(REMOTE_ADDRESS_PARAM, remoteAddress); - return params; - } - }; - } - - private String getExpectedGroupExMsg(String user, String doAs) { - return "User: " + user + " is not allowed to impersonate " + doAs; - } - - private String getExpectedHostExMsg(String user) { - return "Unauthorized connection for super-user: " + user; - } - - @Test - public void testProxyNoConfigGroups() { - SolrClient.RemoteSolrException e = - expectThrows( - SolrClient.RemoteSolrException.class, - () -> solrClient.request(getProxyRequest("noGroups", "bar"))); - assertTrue(e.getMessage().contains(getExpectedGroupExMsg("noGroups", "bar"))); - } - - @Test - public void testProxyWrongHost() { - SolrClient.RemoteSolrException e = - expectThrows( - SolrClient.RemoteSolrException.class, - () -> solrClient.request(getProxyRequest("wrongHost", "bar"))); - assertTrue(e.getMessage().contains(getExpectedHostExMsg("wrongHost"))); - } - - @Test - public void testProxyNoConfigHosts() { - SolrClient.RemoteSolrException e = - expectThrows( - SolrClient.RemoteSolrException.class, - () -> solrClient.request(getProxyRequest("noHosts", "bar"))); - assertTrue(e.getMessage().contains(getExpectedHostExMsg("noHosts"))); - } - - @Test - public void testProxyValidateAnyHostAnyUser() throws Exception { - solrClient.request(getProxyRequest("anyHostAnyUser", "bar", null)); - assertTrue(ImpersonatorCollectionsHandler.called.get()); - } - - @Test - public void testProxyInvalidProxyUser() { - // wrong direction, should fail - SolrClient.RemoteSolrException e = - expectThrows( - SolrClient.RemoteSolrException.class, - () -> solrClient.request(getProxyRequest("bar", "anyHostAnyUser"))); - assertTrue(e.getMessage().contains(getExpectedGroupExMsg("bar", "anyHostAnyUser"))); - } - - @Test - public void testProxyValidateHost() throws Exception { - solrClient.request(getProxyRequest("localHostAnyGroup", "bar")); - assertTrue(ImpersonatorCollectionsHandler.called.get()); - } - - @Test - public void testProxyValidateGroup() throws Exception { - solrClient.request( - getProxyRequest("anyHostUsersGroup", EnvUtils.getProperty("user.name"), null)); - assertTrue(ImpersonatorCollectionsHandler.called.get()); - } - - @Test - public void testProxyUnknownRemote() { - SolrClient.RemoteSolrException e = - expectThrows( - SolrClient.RemoteSolrException.class, - () -> { - // Use a reserved ip address - String nonProxyUserConfiguredIpAddress = "255.255.255.255"; - solrClient.request( - getProxyRequest( - "localHostAnyGroup", - "bar", - "unknownhost.bar.foo", - nonProxyUserConfiguredIpAddress)); - }); - assertTrue(e.getMessage().contains(getExpectedHostExMsg("localHostAnyGroup"))); - } - - @Test - public void testProxyInvalidRemote() { - SolrClient.RemoteSolrException e = - expectThrows( - SolrClient.RemoteSolrException.class, - () -> { - solrClient.request( - getProxyRequest("localHostAnyGroup", "bar", "[ff01::114]", "[::1]")); - }); - assertTrue(e.getMessage().contains(getExpectedHostExMsg("localHostAnyGroup"))); - } - - @Test - public void testProxyInvalidGroup() { - SolrClient.RemoteSolrException e = - expectThrows( - SolrClient.RemoteSolrException.class, - () -> solrClient.request(getProxyRequest("bogusGroup", "bar", null))); - assertTrue(e.getMessage().contains(getExpectedGroupExMsg("bogusGroup", "bar"))); - } - - @Test - public void testProxyNullProxyUser() { - expectThrows( - SolrClient.RemoteSolrException.class, () -> solrClient.request(getProxyRequest("", "bar"))); - } - - @Test - public void testForwarding() throws Exception { - String collectionName = "forwardingCollection"; - miniCluster.uploadConfigSet(TEST_PATH().resolve("collection1/conf"), "conf1"); - create1ShardCollection(collectionName, "conf1", miniCluster); - - // try a command to each node, one of them must be forwarded - for (JettySolrRunner jetty : miniCluster.getJettySolrRunners()) { - try (HttpSolrClient client = - new HttpSolrClient.Builder(jetty.getBaseUrl().toString()) - .withDefaultCollection(collectionName) - .build()) { - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("q", "*:*"); - params.set(USER_PARAM, "user"); - client.query(params); - } - } - } -} diff --git a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/TestZkAclsWithHadoopAuth.java b/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/TestZkAclsWithHadoopAuth.java deleted file mode 100644 index 074d39c51f6..00000000000 --- a/solr/modules/hadoop-auth/src/test/org/apache/solr/security/hadoop/TestZkAclsWithHadoopAuth.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.security.hadoop; - -import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME; -import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_READONLY_PASSWORD_VM_PARAM_NAME; -import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_READONLY_USERNAME_VM_PARAM_NAME; -import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME; - -import java.nio.charset.StandardCharsets; -import java.security.NoSuchAlgorithmException; -import java.util.List; -import java.util.Locale; -import java.util.concurrent.TimeUnit; -import org.apache.solr.cloud.MiniSolrCloudCluster; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.cloud.SecurityAwareZkACLProvider; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider; -import org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider; -import org.apache.zookeeper.ZooDefs; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.data.ACL; -import org.apache.zookeeper.data.Id; -import org.apache.zookeeper.data.Stat; -import org.apache.zookeeper.server.auth.DigestAuthenticationProvider; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -public class TestZkAclsWithHadoopAuth extends SolrCloudTestCase { - protected static final int NUM_SERVERS = 1; - protected static final int NUM_SHARDS = 1; - protected static final int REPLICATION_FACTOR = 1; - private static final String SOLR_PASSWD = "solr"; - private static final String FOO_PASSWD = "foo"; - private static final Id SOLR_ZK_ID = new Id("digest", digest("solr", SOLR_PASSWD)); - private static final Id FOO_ZK_ID = new Id("digest", digest("foo", FOO_PASSWD)); - - @BeforeClass - public static void setupClass() throws Exception { - HadoopTestUtil.checkAssumptions(); - - System.setProperty( - SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME, - VMParamsAllAndReadonlyDigestZkACLProvider.class.getName()); - System.setProperty( - SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME, - VMParamsSingleSetCredentialsDigestZkCredentialsProvider.class.getName()); - System.setProperty(DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, "solr"); - System.setProperty(DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, SOLR_PASSWD); - System.setProperty(DEFAULT_DIGEST_READONLY_USERNAME_VM_PARAM_NAME, "foo"); - System.setProperty(DEFAULT_DIGEST_READONLY_PASSWORD_VM_PARAM_NAME, FOO_PASSWD); - - configureCluster(NUM_SERVERS) // nodes - .withSolrXml(MiniSolrCloudCluster.DEFAULT_CLOUD_SOLR_XML) - .withSecurityJson( - TEST_PATH().resolve("security").resolve("hadoop_simple_auth_with_delegation.json")) - .addConfig( - "conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) - .configure(); - } - - @AfterClass - public static void tearDownClass() { - System.clearProperty(SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME); - System.clearProperty(SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME); - System.clearProperty(DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME); - System.clearProperty(DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME); - System.clearProperty(DEFAULT_DIGEST_READONLY_USERNAME_VM_PARAM_NAME); - System.clearProperty(DEFAULT_DIGEST_READONLY_PASSWORD_VM_PARAM_NAME); - } - - @Test - @SuppressWarnings({"try"}) - public void testZkAcls() throws Exception { - try (ZooKeeper keeper = - new ZooKeeper( - cluster.getZkServer().getZkAddress(), - (int) TimeUnit.MINUTES.toMillis(1), - arg0 -> { - /* Do nothing */ - })) { - keeper.addAuthInfo("digest", ("solr:" + SOLR_PASSWD).getBytes(StandardCharsets.UTF_8)); - - // Test well known paths. - checkSecurityACLs(keeper, "/security/token"); - checkSecurityACLs(keeper, "/security"); - - // Now test all ZK tree. - String zkHost = cluster.getSolrClient().getClusterStateProvider().getQuorumHosts(); - String zkChroot = zkHost.contains("/") ? zkHost.substring(zkHost.indexOf('/')) : null; - walkZkTree(keeper, zkChroot, "/"); - } - } - - private void walkZkTree(ZooKeeper keeper, String zkChroot, String path) throws Exception { - if (isSecurityZNode(zkChroot, path)) { - checkSecurityACLs(keeper, path); - } else { - checkNonSecurityACLs(keeper, path); - } - - List children = keeper.getChildren(path, false); - for (String child : children) { - String subpath = path.endsWith("/") ? path + child : path + "/" + child; - walkZkTree(keeper, zkChroot, subpath); - } - } - - private boolean isSecurityZNode(String zkChroot, String path) { - String temp = path; - if (zkChroot != null) { - temp = path.replace(zkChroot, ""); - } - return temp.startsWith(SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH); - } - - private void checkSecurityACLs(ZooKeeper keeper, String path) throws Exception { - List acls = keeper.getACL(path, new Stat()); - String message = String.format(Locale.ROOT, "Path %s ACLs found %s", path, acls); - assertEquals(message, 1, acls.size()); - assertTrue(message, acls.contains(new ACL(ZooDefs.Perms.ALL, SOLR_ZK_ID))); - } - - private void checkNonSecurityACLs(ZooKeeper keeper, String path) throws Exception { - List acls = keeper.getACL(path, new Stat()); - String message = String.format(Locale.ROOT, "Path %s ACLs found %s", path, acls); - assertEquals(message, 2, acls.size()); - assertTrue(message, acls.contains(new ACL(ZooDefs.Perms.ALL, SOLR_ZK_ID))); - assertTrue(message, acls.contains(new ACL(ZooDefs.Perms.READ, FOO_ZK_ID))); - } - - private static String digest(String userName, String passwd) { - try { - return DigestAuthenticationProvider.generateDigest(userName + ":" + passwd); - } catch (NoSuchAlgorithmException ex) { - throw new RuntimeException(ex); - } - } -} diff --git a/solr/modules/hadoop-auth/src/test/org/apache/solr/util/configuration/HadoopSSLConfigurationsTest.java b/solr/modules/hadoop-auth/src/test/org/apache/solr/util/configuration/HadoopSSLConfigurationsTest.java deleted file mode 100644 index b29c5444a9a..00000000000 --- a/solr/modules/hadoop-auth/src/test/org/apache/solr/util/configuration/HadoopSSLConfigurationsTest.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.util.configuration; - -import static org.apache.hadoop.security.alias.CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH; -import static org.hamcrest.CoreMatchers.is; -import static org.mockito.Mockito.when; - -import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import org.apache.hadoop.conf.Configuration; -import org.apache.lucene.tests.util.TestRuleRestoreSystemProperties; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.common.util.EnvUtils; -import org.apache.solr.util.configuration.providers.EnvSSLCredentialProvider; -import org.apache.solr.util.configuration.providers.SysPropSSLCredentialProvider; -import org.apache.solr.util.configuration.providers.hadoop.HadoopSSLCredentialProvider; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.mockito.Mockito; - -public class HadoopSSLConfigurationsTest extends SolrTestCaseJ4 { - private Map envs; - private SSLConfigurations sut; - - public static final String SAMPLE_PW1 = "pw123"; - public static final String SAMPLE_PW2 = "pw456"; - public static final String SAMPLE_PW3 = "pw789"; - public static final String KEY_STORE_PASSWORD = SSLConfigurations.SysProps.SSL_KEY_STORE_PASSWORD; - public static final String CLIENT_KEY_STORE_PASSWORD = - SSLConfigurations.SysProps.SSL_CLIENT_KEY_STORE_PASSWORD; - - @Rule - public TestRule syspropRestore = - new TestRuleRestoreSystemProperties( - SSLConfigurations.SysProps.SSL_KEY_STORE_PASSWORD, - SSLConfigurations.SysProps.SSL_CLIENT_KEY_STORE_PASSWORD, - CREDENTIAL_PROVIDER_PATH); - - @Override - @Before - public void setUp() throws Exception { - super.setUp(); - envs = new HashMap<>(); - } - - private SSLConfigurations createSut(Configuration mockHadoopConfiguration) { - EnvSSLCredentialProvider envSSLCredentialProvider = new EnvSSLCredentialProvider(); - envSSLCredentialProvider.setEnvVars(envs); - sut = - new SSLConfigurations( - Arrays.asList( - new HadoopSSLCredentialProvider(mockHadoopConfiguration), - envSSLCredentialProvider, - new SysPropSSLCredentialProvider())); - return sut; - } - - @Test - public void testSslConfigKeystorePwFromKeystoreHadoopCredentialProvider() throws IOException { - getSutWithMockedHadoopCredentialProvider( - SSLConfigurations.SysProps.SSL_KEY_STORE_PASSWORD, SAMPLE_PW1) - .init(); - assertThat(EnvUtils.getProperty(CLIENT_KEY_STORE_PASSWORD), is(SAMPLE_PW1)); - } - - @Test - public void testSslConfigKeystorePwFromClientKeystoreHadoopCredentialProvider() - throws IOException { - getSutWithMockedHadoopCredentialProvider( - SSLConfigurations.SysProps.SSL_CLIENT_KEY_STORE_PASSWORD, SAMPLE_PW2) - .init(); - assertThat(EnvUtils.getProperty(CLIENT_KEY_STORE_PASSWORD), is(SAMPLE_PW2)); - } - - @Test - public void testGetKeyStorePasswordFromHadoopCredentialProvider() throws IOException { - SSLConfigurations sut = - getSutWithMockedHadoopCredentialProvider( - SSLConfigurations.SysProps.SSL_KEY_STORE_PASSWORD, SAMPLE_PW3); - assertThat(sut.getKeyStorePassword(), is(SAMPLE_PW3)); - } - - @Test - public void testGetTruststorePasswordFromHadoopCredentialProvider() throws IOException { - SSLConfigurations sut = - getSutWithMockedHadoopCredentialProvider( - SSLConfigurations.SysProps.SSL_TRUST_STORE_PASSWORD, SAMPLE_PW3); - assertThat(sut.getTrustStorePassword(), is(SAMPLE_PW3)); - } - - @Test - public void testGetClientKeyStorePasswordFromHadoopCredentialProvider() throws IOException { - SSLConfigurations sut = - getSutWithMockedHadoopCredentialProvider( - SSLConfigurations.SysProps.SSL_CLIENT_KEY_STORE_PASSWORD, SAMPLE_PW3); - assertThat(sut.getClientKeyStorePassword(), is(SAMPLE_PW3)); - } - - @Test - public void testGetClientTruststorePasswordFromHadoopCredentialProvider() throws IOException { - SSLConfigurations sut = - getSutWithMockedHadoopCredentialProvider( - SSLConfigurations.SysProps.SSL_CLIENT_TRUST_STORE_PASSWORD, SAMPLE_PW3); - assertThat(sut.getClientTrustStorePassword(), is(SAMPLE_PW3)); - } - - @Test - public void testHadoopCredentialProviderPrioritySysPropAndEnvVars() throws IOException { - SSLConfigurations sut = - getSutWithMockedHadoopCredentialProvider(KEY_STORE_PASSWORD, SAMPLE_PW3); - assertThat(sut.getKeyStorePassword(), is(SAMPLE_PW3)); - } - - private SSLConfigurations getSutWithMockedHadoopCredentialProvider(String key, String pw) - throws IOException { - Configuration mockHadoopConfiguration = getMockHadoopConfiguration(); - when(mockHadoopConfiguration.getPassword(key)) - .then( - invocationOnMock -> - invocationOnMock.getArguments()[0].equals(key) ? pw.toCharArray() : null); - System.setProperty(CREDENTIAL_PROVIDER_PATH, "/some/path"); // enables HCP - return createSut(mockHadoopConfiguration); - } - - private Configuration getMockHadoopConfiguration() { - SolrTestCaseJ4.assumeWorkingMockito(); - return Mockito.mock(Configuration.class); - } -} diff --git a/solr/modules/hadoop-auth/src/test/org/apache/solr/util/configuration/providers/hadoop/HadoopSSLCredentialProviderTest.java b/solr/modules/hadoop-auth/src/test/org/apache/solr/util/configuration/providers/hadoop/HadoopSSLCredentialProviderTest.java deleted file mode 100644 index 8a40deeaa31..00000000000 --- a/solr/modules/hadoop-auth/src/test/org/apache/solr/util/configuration/providers/hadoop/HadoopSSLCredentialProviderTest.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.util.configuration.providers.hadoop; - -import static org.apache.hadoop.security.alias.CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH; -import static org.apache.solr.util.configuration.providers.AbstractSSLCredentialProvider.DEFAULT_CREDENTIAL_KEY_MAP; -import static org.hamcrest.core.Is.is; - -import java.io.IOException; -import java.util.Map; -import org.apache.hadoop.conf.Configuration; -import org.apache.lucene.tests.util.TestRuleRestoreSystemProperties; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.util.configuration.SSLCredentialProvider; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.mockito.Mockito; - -/** */ -public class HadoopSSLCredentialProviderTest extends SolrTestCaseJ4 { - - @Rule - public TestRule syspropRestore = new TestRuleRestoreSystemProperties(CREDENTIAL_PROVIDER_PATH); - - @Test(expected = RuntimeException.class) - public void testConstructorRequiresCredPath() { - new HadoopSSLCredentialProvider(getMockHadoopConfiguration()); - } - - @Test - public void testGetCredentials() throws Exception { - int cnt = 0; - for (Map.Entry set : - DEFAULT_CREDENTIAL_KEY_MAP.entrySet()) { - String pw = "pw" + ++cnt; - HadoopSSLCredentialProvider sut = - new HadoopSSLCredentialProvider(getMockedHadoopCredentialProvider(set.getValue(), pw)); - assertThat(sut.getCredential(set.getKey()), is(pw)); - } - } - - private Configuration getMockedHadoopCredentialProvider(String key, String pw) - throws IOException { - Configuration mockHadoopConfiguration = getMockHadoopConfiguration(); - Mockito.when(mockHadoopConfiguration.getPassword(key)) - .then( - invocationOnMock -> - invocationOnMock.getArguments()[0].equals(key) ? pw.toCharArray() : null); - System.setProperty(CREDENTIAL_PROVIDER_PATH, "/some/path"); // enables HCP - return mockHadoopConfiguration; - } - - private Configuration getMockHadoopConfiguration() { - SolrTestCaseJ4.assumeWorkingMockito(); - return Mockito.mock(Configuration.class); - } -} diff --git a/solr/modules/hdfs/build.gradle b/solr/modules/hdfs/build.gradle index 59949458561..b051c9fb1c6 100644 --- a/solr/modules/hdfs/build.gradle +++ b/solr/modules/hdfs/build.gradle @@ -23,52 +23,53 @@ dependencies { implementation project(':solr:core') implementation project(':solr:solrj') - implementation 'org.apache.lucene:lucene-core' + implementation libs.apache.lucene.core - implementation 'org.slf4j:slf4j-api' + implementation libs.slf4j.api - api 'org.apache.hadoop:hadoop-client-api' - runtimeOnly 'org.apache.hadoop:hadoop-client-runtime' + api libs.apache.hadoop.client.api + runtimeOnly libs.apache.hadoop.client.runtime // Guava implements Preconditions, caches, and VisibleForTesting annotations - implementation 'com.google.guava:guava' - implementation 'io.dropwizard.metrics:metrics-core' + implementation libs.google.guava + implementation libs.dropwizard.metrics.core + // Caffeine cache to implement HDFS block caching - implementation 'com.github.ben-manes.caffeine:caffeine' + implementation libs.benmanes.caffeine - implementation 'commons-cli:commons-cli' + implementation libs.commonscli.commonscli testImplementation project(':solr:test-framework') testImplementation project(':solr:solrj-zookeeper') - testImplementation 'org.apache.lucene:lucene-test-framework' - testImplementation 'com.carrotsearch.randomizedtesting:randomizedtesting-runner' - testImplementation 'junit:junit' + testImplementation libs.apache.lucene.testframework + testImplementation libs.carrotsearch.randomizedtesting.runner + testImplementation libs.junit.junit // hadoop dependencies for tests - testImplementation ('org.apache.hadoop:hadoop-hdfs') { transitive = false } - testImplementation ('org.apache.hadoop:hadoop-hdfs::tests') { transitive = false } - testImplementation 'org.apache.hadoop.thirdparty:hadoop-shaded-guava' - testRuntimeOnly 'org.apache.hadoop:hadoop-client-minicluster' + testImplementation(libs.apache.hadoop.hdfs) { transitive = false } + testImplementation(variantOf(libs.apache.hadoop.hdfs) { classifier 'tests' }) { transitive = false } + testImplementation libs.apache.hadoop.thirdparty.shadedguava + testRuntimeOnly libs.apache.hadoop.client.minicluster - testRuntimeOnly 'org.apache.logging.log4j:log4j-1.2-api' + testRuntimeOnly libs.apache.log4j1.api // classes like solr.ICUCollationField, used by NNFailoverTest for example. testRuntimeOnly project(':solr:modules:analysis-extras') // used by the hadoop-specific test framework classes - testImplementation 'commons-io:commons-io' - testImplementation 'org.apache.commons:commons-compress' - testImplementation 'org.apache.commons:commons-collections4' - testImplementation 'org.apache.commons:commons-lang3' - testImplementation 'io.dropwizard.metrics:metrics-core' + testImplementation libs.commonsio.commonsio + testImplementation libs.apache.commons.compress + testImplementation libs.apache.commons.collections4 + testImplementation libs.apache.commons.lang3 + testImplementation libs.dropwizard.metrics.core // Zookeeper dependency - some tests like HdfsCloudBackupRestore need this - permitTestUnusedDeclared 'org.apache.zookeeper:zookeeper' - testImplementation('org.apache.zookeeper:zookeeper', { + permitTestUnusedDeclared libs.apache.zookeeper.zookeeper + testImplementation(libs.apache.zookeeper.zookeeper, { exclude group: "org.apache.yetus", module: "audience-annotations" }) // required for instantiating a Zookeeper server in tests or embedded - testRuntimeOnly 'org.xerial.snappy:snappy-java' + testRuntimeOnly libs.xerial.snappy.java } diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/snapshots/SolrOnHdfsSnapshotsTool.java b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/snapshots/SolrOnHdfsSnapshotsTool.java index c0ce936da1d..92c0f73bf5c 100644 --- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/snapshots/SolrOnHdfsSnapshotsTool.java +++ b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/snapshots/SolrOnHdfsSnapshotsTool.java @@ -35,13 +35,14 @@ import java.util.Optional; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; -import org.apache.commons.cli.PosixParser; import org.apache.hadoop.fs.Path; import org.apache.solr.cli.CLIO; +import org.apache.solr.cli.CommonCLIOptions; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; @@ -61,28 +62,59 @@ * This class provides utility functions required for Solr on HDFS specific snapshots' * functionality. * + *

    If HDFS remains in Solr 10, then we should migrate this to extending ToolBase + * *

    For general purpose snapshot tooling see the related classes in the {@link * org.apache.solr.cli} package. */ public class SolrOnHdfsSnapshotsTool implements Closeable, CLIO { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private static final String PREPARE_FOR_EXPORT = "prepare-snapshot-export"; - private static final String HELP = "help"; - private static final String COLLECTION = "c"; - private static final String TEMP_DIR = "t"; - private static final String DEST_DIR = "d"; - private static final String SOLR_ZK_ENSEMBLE = "z"; - private static final String HDFS_PATH_PREFIX = "p"; + private static final Option PREPARE_FOR_EXPORT_OPTION = + Option.builder() + .longOpt("prepare-snapshot-export") + .hasArg() + .desc( + "The authentication mechanism to enable (basicAuth or kerberos). Defaults to 'basicAuth'.") + .build(); + private static final Option HDFS_PATH_PREFIX_OPTION = + Option.builder("p") + .hasArg() + .desc( + "This parameter specifies the HDFS URI prefix to be used " + + "during snapshot export preparation. This is applicable only if the Solr collection index files are stored on HDFS.") + .build(); + + private static final Option TEMP_DIR_OPTION = + Option.builder("t") + .hasArg() + .desc( + "This parameter specifies the path of a temporary directory on local filesystem" + + " during prepare-snapshot-export command.") + .build(); + private static final Option COLLECTION_OPTION = + Option.builder("c") + .hasArg() + .desc( + "This parameter specifies the name of the collection to be used during snapshot operation") + .build(); + private static final Option DEST_DIR_OPTION = + Option.builder("d") + .hasArg() + .desc( + "This parameter specifies the path on shared file-system (e.g. HDFS) where the snapshot related" + + " information should be stored.") + .build(); + private static final List OPTION_HELP_ORDER = Arrays.asList( - PREPARE_FOR_EXPORT, - HELP, - SOLR_ZK_ENSEMBLE, - COLLECTION, - DEST_DIR, - TEMP_DIR, - HDFS_PATH_PREFIX); + PREPARE_FOR_EXPORT_OPTION.getLongOpt(), + CommonCLIOptions.HELP_OPTION.getOpt(), + CommonCLIOptions.ZK_HOST_OPTION.getOpt(), + COLLECTION_OPTION.getOpt(), + DEST_DIR_OPTION.getOpt(), + TEMP_DIR_OPTION.getOpt(), + HDFS_PATH_PREFIX_OPTION.getOpt()); private final CloudSolrClient solrClient; @@ -248,43 +280,17 @@ public void prepareForExport( } public static void main(String[] args) throws IOException { - CommandLineParser parser = new PosixParser(); + CommandLineParser parser = new DefaultParser(); Options options = new Options(); - options.addOption( - null, - PREPARE_FOR_EXPORT, - true, - "This command will prepare copylistings for the specified snapshot." - + " This command should only be used only if Solr is deployed with Hadoop and collection index files are stored on a shared" - + " file-system e.g. HDFS"); - - options.addOption( - null, - HELP, - false, - "This command will print the help message for the snapshots related commands."); - options.addOption( - TEMP_DIR, - true, - "This parameter specifies the path of a temporary directory on local filesystem" - + " during prepare-snapshot-export command."); - options.addOption( - DEST_DIR, - true, - "This parameter specifies the path on shared file-system (e.g. HDFS) where the snapshot related" - + " information should be stored."); - options.addOption( - COLLECTION, - true, - "This parameter specifies the name of the collection to be used during snapshot operation"); - options.addOption( - SOLR_ZK_ENSEMBLE, true, "This parameter specifies the Solr Zookeeper ensemble address"); - options.addOption( - HDFS_PATH_PREFIX, - true, - "This parameter specifies the HDFS URI prefix to be used" - + " during snapshot export preparation. This is applicable only if the Solr collection index files are stored on HDFS."); + options.addOption(PREPARE_FOR_EXPORT_OPTION); + + options.addOption(CommonCLIOptions.HELP_OPTION); + options.addOption(TEMP_DIR_OPTION); + options.addOption(DEST_DIR_OPTION); + options.addOption(COLLECTION_OPTION); + options.addOption(CommonCLIOptions.ZK_HOST_OPTION); + options.addOption(HDFS_PATH_PREFIX_OPTION); CommandLine cmd = null; try { @@ -295,15 +301,15 @@ public static void main(String[] args) throws IOException { System.exit(1); } - if (cmd.hasOption(PREPARE_FOR_EXPORT)) { + if (cmd.hasOption(PREPARE_FOR_EXPORT_OPTION)) { try (SolrOnHdfsSnapshotsTool tool = - new SolrOnHdfsSnapshotsTool(requiredArg(options, cmd, SOLR_ZK_ENSEMBLE))) { - if (cmd.hasOption(PREPARE_FOR_EXPORT)) { - String snapshotName = cmd.getOptionValue(PREPARE_FOR_EXPORT); - String collectionName = requiredArg(options, cmd, COLLECTION); - String localFsDir = requiredArg(options, cmd, TEMP_DIR); - String hdfsOpDir = requiredArg(options, cmd, DEST_DIR); - String pathPrefix = cmd.getOptionValue(HDFS_PATH_PREFIX); + new SolrOnHdfsSnapshotsTool(requiredArg(options, cmd, CommonCLIOptions.ZK_HOST_OPTION))) { + if (cmd.hasOption(PREPARE_FOR_EXPORT_OPTION)) { + String snapshotName = cmd.getOptionValue(PREPARE_FOR_EXPORT_OPTION); + String collectionName = requiredArg(options, cmd, COLLECTION_OPTION); + String localFsDir = requiredArg(options, cmd, TEMP_DIR_OPTION); + String hdfsOpDir = requiredArg(options, cmd, DEST_DIR_OPTION); + String pathPrefix = cmd.getOptionValue(HDFS_PATH_PREFIX_OPTION); if (pathPrefix != null) { try { @@ -320,7 +326,7 @@ public static void main(String[] args) throws IOException { tool.prepareForExport(collectionName, snapshotName, localFsDir, pathPrefix, hdfsOpDir); } } - } else if (cmd.hasOption(HELP)) { + } else if (cmd.hasOption(CommonCLIOptions.HELP_OPTION)) { printHelp(options); } else { CLIO.out("Unknown command specified."); @@ -328,7 +334,7 @@ public static void main(String[] args) throws IOException { } } - private static String requiredArg(Options options, CommandLine cmd, String optVal) { + private static String requiredArg(Options options, CommandLine cmd, Option optVal) { if (!cmd.hasOption(optVal)) { CLIO.out("Please specify the value for option " + optVal); printHelp(options); diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/HdfsLockFactory.java b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/HdfsLockFactory.java index e461c6de8f7..06e43b7dd2d 100644 --- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/HdfsLockFactory.java +++ b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/HdfsLockFactory.java @@ -42,11 +42,10 @@ private HdfsLockFactory() {} @Override public Lock obtainLock(Directory dir, String lockName) throws IOException { - if (!(dir instanceof HdfsDirectory)) { + if (!(dir instanceof HdfsDirectory hdfsDir)) { throw new UnsupportedOperationException( "HdfsLockFactory can only be used with HdfsDirectory subclasses, got: " + dir); } - final HdfsDirectory hdfsDir = (HdfsDirectory) dir; final Configuration conf = hdfsDir.getConfiguration(); final Path lockPath = hdfsDir.getHdfsDirPath(); final Path lockFile = new Path(lockPath, lockName); diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/BlockCacheKey.java b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/BlockCacheKey.java index ed02ee53deb..be6e2daa4ee 100644 --- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/BlockCacheKey.java +++ b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/store/blockcache/BlockCacheKey.java @@ -64,8 +64,7 @@ public int hashCode() { @Override public boolean equals(Object obj) { if (this == obj) return true; - if (!(obj instanceof BlockCacheKey)) return false; - BlockCacheKey other = (BlockCacheKey) obj; + if (!(obj instanceof BlockCacheKey other)) return false; return block == other.block && file == other.file && Objects.equals(path, other.path); } diff --git a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/SharedFileSystemAutoReplicaFailoverTest.java b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/SharedFileSystemAutoReplicaFailoverTest.java index 34e3b2a1492..68fd8715181 100644 --- a/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/SharedFileSystemAutoReplicaFailoverTest.java +++ b/solr/modules/hdfs/src/test/org/apache/solr/hdfs/cloud/SharedFileSystemAutoReplicaFailoverTest.java @@ -47,11 +47,8 @@ import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.ChaosMonkey; -import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.ClusterStateUtil; -import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.ZkStateReader; @@ -382,44 +379,17 @@ private boolean waitingForReplicasNotLive( .filter(jetty -> jetty.getCoreContainer() != null) .map(JettySolrRunner::getNodeName) .collect(Collectors.toSet()); - long timeout = - System.nanoTime() + TimeUnit.NANOSECONDS.convert(timeoutInMs, TimeUnit.MILLISECONDS); - boolean success = false; - while (!success && System.nanoTime() < timeout) { - success = true; - ClusterState clusterState = zkStateReader.getClusterState(); - if (clusterState != null) { - Map collections = clusterState.getCollectionsMap(); - for (Map.Entry entry : collections.entrySet()) { - DocCollection docCollection = entry.getValue(); - Collection slices = docCollection.getSlices(); - for (Slice slice : slices) { - // only look at active shards - if (slice.getState() == Slice.State.ACTIVE) { - Collection replicas = slice.getReplicas(); - for (Replica replica : replicas) { - if (nodeNames.contains(replica.getNodeName())) { - boolean live = clusterState.liveNodesContain(replica.getNodeName()); - if (live) { - success = false; - } - } - } - } - } - } - if (!success) { - try { - Thread.sleep(500); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Interrupted"); - } - } - } - } - - return success; + return ClusterStateUtil.waitFor( + zkStateReader, + null, + timeoutInMs, + TimeUnit.MILLISECONDS, + (liveNodes, state) -> + ClusterStateUtil.replicasOfActiveSlicesStream(state) + .noneMatch( + replica -> + nodeNames.contains(replica.getNodeName()) + && liveNodes.contains(replica.getNodeName()))); } private void assertSliceAndReplicaCount( diff --git a/solr/modules/jwt-auth/build.gradle b/solr/modules/jwt-auth/build.gradle index c2a4990b5b3..6d3f3ac42b0 100644 --- a/solr/modules/jwt-auth/build.gradle +++ b/solr/modules/jwt-auth/build.gradle @@ -28,43 +28,43 @@ configurations { } dependencies { - constraintsOnly(platform("com.fasterxml.jackson:jackson-bom")) + constraintsOnly(platform(libs.fasterxml.jackson.bom)) implementation project(':solr:core') implementation project(':solr:solrj') - implementation 'org.bitbucket.b_c:jose4j' + implementation libs.bc.jose4j - implementation 'io.dropwizard.metrics:metrics-core' - implementation 'org.apache.httpcomponents:httpclient' - implementation 'org.apache.httpcomponents:httpcore' - implementation 'org.eclipse.jetty:jetty-client' - implementation 'org.eclipse.jetty:jetty-http' - implementation 'org.eclipse.jetty.toolchain:jetty-servlet-api' - implementation 'com.google.guava:guava' - implementation 'org.slf4j:slf4j-api' + implementation libs.dropwizard.metrics.core + implementation libs.apache.httpcomponents.httpclient + implementation libs.apache.httpcomponents.httpcore + implementation libs.eclipse.jetty.client + implementation libs.eclipse.jetty.http + implementation libs.eclipse.jetty.toolchain.servletapi + implementation libs.google.guava + implementation libs.slf4j.api testImplementation project(':solr:test-framework') - testImplementation 'org.apache.lucene:lucene-test-framework' - testImplementation 'junit:junit' + testImplementation libs.apache.lucene.testframework + testImplementation libs.junit.junit - testImplementation('org.mockito:mockito-core', { + testImplementation(libs.mockito.core, { exclude group: "net.bytebuddy", module: "byte-buddy-agent" }) - testRuntimeOnly('org.mockito:mockito-subclass', { + testRuntimeOnly(libs.mockito.subclass, { exclude group: "net.bytebuddy", module: "byte-buddy-agent" }) - testImplementation('no.nav.security:mock-oauth2-server', { + testImplementation(libs.navsecurity.mockoauth2server, { exclude group: "io.netty", module: "netty-all" }) // required by mock-oauth2-server - testImplementation 'com.fasterxml.jackson.core:jackson-databind' - permitTestUnusedDeclared 'com.fasterxml.jackson.core:jackson-databind' + testImplementation libs.fasterxml.jackson.core.databind + permitTestUnusedDeclared libs.fasterxml.jackson.core.databind - testImplementation 'org.bouncycastle:bcpkix-jdk18on' - testImplementation 'org.bouncycastle:bcprov-jdk18on' - testImplementation 'com.nimbusds:nimbus-jose-jwt' - testImplementation 'com.squareup.okhttp3:mockwebserver' - testImplementation 'com.squareup.okhttp3:okhttp' - testRuntimeOnly 'io.netty:netty-codec-http' + testImplementation libs.bouncycastle.bcpkix + testImplementation libs.bouncycastle.bcprov + testImplementation libs.nimbusds.josejwt + testImplementation libs.squareup.okhttp3.mockwebserver + testImplementation libs.squareup.okhttp3.okhttp + testRuntimeOnly libs.netty.codechttp } diff --git a/solr/modules/jwt-auth/src/java/org/apache/solr/security/jwt/JWTAuthPlugin.java b/solr/modules/jwt-auth/src/java/org/apache/solr/security/jwt/JWTAuthPlugin.java index bb64f458f4e..5b9cdd42820 100644 --- a/solr/modules/jwt-auth/src/java/org/apache/solr/security/jwt/JWTAuthPlugin.java +++ b/solr/modules/jwt-auth/src/java/org/apache/solr/security/jwt/JWTAuthPlugin.java @@ -956,10 +956,8 @@ AuthCode getAuthCode() { @Override protected boolean interceptInternodeRequest(HttpRequest httpRequest, HttpContext httpContext) { - if (httpContext instanceof HttpClientContext) { - HttpClientContext httpClientContext = (HttpClientContext) httpContext; - if (httpClientContext.getUserToken() instanceof JWTPrincipal) { - JWTPrincipal jwtPrincipal = (JWTPrincipal) httpClientContext.getUserToken(); + if (httpContext instanceof HttpClientContext httpClientContext) { + if (httpClientContext.getUserToken() instanceof JWTPrincipal jwtPrincipal) { httpRequest.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + jwtPrincipal.getToken()); return true; } @@ -970,8 +968,7 @@ protected boolean interceptInternodeRequest(HttpRequest httpRequest, HttpContext @Override protected boolean interceptInternodeRequest(Request request) { Object userToken = request.getAttributes().get(Http2SolrClient.REQ_PRINCIPAL_KEY); - if (userToken instanceof JWTPrincipal) { - JWTPrincipal jwtPrincipal = (JWTPrincipal) userToken; + if (userToken instanceof JWTPrincipal jwtPrincipal) { request.headers(h -> h.put(HttpHeaders.AUTHORIZATION, "Bearer " + jwtPrincipal.getToken())); return true; } diff --git a/solr/modules/jwt-auth/src/java/org/apache/solr/security/jwt/JWTPrincipal.java b/solr/modules/jwt-auth/src/java/org/apache/solr/security/jwt/JWTPrincipal.java index 93d2764e83f..e7c26d98f58 100644 --- a/solr/modules/jwt-auth/src/java/org/apache/solr/security/jwt/JWTPrincipal.java +++ b/solr/modules/jwt-auth/src/java/org/apache/solr/security/jwt/JWTPrincipal.java @@ -62,8 +62,7 @@ public Map getClaims() { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof JWTPrincipal)) return false; - JWTPrincipal that = (JWTPrincipal) o; + if (!(o instanceof JWTPrincipal that)) return false; return Objects.equals(username, that.username) && Objects.equals(token, that.token) && Objects.equals(claims, that.claims); diff --git a/solr/modules/jwt-auth/src/java/org/apache/solr/security/jwt/JWTPrincipalWithUserRoles.java b/solr/modules/jwt-auth/src/java/org/apache/solr/security/jwt/JWTPrincipalWithUserRoles.java index 1f298a18ab3..03d870c4e82 100644 --- a/solr/modules/jwt-auth/src/java/org/apache/solr/security/jwt/JWTPrincipalWithUserRoles.java +++ b/solr/modules/jwt-auth/src/java/org/apache/solr/security/jwt/JWTPrincipalWithUserRoles.java @@ -48,8 +48,7 @@ public Set getVerifiedRoles() { @Override public boolean equals(Object o) { - if (!(o instanceof JWTPrincipalWithUserRoles)) return false; - JWTPrincipalWithUserRoles that = (JWTPrincipalWithUserRoles) o; + if (!(o instanceof JWTPrincipalWithUserRoles that)) return false; return super.equals(o) && roles.equals(that.roles); } diff --git a/solr/modules/jwt-auth/src/test/org/apache/solr/security/jwt/JWTAuthPluginTest.java b/solr/modules/jwt-auth/src/test/org/apache/solr/security/jwt/JWTAuthPluginTest.java index a5642b9f6cd..0d6cc759cb2 100644 --- a/solr/modules/jwt-auth/src/test/org/apache/solr/security/jwt/JWTAuthPluginTest.java +++ b/solr/modules/jwt-auth/src/test/org/apache/solr/security/jwt/JWTAuthPluginTest.java @@ -73,7 +73,7 @@ public class JWTAuthPluginTest extends SolrTestCaseJ4 { static HashMap testJwk; public static Path JWT_TEST_PATH() { - return getFile("solr/security").getParentFile().toPath(); + return getFile("solr/security").getParent(); } static { diff --git a/solr/modules/langid/build.gradle b/solr/modules/langid/build.gradle index c25227a7635..dc63a9f9bbc 100644 --- a/solr/modules/langid/build.gradle +++ b/solr/modules/langid/build.gradle @@ -23,14 +23,13 @@ dependencies { implementation project(':solr:core') implementation project(':solr:solrj') - implementation ('org.apache.tika:tika-core') { transitive = false } - implementation 'commons-io:commons-io' - implementation 'com.cybozu.labs:langdetect' - // NOTE: Currently not defined in versions.props since we need to stay on same version as Lucene due to opennlp - implementation 'org.apache.opennlp:opennlp-tools' - implementation 'org.slf4j:slf4j-api' + implementation(libs.apache.tika.core) { transitive = false } + implementation libs.commonsio.commonsio + implementation libs.cybozulabs.langdetect + implementation libs.apache.opennlp.tools + implementation libs.slf4j.api testImplementation project(':solr:test-framework') - testImplementation 'com.carrotsearch.randomizedtesting:randomizedtesting-runner' - testImplementation 'junit:junit' + testImplementation libs.carrotsearch.randomizedtesting.runner + testImplementation libs.junit.junit } diff --git a/solr/modules/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java b/solr/modules/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java index 21921440cae..f4f1b9cc83c 100644 --- a/solr/modules/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java +++ b/solr/modules/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java @@ -111,8 +111,8 @@ private void initParams(SolrParams params) { overwrite = params.getBool(OVERWRITE, false); langAllowlist = new HashSet<>(); threshold = params.getDouble(THRESHOLD, DOCID_THRESHOLD_DEFAULT); - String legacyAllowList = params.get(LANG_WHITELIST, ""); - if (legacyAllowList.length() > 0) { + final String legacyAllowList = params.get(LANG_WHITELIST, "").trim(); + if (!legacyAllowList.isEmpty()) { // nowarn compile time string concatenation log.warn( LANG_WHITELIST @@ -120,11 +120,10 @@ private void initParams(SolrParams params) { + LANG_ALLOWLIST + " instead."); // nowarn } - if (params.get(LANG_ALLOWLIST, legacyAllowList).length() > 0) { - for (String lang : params.get(LANG_ALLOWLIST, "").split(",")) { - langAllowlist.add(lang); - } - } + Arrays.stream(params.get(LANG_ALLOWLIST, legacyAllowList).split(",")) + .map(String::trim) + .filter(lang -> !lang.isEmpty()) + .forEach(langAllowlist::add); // Mapping params (field centric) enableMapping = params.getBool(MAP_ENABLE, false); diff --git a/solr/modules/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java b/solr/modules/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java index 4d8d398a25c..4690fc1569c 100644 --- a/solr/modules/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java +++ b/solr/modules/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java @@ -18,6 +18,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Set; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.ModifiableSolrParams; @@ -39,7 +40,7 @@ public static void beforeClass() throws Exception { initCore( "solrconfig-languageidentifier.xml", "schema.xml", - getFile("langid/solr").getAbsolutePath()); + getFile("langid/solr").toAbsolutePath().toString()); SolrCore core = h.getCore(); UpdateRequestProcessorChain chained = core.getUpdateProcessingChain("lang_id_tika"); assertNotNull(chained); @@ -464,6 +465,31 @@ public void testMapIndividual() throws Exception { assertTrue(mappedIndividual.containsKey("text2_ru")); } + @Test + public void testAllowlist() throws Exception { + ModifiableSolrParams parameters = new ModifiableSolrParams(); + parameters.add("langid.fl", "name,subject"); + parameters.add("langid.langField", "language_s"); + parameters.add("langid.allowlist", "no,en ,, ,sv, sv"); + liProcessor = createLangIdProcessor(parameters); + + // Make sure that empty language codes have been filtered out and others trimmed. + assertEquals(Set.of("no", "en", "sv"), liProcessor.langAllowlist); + } + + @Test + public void testAllowlistBackwardsCompatabilityWithLegacyAllowlist() throws Exception { + // The "legacy allowlist" is "langid.whitelist" + ModifiableSolrParams parameters = new ModifiableSolrParams(); + parameters.add("langid.fl", "name,subject"); + parameters.add("langid.langField", "language_s"); + parameters.add("langid.whitelist", "no,en ,, ,sv, sv"); + liProcessor = createLangIdProcessor(parameters); + + // Make sure that empty language codes have been filtered out and others trimmed. + assertEquals(Set.of("no", "en", "sv"), liProcessor.langAllowlist); + } + // Various utility methods private SolrInputDocument englishDoc() { diff --git a/solr/modules/hadoop-auth/src/test-files/core-site.xml b/solr/modules/llm/README.md similarity index 73% rename from solr/modules/hadoop-auth/src/test-files/core-site.xml rename to solr/modules/llm/README.md index 66718b71127..2467434a14b 100644 --- a/solr/modules/hadoop-auth/src/test-files/core-site.xml +++ b/solr/modules/llm/README.md @@ -1,4 +1,3 @@ - - - - hadoop.security.group.mapping - org.apache.solr.security.hadoop.HadoopAuthFakeGroupMapping - - +--> + +The Large Language Model module for Solr provides a set of mechanisms for plugging in third party LLM implementations. +It currently provides text vectorisation through langChain4j. + +See https://solr.apache.org/guide/solr/latest/query-guide/text-to-vector.html for how to get started. diff --git a/solr/modules/llm/build.gradle b/solr/modules/llm/build.gradle new file mode 100644 index 00000000000..b6c3d57eb15 --- /dev/null +++ b/solr/modules/llm/build.gradle @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +apply plugin: 'java-library' + +description = 'Indexing/Query time integration with LLM' + +dependencies { + compileOnly libs.stephenc.jcip.annotations + implementation project(':solr:core') + implementation project(':solr:solrj') + + implementation libs.apache.lucene.core + + implementation libs.langchain4j.core + runtimeOnly libs.langchain4j.cohere + runtimeOnly libs.langchain4j.hugging.face + runtimeOnly libs.langchain4j.mistral.ai + runtimeOnly libs.langchain4j.open.ai + + implementation libs.slf4j.api + + testImplementation project(':solr:test-framework') + testImplementation libs.junit.junit + testImplementation libs.commonsio.commonsio + +} diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/model/SolrTextToVectorModel.java b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/model/SolrTextToVectorModel.java new file mode 100644 index 00000000000..f798f27db5d --- /dev/null +++ b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/model/SolrTextToVectorModel.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.llm.texttovector.model; + +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.model.embedding.EmbeddingModel; +import java.lang.reflect.Method; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Map; +import java.util.Objects; +import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.RamUsageEstimator; +import org.apache.solr.core.SolrResourceLoader; +import org.apache.solr.llm.texttovector.store.TextToVectorModelException; +import org.apache.solr.llm.texttovector.store.rest.ManagedTextToVectorModelStore; + +/** + * This object wraps a {@link dev.langchain4j.model.embedding.EmbeddingModel} to encode text to + * vector. It's meant to be used as a managed resource with the {@link + * ManagedTextToVectorModelStore} + */ +public class SolrTextToVectorModel implements Accountable { + private static final long BASE_RAM_BYTES = + RamUsageEstimator.shallowSizeOfInstance(SolrTextToVectorModel.class); + private static final String TIMEOUT_PARAM = "timeout"; + private static final String MAX_SEGMENTS_PER_BATCH_PARAM = "maxSegmentsPerBatch"; + private static final String MAX_RETRIES_PARAM = "maxRetries"; + + private final String name; + private final Map params; + private final EmbeddingModel textToVector; + private final int hashCode; + + public static SolrTextToVectorModel getInstance( + SolrResourceLoader solrResourceLoader, + String className, + String name, + Map params) + throws TextToVectorModelException { + try { + /* + * The idea here is to build a {@link dev.langchain4j.model.embedding.EmbeddingModel} using inversion + * of control. + * Each model has its own list of parameters we don't know beforehand, but each {@link dev.langchain4j.model.embedding.EmbeddingModel} class + * has its own builder that uses setters with the same name of the parameter in input. + * */ + EmbeddingModel textToVector; + Class modelClass = solrResourceLoader.findClass(className, EmbeddingModel.class); + var builder = modelClass.getMethod("builder").invoke(null); + if (params != null) { + /* + * This block of code has the responsibility of instantiate a {@link + * dev.langchain4j.model.embedding.EmbeddingModel} using the params provided.classes have + * params of The specific implementation of {@link + * dev.langchain4j.model.embedding.EmbeddingModel} is not known beforehand. So we benefit of + * the design choice in langchain4j that each subclass implementing {@link + * dev.langchain4j.model.embedding.EmbeddingModel} uses setters with the same name of the + * param. + */ + for (String paramName : params.keySet()) { + /* + * When a param is not primitive, we need to instantiate the object explicitly and then call the + * setter method. + * N.B. when adding support to new models, pay attention to all the parameters they + * support, some of them may require to be handled in here as separate switch cases + */ + switch (paramName) { + case TIMEOUT_PARAM: + Duration timeOut = Duration.ofSeconds((Long) params.get(paramName)); + builder.getClass().getMethod(paramName, Duration.class).invoke(builder, timeOut); + break; + case MAX_SEGMENTS_PER_BATCH_PARAM: + builder + .getClass() + .getMethod(paramName, Integer.class) + .invoke(builder, ((Long) params.get(paramName)).intValue()); + break; + case MAX_RETRIES_PARAM: + builder + .getClass() + .getMethod(paramName, Integer.class) + .invoke(builder, ((Long) params.get(paramName)).intValue()); + break; + /* + * For primitive params if there's only one setter available, we call it. + * If there's choice we default to the string one + */ + default: + ArrayList paramNameMatches = new ArrayList<>(); + for (var method : builder.getClass().getMethods()) { + if (paramName.equals(method.getName()) && method.getParameterCount() == 1) { + paramNameMatches.add(method); + } + } + if (paramNameMatches.size() == 1) { + paramNameMatches.get(0).invoke(builder, params.get(paramName)); + } else { + builder + .getClass() + .getMethod(paramName, String.class) + .invoke(builder, params.get(paramName).toString()); + } + } + } + } + textToVector = (EmbeddingModel) builder.getClass().getMethod("build").invoke(builder); + return new SolrTextToVectorModel(name, textToVector, params); + } catch (final Exception e) { + throw new TextToVectorModelException("Model loading failed for " + className, e); + } + } + + public SolrTextToVectorModel( + String name, EmbeddingModel textToVector, Map params) { + this.name = name; + this.textToVector = textToVector; + this.params = params; + this.hashCode = calculateHashCode(); + } + + public float[] vectorise(String text) { + Embedding vector = textToVector.embed(text).content(); + return vector.vector(); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "(name=" + getName() + ")"; + } + + @Override + public long ramBytesUsed() { + return BASE_RAM_BYTES + + RamUsageEstimator.sizeOfObject(name) + + RamUsageEstimator.sizeOfObject(textToVector); + } + + @Override + public int hashCode() { + return hashCode; + } + + private int calculateHashCode() { + final int prime = 31; + int result = 1; + result = (prime * result) + Objects.hashCode(name); + result = (prime * result) + Objects.hashCode(textToVector); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (!(obj instanceof SolrTextToVectorModel)) return false; + final SolrTextToVectorModel other = (SolrTextToVectorModel) obj; + return Objects.equals(textToVector, other.textToVector) && Objects.equals(name, other.name); + } + + public String getName() { + return name; + } + + public String getEmbeddingModelClassName() { + return textToVector.getClass().getName(); + } + + public Map getParams() { + return params; + } +} diff --git a/solr/modules/hadoop-auth/src/java/org/apache/solr/util/configuration/providers/hadoop/package-info.java b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/model/package-info.java similarity index 87% rename from solr/modules/hadoop-auth/src/java/org/apache/solr/util/configuration/providers/hadoop/package-info.java rename to solr/modules/llm/src/java/org/apache/solr/llm/texttovector/model/package-info.java index 07a5a223d42..64e50f6f88b 100644 --- a/solr/modules/hadoop-auth/src/java/org/apache/solr/util/configuration/providers/hadoop/package-info.java +++ b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/model/package-info.java @@ -15,5 +15,5 @@ * limitations under the License. */ -/** Commonly used classes for Solr security framework. */ -package org.apache.solr.util.configuration.providers.hadoop; +/** APIs and classes for implementing text to vector logic. */ +package org.apache.solr.llm.texttovector.model; diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/search/TextToVectorQParserPlugin.java b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/search/TextToVectorQParserPlugin.java new file mode 100644 index 00000000000..17749eba7b6 --- /dev/null +++ b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/search/TextToVectorQParserPlugin.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.llm.texttovector.search; + +import java.io.IOException; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.ResourceLoader; +import org.apache.lucene.util.ResourceLoaderAware; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.SolrResourceLoader; +import org.apache.solr.llm.texttovector.model.SolrTextToVectorModel; +import org.apache.solr.llm.texttovector.store.rest.ManagedTextToVectorModelStore; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.rest.ManagedResource; +import org.apache.solr.rest.ManagedResourceObserver; +import org.apache.solr.schema.DenseVectorField; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.search.QParser; +import org.apache.solr.search.QParserPlugin; +import org.apache.solr.search.SyntaxError; +import org.apache.solr.search.neural.KnnQParser; + +/** + * A neural query parser that encode the query to a vector and then run K-nearest neighbors search + * on Dense Vector fields. See Wiki page + * https://solr.apache.org/guide/solr/latest/query-guide/dense-vector-search.html + */ +public class TextToVectorQParserPlugin extends QParserPlugin + implements ResourceLoaderAware, ManagedResourceObserver { + public static final String EMBEDDING_MODEL_PARAM = "model"; + private ManagedTextToVectorModelStore modelStore = null; + + @Override + public QParser createParser( + String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { + return new TextToVectorQParser(qstr, localParams, params, req); + } + + @Override + public void inform(ResourceLoader loader) throws IOException { + final SolrResourceLoader solrResourceLoader = (SolrResourceLoader) loader; + ManagedTextToVectorModelStore.registerManagedTextToVectorModelStore(solrResourceLoader, this); + } + + @Override + public void onManagedResourceInitialized(NamedList args, ManagedResource res) + throws SolrException { + if (res instanceof ManagedTextToVectorModelStore) { + modelStore = (ManagedTextToVectorModelStore) res; + } + if (modelStore != null) { + modelStore.loadStoredModels(); + } + } + + public class TextToVectorQParser extends KnnQParser { + + public TextToVectorQParser( + String queryString, SolrParams localParams, SolrParams params, SolrQueryRequest req) { + super(queryString, localParams, params, req); + } + + @Override + public Query parse() throws SyntaxError { + checkParam(qstr, "Query string is empty, nothing to vectorise"); + final String embeddingModelName = localParams.get(EMBEDDING_MODEL_PARAM); + checkParam(embeddingModelName, "The 'model' parameter is missing"); + SolrTextToVectorModel textToVector = modelStore.getModel(embeddingModelName); + + if (textToVector != null) { + final SchemaField schemaField = req.getCore().getLatestSchema().getField(getFieldName()); + final DenseVectorField denseVectorType = getCheckedFieldType(schemaField); + int fieldDimensions = denseVectorType.getDimension(); + VectorEncoding vectorEncoding = denseVectorType.getVectorEncoding(); + final int topK = localParams.getInt(TOP_K, DEFAULT_TOP_K); + + switch (vectorEncoding) { + case FLOAT32: + { + float[] vectorToSearch = textToVector.vectorise(qstr); + checkVectorDimension(vectorToSearch.length, fieldDimensions); + return new KnnFloatVectorQuery( + schemaField.getName(), vectorToSearch, topK, getFilterQuery()); + } + default: + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Vector Encoding not supported : " + vectorEncoding); + } + } else { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "The model requested '" + + embeddingModelName + + "' can't be found in the store: " + + ManagedTextToVectorModelStore.REST_END_POINT); + } + } + } + + private void checkVectorDimension(int inputVectorDimension, int fieldVectorDimension) { + if (inputVectorDimension != fieldVectorDimension) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "incorrect vector dimension." + + " The vector value has size " + + inputVectorDimension + + " while it is expected a vector with size " + + fieldVectorDimension); + } + } + + private void checkParam(String value, String message) { + if (value == null || value.isBlank()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, message); + } + } +} diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/search/package-info.java b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/search/package-info.java new file mode 100644 index 00000000000..9fbf84e62c6 --- /dev/null +++ b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/search/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** APIs and classes for implementing text to vector QueryParsers. */ +package org.apache.solr.llm.texttovector.search; diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/BeforeReconnect.java b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/TextToVectorModelException.java similarity index 61% rename from solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/BeforeReconnect.java rename to solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/TextToVectorModelException.java index 6e028056013..076f2b45f9b 100644 --- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/BeforeReconnect.java +++ b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/TextToVectorModelException.java @@ -14,19 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.common.cloud; +package org.apache.solr.llm.texttovector.store; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.state.ConnectionState; -import org.apache.curator.framework.state.ConnectionStateListener; +public class TextToVectorModelException extends RuntimeException { -public interface BeforeReconnect extends ConnectionStateListener { - public void command(); + private static final long serialVersionUID = 1L; - @Override - default void stateChanged(CuratorFramework client, ConnectionState newState) { - if (newState == ConnectionState.LOST || newState == ConnectionState.SUSPENDED) { - command(); - } + public TextToVectorModelException(String message) { + super(message); + } + + public TextToVectorModelException(String message, Exception cause) { + super(message, cause); } } diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/TextToVectorModelStore.java b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/TextToVectorModelStore.java new file mode 100644 index 00000000000..cf1db239d44 --- /dev/null +++ b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/TextToVectorModelStore.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.llm.texttovector.store; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.solr.llm.texttovector.model.SolrTextToVectorModel; + +/** Simple store to manage CRUD operations on the {@link SolrTextToVectorModel} */ +public class TextToVectorModelStore { + + private final Map availableModels; + + public TextToVectorModelStore() { + availableModels = Collections.synchronizedMap(new LinkedHashMap<>()); + } + + public SolrTextToVectorModel getModel(String name) { + return availableModels.get(name); + } + + public void clear() { + availableModels.clear(); + } + + public List getModels() { + synchronized (availableModels) { + final List availableModelsValues = + new ArrayList(availableModels.values()); + return Collections.unmodifiableList(availableModelsValues); + } + } + + @Override + public String toString() { + return "ModelStore [availableModels=" + availableModels.keySet() + "]"; + } + + public SolrTextToVectorModel delete(String modelName) { + return availableModels.remove(modelName); + } + + public void addModel(SolrTextToVectorModel modeldata) throws TextToVectorModelException { + final String name = modeldata.getName(); + if (availableModels.putIfAbsent(modeldata.getName(), modeldata) != null) { + throw new TextToVectorModelException( + "model '" + name + "' already exists. Please use a different name"); + } + } +} diff --git a/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/package-info.java b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/package-info.java similarity index 89% rename from solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/package-info.java rename to solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/package-info.java index cfe322e9b26..630ac6085a8 100644 --- a/solr/modules/hadoop-auth/src/java/org/apache/solr/security/hadoop/package-info.java +++ b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/package-info.java @@ -15,5 +15,5 @@ * limitations under the License. */ -/** Commonly used classes for Solr security framework. */ -package org.apache.solr.security.hadoop; +/** Contains model store related classes. */ +package org.apache.solr.llm.texttovector.store; diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/rest/ManagedTextToVectorModelStore.java b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/rest/ManagedTextToVectorModelStore.java new file mode 100644 index 00000000000..0652ec54d77 --- /dev/null +++ b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/rest/ManagedTextToVectorModelStore.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.llm.texttovector.store.rest; + +import java.lang.invoke.MethodHandles; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import net.jcip.annotations.ThreadSafe; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SolrResourceLoader; +import org.apache.solr.llm.texttovector.model.SolrTextToVectorModel; +import org.apache.solr.llm.texttovector.store.TextToVectorModelException; +import org.apache.solr.llm.texttovector.store.TextToVectorModelStore; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.rest.BaseSolrResource; +import org.apache.solr.rest.ManagedResource; +import org.apache.solr.rest.ManagedResourceObserver; +import org.apache.solr.rest.ManagedResourceStorage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Managed Resource wrapper for the the {@link TextToVectorModelStore} to expose it via REST */ +@ThreadSafe +public class ManagedTextToVectorModelStore extends ManagedResource + implements ManagedResource.ChildResourceSupport { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + /** the model store rest endpoint */ + public static final String REST_END_POINT = "/schema/text-to-vector-model-store"; + + /** Managed model store: the name of the attribute containing all the models of a model store */ + private static final String MODELS_JSON_FIELD = "models"; + + /** name of the attribute containing a class */ + static final String CLASS_KEY = "class"; + + /** name of the attribute containing a name */ + static final String NAME_KEY = "name"; + + /** name of the attribute containing parameters */ + static final String PARAMS_KEY = "params"; + + public static void registerManagedTextToVectorModelStore( + SolrResourceLoader solrResourceLoader, ManagedResourceObserver managedResourceObserver) { + solrResourceLoader + .getManagedResourceRegistry() + .registerManagedResource( + REST_END_POINT, ManagedTextToVectorModelStore.class, managedResourceObserver); + } + + public static ManagedTextToVectorModelStore getManagedModelStore(SolrCore core) { + return (ManagedTextToVectorModelStore) core.getRestManager().getManagedResource(REST_END_POINT); + } + + /** + * Returns the available models as a list of Maps objects. After an update the managed resources + * needs to return the resources in this format in order to store in json somewhere (zookeeper, + * disk...) + * + * @return the available models as a list of Maps objects + */ + private static List modelsAsManagedResources(List models) { + return models.stream() + .map(ManagedTextToVectorModelStore::toModelMap) + .collect(Collectors.toList()); + } + + @SuppressWarnings("unchecked") + public static SolrTextToVectorModel fromModelMap( + SolrResourceLoader solrResourceLoader, Map embeddingModel) { + return SolrTextToVectorModel.getInstance( + solrResourceLoader, + (String) embeddingModel.get(CLASS_KEY), // modelClassName + (String) embeddingModel.get(NAME_KEY), // modelName + (Map) embeddingModel.get(PARAMS_KEY)); + } + + private static LinkedHashMap toModelMap(SolrTextToVectorModel model) { + final LinkedHashMap modelMap = new LinkedHashMap<>(5, 1.0f); + modelMap.put(NAME_KEY, model.getName()); + modelMap.put(CLASS_KEY, model.getEmbeddingModelClassName()); + modelMap.put(PARAMS_KEY, model.getParams()); + return modelMap; + } + + private final TextToVectorModelStore store; + private Object managedData; + + public ManagedTextToVectorModelStore( + String resourceId, SolrResourceLoader loader, ManagedResourceStorage.StorageIO storageIO) + throws SolrException { + super(resourceId, loader, storageIO); + store = new TextToVectorModelStore(); + } + + @Override + protected ManagedResourceStorage createStorage( + ManagedResourceStorage.StorageIO storageIO, SolrResourceLoader loader) throws SolrException { + return new ManagedResourceStorage.JsonStorage(storageIO, loader, -1); + } + + @Override + protected void onManagedDataLoadedFromStorage(NamedList managedInitArgs, Object managedData) + throws SolrException { + store.clear(); + this.managedData = managedData; + } + + public void loadStoredModels() { + log.info("------ managed models ~ loading ------"); + + if ((managedData != null) && (managedData instanceof List)) { + @SuppressWarnings({"unchecked"}) + final List> textToVectorModels = (List>) managedData; + for (final Map textToVectorModel : textToVectorModels) { + addModelFromMap(textToVectorModel); + } + } + } + + private void addModelFromMap(Map modelMap) { + try { + addModel(fromModelMap(solrResourceLoader, modelMap)); + } catch (final TextToVectorModelException e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); + } + } + + public void addModel(SolrTextToVectorModel model) throws TextToVectorModelException { + try { + if (log.isInfoEnabled()) { + log.info("adding model {}", model.getName()); + } + store.addModel(model); + } catch (final TextToVectorModelException e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); + } + } + + @SuppressWarnings("unchecked") + @Override + protected Object applyUpdatesToManagedData(Object updates) { + if (updates instanceof List) { + final List> textToVectorModels = (List>) updates; + for (final Map textToVectorModel : textToVectorModels) { + addModelFromMap(textToVectorModel); + } + } + + if (updates instanceof Map) { + final Map map = (Map) updates; + addModelFromMap(map); + } + + return modelsAsManagedResources(store.getModels()); + } + + @Override + public void doDeleteChild(BaseSolrResource endpoint, String childId) { + store.delete(childId); + storeManagedData(applyUpdatesToManagedData(null)); + } + + /** + * Called to retrieve a named part (the given childId) of the resource at the given endpoint. + * Note: since we have a unique child managed store we ignore the childId. + */ + @Override + public void doGet(BaseSolrResource endpoint, String childId) { + final SolrQueryResponse response = endpoint.getSolrResponse(); + response.add(MODELS_JSON_FIELD, modelsAsManagedResources(store.getModels())); + } + + public SolrTextToVectorModel getModel(String modelName) { + return store.getModel(modelName); + } + + @Override + public String toString() { + return "ManagedModelStore [store=" + store + "]"; + } +} diff --git a/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/rest/package-info.java b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/rest/package-info.java new file mode 100644 index 00000000000..56ae30f2ebe --- /dev/null +++ b/solr/modules/llm/src/java/org/apache/solr/llm/texttovector/store/rest/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Contains the {@link org.apache.solr.rest.ManagedResource} that encapsulate the model stores. */ +package org.apache.solr.llm.texttovector.store.rest; diff --git a/solr/core/src/test-files/solr/configsets/upload/with-lib-directive/managed-schema.xml b/solr/modules/llm/src/java/overview.html similarity index 75% rename from solr/core/src/test-files/solr/configsets/upload/with-lib-directive/managed-schema.xml rename to solr/modules/llm/src/java/overview.html index 25a37e6eee3..29daef11a7b 100644 --- a/solr/core/src/test-files/solr/configsets/upload/with-lib-directive/managed-schema.xml +++ b/solr/modules/llm/src/java/overview.html @@ -1,4 +1,3 @@ - - - - - - - - - + + +Apache Solr Search Server: text to vector module + + diff --git a/solr/modules/hadoop-auth/src/test-files/log4j2.xml b/solr/modules/llm/src/test-files/log4j2.xml similarity index 86% rename from solr/modules/hadoop-auth/src/test-files/log4j2.xml rename to solr/modules/llm/src/test-files/log4j2.xml index 0e8f08c0c61..5e696935965 100644 --- a/solr/modules/hadoop-auth/src/test-files/log4j2.xml +++ b/solr/modules/llm/src/test-files/log4j2.xml @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. --> - + @@ -27,14 +27,13 @@ + - - diff --git a/solr/modules/llm/src/test-files/modelExamples/cohere-model.json b/solr/modules/llm/src/test-files/modelExamples/cohere-model.json new file mode 100644 index 00000000000..5eef83667b5 --- /dev/null +++ b/solr/modules/llm/src/test-files/modelExamples/cohere-model.json @@ -0,0 +1,13 @@ +{ + "class": "dev.langchain4j.model.cohere.CohereEmbeddingModel", + "name": "cohere-1", + "params": { + "baseUrl": "https://api.cohere.ai/v1/", + "apiKey": "apiKey-cohere", + "modelName": "embed-english-light-v3.0", + "inputType": "search_document", + "timeout": 60, + "logRequests": true, + "logResponses": true + } +} diff --git a/solr/modules/llm/src/test-files/modelExamples/dummy-model-ambiguous.json b/solr/modules/llm/src/test-files/modelExamples/dummy-model-ambiguous.json new file mode 100644 index 00000000000..43de925cf9d --- /dev/null +++ b/solr/modules/llm/src/test-files/modelExamples/dummy-model-ambiguous.json @@ -0,0 +1,8 @@ +{ + "class": "org.apache.solr.llm.texttovector.model.DummyEmbeddingModel", + "name": "dummy-1", + "params": { + "embedding": [1.0, 2.0, 3.0, 4.0], + "ambiguous": 10 + } +} diff --git a/solr/modules/llm/src/test-files/modelExamples/dummy-model-unsupported.json b/solr/modules/llm/src/test-files/modelExamples/dummy-model-unsupported.json new file mode 100644 index 00000000000..9af02f14003 --- /dev/null +++ b/solr/modules/llm/src/test-files/modelExamples/dummy-model-unsupported.json @@ -0,0 +1,8 @@ +{ + "class": "org.apache.solr.llm.texttovector.model.DummyEmbeddingModel", + "name": "dummy-1", + "params": { + "embedding": [1.0, 2.0, 3.0, 4.0], + "unsupported": 10 + } +} diff --git a/solr/modules/llm/src/test-files/modelExamples/dummy-model.json b/solr/modules/llm/src/test-files/modelExamples/dummy-model.json new file mode 100644 index 00000000000..00603b8369b --- /dev/null +++ b/solr/modules/llm/src/test-files/modelExamples/dummy-model.json @@ -0,0 +1,7 @@ +{ + "class": "org.apache.solr.llm.texttovector.model.DummyEmbeddingModel", + "name": "dummy-1", + "params": { + "embedding": [1.0, 2.0, 3.0, 4.0] + } +} diff --git a/solr/modules/llm/src/test-files/modelExamples/huggingface-model.json b/solr/modules/llm/src/test-files/modelExamples/huggingface-model.json new file mode 100644 index 00000000000..f0a72cdf758 --- /dev/null +++ b/solr/modules/llm/src/test-files/modelExamples/huggingface-model.json @@ -0,0 +1,8 @@ +{ + "class": "dev.langchain4j.model.huggingface.HuggingFaceEmbeddingModel", + "name": "huggingface-1", + "params": { + "accessToken": "apiKey-huggingface", + "modelId": "sentence-transformers/all-MiniLM-L6-v2" + } +} diff --git a/solr/modules/llm/src/test-files/modelExamples/mistralai-model.json b/solr/modules/llm/src/test-files/modelExamples/mistralai-model.json new file mode 100644 index 00000000000..082a1a434f1 --- /dev/null +++ b/solr/modules/llm/src/test-files/modelExamples/mistralai-model.json @@ -0,0 +1,13 @@ +{ + "class": "dev.langchain4j.model.mistralai.MistralAiEmbeddingModel", + "name": "mistralai-1", + "params": { + "baseUrl": "https://api.mistral.ai/v1", + "apiKey": "apiKey-mistralAI", + "modelName": "mistral-embed", + "timeout": 60, + "logRequests": true, + "logResponses": true, + "maxRetries": 5 + } +} diff --git a/solr/modules/llm/src/test-files/modelExamples/openai-model.json b/solr/modules/llm/src/test-files/modelExamples/openai-model.json new file mode 100644 index 00000000000..a90d5bd6285 --- /dev/null +++ b/solr/modules/llm/src/test-files/modelExamples/openai-model.json @@ -0,0 +1,13 @@ +{ + "class": "dev.langchain4j.model.openai.OpenAiEmbeddingModel", + "name": "openai-1", + "params": { + "baseUrl": "https://api.openai.com/v1", + "apiKey": "apiKey-openAI", + "modelName": "text-embedding-3-small", + "timeout": 60, + "logRequests": true, + "logResponses": true, + "maxRetries": 5 + } +} diff --git a/solr/modules/llm/src/test-files/solr/collection1/conf/schema.xml b/solr/modules/llm/src/test-files/solr/collection1/conf/schema.xml new file mode 100644 index 00000000000..92aa0e76591 --- /dev/null +++ b/solr/modules/llm/src/test-files/solr/collection1/conf/schema.xml @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + diff --git a/solr/modules/llm/src/test-files/solr/collection1/conf/solrconfig-llm.xml b/solr/modules/llm/src/test-files/solr/collection1/conf/solrconfig-llm.xml new file mode 100644 index 00000000000..3a1d05285fc --- /dev/null +++ b/solr/modules/llm/src/test-files/solr/collection1/conf/solrconfig-llm.xml @@ -0,0 +1,57 @@ + + + + + ${tests.luceneMatchVersion:LATEST} + ${solr.data.dir:} + + + + + + + + + + + + + + + + + + 15000 + false + + + 1000 + + + ${solr.data.dir:} + + + + + + + explicit + json + true + id + + + + diff --git a/solr/modules/llm/src/test-files/solr/solr.xml b/solr/modules/llm/src/test-files/solr/solr.xml new file mode 100644 index 00000000000..7506c1c8951 --- /dev/null +++ b/solr/modules/llm/src/test-files/solr/solr.xml @@ -0,0 +1,41 @@ + + + + + + ${shareSchema:false} + ${configSetBaseDir:configsets} + ${coreRootDirectory:.} + + + ${urlScheme:} + ${socketTimeout:90000} + ${connTimeout:15000} + + + + 127.0.0.1 + ${hostPort:8983} + ${solr.zkclienttimeout:30000} + ${genericCoreNodeNames:true} + ${leaderVoteWait:10000} + ${distribUpdateConnTimeout:45000} + ${distribUpdateSoTimeout:340000} + + + diff --git a/solr/modules/llm/src/test/org/apache/solr/llm/TestLlmBase.java b/solr/modules/llm/src/test/org/apache/solr/llm/TestLlmBase.java new file mode 100644 index 00000000000..6fb391ad364 --- /dev/null +++ b/solr/modules/llm/src/test/org/apache/solr/llm/TestLlmBase.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.llm; + +import java.lang.invoke.MethodHandles; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.apache.commons.io.file.PathUtils; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.llm.texttovector.store.rest.ManagedTextToVectorModelStore; +import org.apache.solr.util.RestTestBase; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestLlmBase extends RestTestBase { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + protected static Path tmpSolrHome; + protected static Path tmpConfDir; + + public static final String MODEL_FILE_NAME = "_schema_text-to-vector-model-store.json"; + protected static final String COLLECTION = "collection1"; + protected static final String CONF_DIR = COLLECTION + "/conf"; + + protected static Path embeddingModelStoreFile = null; + + protected static String IDField = "id"; + protected static String vectorField = "vector"; + protected static String vectorField2 = "vector2"; + protected static String vectorFieldByteEncoding = "vector_byte_encoding"; + + protected static void setupTest( + String solrconfig, String schema, boolean buildIndex, boolean persistModelStore) + throws Exception { + initFolders(persistModelStore); + createJettyAndHarness( + tmpSolrHome.toAbsolutePath().toString(), solrconfig, schema, "/solr", true, null); + if (buildIndex) prepareIndex(); + } + + protected static void initFolders(boolean isPersistent) throws Exception { + tmpSolrHome = createTempDir(); + tmpConfDir = tmpSolrHome.resolve(CONF_DIR); + tmpConfDir.toFile().deleteOnExit(); + PathUtils.copyDirectory(TEST_PATH(), tmpSolrHome.toAbsolutePath()); + + final Path modelStore = tmpConfDir.resolve(MODEL_FILE_NAME); + + if (isPersistent) { + embeddingModelStoreFile = modelStore; + } + + if (Files.exists(modelStore)) { + if (log.isInfoEnabled()) { + log.info("remove model store config file in {}", modelStore.toAbsolutePath()); + } + Files.delete(modelStore); + } + + System.setProperty("managed.schema.mutable", "true"); + } + + protected static void afterTest() throws Exception { + if (null != restTestHarness) { + restTestHarness.close(); + restTestHarness = null; + } + solrClientTestRule.reset(); + if (null != tmpSolrHome) { + PathUtils.deleteDirectory(tmpSolrHome); + tmpSolrHome = null; + } + System.clearProperty("managed.schema.mutable"); + } + + public static void loadModel(String fileName, String status) throws Exception { + final URL url = TestLlmBase.class.getResource("/modelExamples/" + fileName); + final String multipleModels = Files.readString(Path.of(url.toURI()), StandardCharsets.UTF_8); + + assertJPut( + ManagedTextToVectorModelStore.REST_END_POINT, + multipleModels, + "/responseHeader/status==" + status); + } + + public static void loadModel(String fileName) throws Exception { + final URL url = TestLlmBase.class.getResource("/modelExamples/" + fileName); + final String multipleModels = Files.readString(Path.of(url.toURI()), StandardCharsets.UTF_8); + + assertJPut( + ManagedTextToVectorModelStore.REST_END_POINT, multipleModels, "/responseHeader/status==0"); + } + + protected static void prepareIndex() throws Exception { + List docsToIndex = prepareDocs(); + for (SolrInputDocument doc : docsToIndex) { + assertU(adoc(doc)); + } + + assertU(commit()); + } + + private static List prepareDocs() { + int docsCount = 13; + List docs = new ArrayList<>(docsCount); + for (int i = 1; i < docsCount + 1; i++) { + SolrInputDocument doc = new SolrInputDocument(); + doc.addField(IDField, i); + docs.add(doc); + } + + docs.get(0) + .addField(vectorField, Arrays.asList(1f, 2f, 3f, 4f)); // cosine distance vector1= 1.0 + docs.get(1) + .addField( + vectorField, Arrays.asList(1.5f, 2.5f, 3.5f, 4.5f)); // cosine distance vector1= 0.998 + docs.get(2) + .addField( + vectorField, + Arrays.asList(7.5f, 15.5f, 17.5f, 22.5f)); // cosine distance vector1= 0.992 + docs.get(3) + .addField( + vectorField, Arrays.asList(1.4f, 2.4f, 3.4f, 4.4f)); // cosine distance vector1= 0.999 + docs.get(4) + .addField(vectorField, Arrays.asList(30f, 22f, 35f, 20f)); // cosine distance vector1= 0.862 + docs.get(5) + .addField(vectorField, Arrays.asList(40f, 1f, 1f, 200f)); // cosine distance vector1= 0.756 + docs.get(6) + .addField(vectorField, Arrays.asList(5f, 10f, 20f, 40f)); // cosine distance vector1= 0.970 + docs.get(7) + .addField( + vectorField, Arrays.asList(120f, 60f, 30f, 15f)); // cosine distance vector1= 0.515 + docs.get(8) + .addField( + vectorField, Arrays.asList(200f, 50f, 100f, 25f)); // cosine distance vector1= 0.554 + docs.get(9) + .addField( + vectorField, Arrays.asList(1.8f, 2.5f, 3.7f, 4.9f)); // cosine distance vector1= 0.997 + docs.get(10) + .addField(vectorField2, Arrays.asList(1f, 2f, 3f, 4f)); // cosine distance vector2= 1 + docs.get(11) + .addField( + vectorField2, + Arrays.asList(7.5f, 15.5f, 17.5f, 22.5f)); // cosine distance vector2= 0.992 + docs.get(12) + .addField( + vectorField2, Arrays.asList(1.5f, 2.5f, 3.5f, 4.5f)); // cosine distance vector2= 0.998 + + docs.get(0).addField(vectorFieldByteEncoding, Arrays.asList(1, 2, 3, 4)); + docs.get(1).addField(vectorFieldByteEncoding, Arrays.asList(2, 2, 1, 4)); + docs.get(2).addField(vectorFieldByteEncoding, Arrays.asList(1, 2, 1, 2)); + docs.get(3).addField(vectorFieldByteEncoding, Arrays.asList(7, 2, 1, 3)); + docs.get(4).addField(vectorFieldByteEncoding, Arrays.asList(19, 2, 4, 4)); + docs.get(5).addField(vectorFieldByteEncoding, Arrays.asList(19, 2, 4, 4)); + docs.get(6).addField(vectorFieldByteEncoding, Arrays.asList(18, 2, 4, 4)); + docs.get(7).addField(vectorFieldByteEncoding, Arrays.asList(8, 3, 2, 4)); + + return docs; + } +} diff --git a/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/model/DummyEmbeddingModel.java b/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/model/DummyEmbeddingModel.java new file mode 100644 index 00000000000..00edcba114b --- /dev/null +++ b/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/model/DummyEmbeddingModel.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.llm.texttovector.model; + +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.embedding.EmbeddingModel; +import dev.langchain4j.model.output.Response; +import java.util.ArrayList; +import java.util.List; + +public class DummyEmbeddingModel implements EmbeddingModel { + final float[] embedding; + + public DummyEmbeddingModel(float[] embedding) { + this.embedding = embedding; + } + + @Override + public Response embed(String text) { + Embedding dummy = new Embedding(embedding); + return new Response(dummy); + } + + @Override + public Response embed(TextSegment textSegment) { + Embedding dummy = new Embedding(embedding); + return new Response(dummy); + } + + @Override + public Response> embedAll(List textSegments) { + return null; + } + + @Override + public int dimension() { + return embedding.length; + } + + public static DummyEmbeddingModelBuilder builder() { + return new DummyEmbeddingModelBuilder(); + } + + public static class DummyEmbeddingModelBuilder { + private float[] builderEmbeddings; + private int intValue; + + public DummyEmbeddingModelBuilder() {} + + public DummyEmbeddingModelBuilder embedding(ArrayList embeddings) { + this.builderEmbeddings = new float[embeddings.size()]; + for (int i = 0; i < embeddings.size(); i++) { + this.builderEmbeddings[i] = embeddings.get(i).floatValue(); + } + return this; + } + + public DummyEmbeddingModelBuilder unsupported(Integer input) { + return this; + } + + public DummyEmbeddingModelBuilder ambiguous(int input) { + this.intValue = input; + return this; + } + + public DummyEmbeddingModelBuilder ambiguous(String input) { + this.intValue = Integer.valueOf(input); + return this; + } + + public DummyEmbeddingModel build() { + return new DummyEmbeddingModel(this.builderEmbeddings); + } + } +} diff --git a/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/model/DummyEmbeddingModelTest.java b/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/model/DummyEmbeddingModelTest.java new file mode 100644 index 00000000000..823f591fb95 --- /dev/null +++ b/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/model/DummyEmbeddingModelTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.llm.texttovector.model; + +import org.apache.solr.SolrTestCase; +import org.junit.Test; + +public class DummyEmbeddingModelTest extends SolrTestCase { + + @Test + public void constructAndEmbed() throws Exception { + assertEquals( + "[1.0, 2.0, 3.0, 4.0]", + new DummyEmbeddingModel(new float[] {1, 2, 3, 4}) + .embed("hello") + .content() + .vectorAsList() + .toString()); + assertEquals( + "[8.0, 7.0, 6.0, 5.0]", + new DummyEmbeddingModel(new float[] {8, 7, 6, 5}) + .embed("world") + .content() + .vectorAsList() + .toString()); + assertEquals( + "[0.0, 0.0, 4.0, 2.0]", + new DummyEmbeddingModel(new float[] {0, 0, 4, 2}) + .embed("answer") + .content() + .vectorAsList() + .toString()); + } +} diff --git a/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/search/TextToVectorQParserTest.java b/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/search/TextToVectorQParserTest.java new file mode 100644 index 00000000000..5c406f08217 --- /dev/null +++ b/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/search/TextToVectorQParserTest.java @@ -0,0 +1,386 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.llm.texttovector.search; + +import java.util.Arrays; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.llm.TestLlmBase; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TextToVectorQParserTest extends TestLlmBase { + @BeforeClass + public static void init() throws Exception { + setupTest("solrconfig-llm.xml", "schema.xml", true, false); + loadModel("dummy-model.json"); + } + + @Test + public void notExistentModel_shouldThrowException() throws Exception { + final String solrQuery = "{!knn_text_to_vector model=not-exist f=vector topK=5}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/error/msg=='The model requested \\'not-exist\\' can\\'t be found in the store: /schema/text-to-vector-model-store'", + "/error/code==400"); + } + + @Test + public void missingModelParam_shouldThrowException() throws Exception { + final String solrQuery = "{!knn_text_to_vector f=vector topK=5}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/error/msg=='The \\'model\\' parameter is missing'", + "/error/code==400"); + } + + @Test + public void incorrectVectorFieldType_shouldThrowException() throws Exception { + final String solrQuery = "{!knn_text_to_vector model=dummy-1 f=id topK=5}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/error/msg=='only DenseVectorField is compatible with Vector Query Parsers'", + "/error/code==400"); + } + + @Test + public void undefinedVectorField_shouldThrowException() throws Exception { + final String solrQuery = "{!knn_text_to_vector model=dummy-1 f=notExistent topK=5}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/error/msg=='undefined field: \"notExistent\"'", + "/error/code==400"); + } + + @Test + public void missingVectorFieldParam_shouldThrowException() throws Exception { + final String solrQuery = "{!knn_text_to_vector model=dummy-1 topK=5}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/error/msg=='the Dense Vector field \\'f\\' is missing'", + "/error/code==400"); + } + + @Test + public void vectorByteEncodingField_shouldRaiseException() throws Exception { + final String solrQuery = + "{!knn_text_to_vector model=dummy-1 f=vector_byte_encoding topK=5}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/error/msg=='Vector Encoding not supported : BYTE'", + "/error/code==500"); + } + + @Test + public void missingQueryToEmbed_shouldThrowException() throws Exception { + final String solrQuery = "{!knn_text_to_vector model=dummy-1 f=vector topK=5}"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/error/msg=='Query string is empty, nothing to vectorise'", + "/error/code==400"); + } + + @Test + public void incorrectVectorToSearchDimension_shouldThrowException() throws Exception { + final String solrQuery = + "{!knn_text_to_vector model=dummy-1 f=2048_float_vector topK=5}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/error/msg=='incorrect vector dimension. The vector value has size 4 while it is expected a vector with size 2048'", + "/error/code==400"); + } + + @Test + public void topK_shouldEmbedAndReturnOnlyTopKResults() throws Exception { + final String solrQuery = "{!knn_text_to_vector model=dummy-1 f=vector topK=5}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/response/numFound==5]", + "/response/docs/[0]/id=='1'", + "/response/docs/[1]/id=='4'", + "/response/docs/[2]/id=='2'", + "/response/docs/[3]/id=='10'", + "/response/docs/[4]/id=='3'"); + } + + @Test + public void vectorFieldParam_shouldSearchOnThatField() throws Exception { + final String solrQuery = "{!knn_text_to_vector model=dummy-1 f=vector2 topK=5}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/response/numFound==3]", + "/response/docs/[0]/id=='11'", + "/response/docs/[1]/id=='13'", + "/response/docs/[2]/id=='12'"); + } + + @Test + public void embeddedQuery_shouldRankBySimilarityFunction() throws Exception { + final String solrQuery = "{!knn_text_to_vector model=dummy-1 f=vector topK=10}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/response/numFound==10]", + "/response/docs/[0]/id=='1'", + "/response/docs/[1]/id=='4'", + "/response/docs/[2]/id=='2'", + "/response/docs/[3]/id=='10'", + "/response/docs/[4]/id=='3'", + "/response/docs/[5]/id=='7'", + "/response/docs/[6]/id=='5'", + "/response/docs/[7]/id=='6'", + "/response/docs/[8]/id=='9'", + "/response/docs/[9]/id=='8'"); + } + + @Test + public void embeddedQueryUsedInFilter_shouldFilterResultsBeforeTheQueryExecution() + throws Exception { + final String solrQuery = "{!knn_text_to_vector model=dummy-1 f=vector topK=4}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery("id:(3 4 9 2)"); + query.setFilterQueries(solrQuery); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/response/numFound==2]", + "/response/docs/[0]/id=='2'", + "/response/docs/[1]/id=='4'"); + } + + @Test + public void embeddedQueryUsedInFilters_shouldFilterResultsBeforeTheQueryExecution() + throws Exception { + final String solrQuery = "{!knn_text_to_vector model=dummy-1 f=vector topK=4}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery("id:(3 4 9 2)"); + query.setFilterQueries(solrQuery, "id:(4 20 9)"); + query.add("fl", "id"); + + // topK=4 -> 1,4,2,10 + assertJQ( + "/query" + query.toQueryString(), "/response/numFound==1]", "/response/docs/[0]/id=='4'"); + } + + @Test + public void embeddedQueryUsedInFiltersWithPreFilter_shouldFilterResultsBeforeTheQueryExecution() + throws Exception { + final String solrQuery = + "{!knn_text_to_vector model=dummy-1 f=vector topK=4 preFilter='id:(1 4 7 8 9)'}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery("id:(3 4 9 2)"); + query.setFilterQueries(solrQuery, "id:(4 20 9)"); + query.add("fl", "id"); + + // topK=4 w/localparam preFilter -> 1,4,7,9 + assertJQ( + "/query" + query.toQueryString(), + "/response/numFound==2]", + "/response/docs/[0]/id=='4'", + "/response/docs/[1]/id=='9'"); + } + + @Test + public void embeddedQueryUsedInFilters_rejectIncludeExclude() throws Exception { + for (String fq : + Arrays.asList( + "{!knn_text_to_vector model=dummy-1 f=vector topK=5 includeTags=xxx}hello world", + "{!knn_text_to_vector model=dummy-1 f=vector topK=5 excludeTags=xxx}hello world")) { + final SolrQuery query = new SolrQuery(); + query.setQuery("*:*"); + query.setFilterQueries(fq); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/error/msg=='Knn Query Parser used as a filter does not support includeTags or excludeTags localparams'", + "/error/code==400"); + } + } + + @Test + public void embeddedQueryAsSubQuery() throws Exception { + final String solrQuery = + "*:* AND {!knn_text_to_vector model=dummy-1 f=vector topK=5 v='hello world'}"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.setFilterQueries("id:(2 4 7 9 8 20 3)"); + query.add("fl", "id"); + + // When knn parser is a subquery, it should not pre-filter on any global fq params + // topK -> 1,4,2,10,3 -> fq -> 4,2,3 + assertJQ( + "/query" + query.toQueryString(), + "/response/numFound==3]", + "/response/docs/[0]/id=='4'", + "/response/docs/[1]/id=='2'", + "/response/docs/[2]/id=='3'"); + } + + @Test + public void embeddedQueryAsSubQuery_withPreFilter() throws Exception { + final String solrQuery = + "*:* AND {!knn_text_to_vector model=dummy-1 f=vector topK=5 preFilter='id:(2 4 7 9 8 20 3)' v='hello world'}"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.add("fl", "id"); + + // knn subquery should still accept `preFilter` local param + // filt -> topK -> 4,2,3,7,9 + assertJQ( + "/query" + query.toQueryString(), + "/response/numFound==5]", + "/response/docs/[0]/id=='4'", + "/response/docs/[1]/id=='2'", + "/response/docs/[2]/id=='3'", + "/response/docs/[3]/id=='7'", + "/response/docs/[4]/id=='9'"); + } + + @Test + public void embeddedQueryAsSubQuery_rejectIncludeExclude() throws Exception { + for (String q : + Arrays.asList( + "{!knn_text_to_vector model=dummy-1 f=vector topK=5 includeTags=xxx}hello world", + "{!knn_text_to_vector model=dummy-1 f=vector topK=5 excludeTags=xxx}hello world")) { + final SolrQuery query = new SolrQuery(); + query.setQuery("*:* OR " + q); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/error/msg=='Knn Query Parser used as a sub-query does not support includeTags or excludeTags localparams'", + "/error/code==400"); + } + } + + @Test + public void embeddedQueryWithCostlyFq_shouldPerformKnnSearchWithPostFilter() throws Exception { + final String solrQuery = "{!knn_text_to_vector model=dummy-1 f=vector topK=10}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.setFilterQueries("{!frange cache=false l=0.99}$q"); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/response/numFound==5]", + "/response/docs/[0]/id=='1'", + "/response/docs/[1]/id=='4'", + "/response/docs/[2]/id=='2'", + "/response/docs/[3]/id=='10'", + "/response/docs/[4]/id=='3'"); + } + + @Test + public void embeddedQueryWithFilterQueries_shouldPerformKnnSearchWithPreFiltersAndPostFilters() + throws Exception { + final String solrQuery = "{!knn_text_to_vector model=dummy-1 f=vector topK=4}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.setFilterQueries("id:(3 4 9 2)", "{!frange cache=false l=0.99}$q"); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/response/numFound==2]", + "/response/docs/[0]/id=='4'", + "/response/docs/[1]/id=='2'"); + } + + @Test + public void embeddedQueryWithNegativeFilterQuery_shouldPerformKnnSearchInPreFilteredResults() + throws Exception { + final String solrQuery = "{!knn_text_to_vector model=dummy-1 f=vector topK=4}hello world"; + final SolrQuery query = new SolrQuery(); + query.setQuery(solrQuery); + query.setFilterQueries("-id:4"); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/response/numFound==4]", + "/response/docs/[0]/id=='1'", + "/response/docs/[1]/id=='2'", + "/response/docs/[2]/id=='10'", + "/response/docs/[3]/id=='3'"); + } + + /** + * See {@link org.apache.solr.search.ReRankQParserPlugin.ReRankQueryRescorer#combine(float, + * boolean, float)}} for more details. + */ + @Test + public void embeddedQueryAsRerank_shouldAddSimilarityFunctionScore() throws Exception { + final SolrQuery query = new SolrQuery(); + query.set("rq", "{!rerank reRankQuery=$rqq reRankDocs=4 reRankWeight=1}"); + query.set("rqq", "{!knn_text_to_vector model=dummy-1 f=vector topK=4}hello world"); + query.setQuery("id:(3 4 9 2)"); + query.add("fl", "id"); + + assertJQ( + "/query" + query.toQueryString(), + "/response/numFound==4]", + "/response/docs/[0]/id=='4'", + "/response/docs/[1]/id=='2'", + "/response/docs/[2]/id=='3'", + "/response/docs/[3]/id=='9'"); + } +} diff --git a/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/store/rest/TestModelManager.java b/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/store/rest/TestModelManager.java new file mode 100644 index 00000000000..37d40b3f6c4 --- /dev/null +++ b/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/store/rest/TestModelManager.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.llm.texttovector.store.rest; + +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.SolrResourceLoader; +import org.apache.solr.llm.TestLlmBase; +import org.apache.solr.llm.texttovector.search.TextToVectorQParserPlugin; +import org.apache.solr.rest.ManagedResource; +import org.apache.solr.rest.ManagedResourceStorage; +import org.apache.solr.rest.RestManager; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestModelManager extends TestLlmBase { + + @BeforeClass + public static void init() throws Exception { + setupTest("solrconfig-llm.xml", "schema.xml", false, false); + } + + @Test + public void test() throws Exception { + final SolrResourceLoader loader = new SolrResourceLoader(tmpSolrHome); + + final RestManager.Registry registry = loader.getManagedResourceRegistry(); + assertNotNull( + "Expected a non-null RestManager.Registry from the SolrResourceLoader!", registry); + + final String resourceId = "/schema/mstore1"; + registry.registerManagedResource( + resourceId, ManagedTextToVectorModelStore.class, new TextToVectorQParserPlugin()); + + final NamedList initArgs = new NamedList<>(); + + final RestManager restManager = new RestManager(); + restManager.init(loader, initArgs, new ManagedResourceStorage.InMemoryStorageIO()); + + final ManagedResource res = restManager.getManagedResource(resourceId); + assertTrue(res instanceof ManagedTextToVectorModelStore); + assertEquals(res.getResourceId(), resourceId); + } + + @Test + public void testRestManagerEndpoints() throws Exception { + assertJQ("/schema/managed", "/responseHeader/status==0"); + + final String cohereModelClassName = "dev.langchain4j.model.cohere.CohereEmbeddingModel"; + + // Add models + String model = "{ \"name\":\"testModel1\", \"class\":\"" + cohereModelClassName + "\"}"; + // fails since it does not have params + assertJPut(ManagedTextToVectorModelStore.REST_END_POINT, model, "/responseHeader/status==400"); + // success + model = + "{ name:\"testModel2\", class:\"" + + cohereModelClassName + + "\"," + + "params:{" + + "baseUrl:\"https://api.cohere.ai/v1/\"," + + "apiKey:\"cohereApiKey2\"," + + "modelName:\"embed-english-light-v3.0\"," + + "inputType:\"search_document\"," + + "logRequests:true," + + "logResponses:false" + + "}}"; + assertJPut(ManagedTextToVectorModelStore.REST_END_POINT, model, "/responseHeader/status==0"); + // success + final String multipleModels = + "[{ name:\"testModel3\", class:\"" + + cohereModelClassName + + "\"," + + "params:{baseUrl:\"https://api.cohere.ai/v1/\"," + + "apiKey:\"cohereApiKey3\"," + + "modelName:\"embed-english-light-v3.0\"," + + "inputType:\"search_document\"," + + "logRequests:true," + + "logResponses:false" + + "}}\n" + + ",{ name:\"testModel4\", class:\"" + + cohereModelClassName + + "\"," + + "params:{baseUrl:\"https://api.cohere.ai/v1/\"," + + "apiKey:\"cohereApiKey4\"," + + "modelName:\"embed-english-light-v3.0\"," + + "inputType:\"search_document\"," + + "logRequests:true," + + "logResponses:false" + + "}}]"; + assertJPut( + ManagedTextToVectorModelStore.REST_END_POINT, multipleModels, "/responseHeader/status==0"); + final String qryResult = JQ(ManagedTextToVectorModelStore.REST_END_POINT); + + assertTrue( + qryResult.contains("\"name\":\"testModel2\"") + && qryResult.contains("\"name\":\"testModel3\"") + && qryResult.contains("\"name\":\"testModel4\"")); + + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/name=='testModel2'"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[1]/name=='testModel3'"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[2]/name=='testModel4'"); + restTestHarness.delete(ManagedTextToVectorModelStore.REST_END_POINT + "/testModel2"); + restTestHarness.delete(ManagedTextToVectorModelStore.REST_END_POINT + "/testModel3"); + restTestHarness.delete(ManagedTextToVectorModelStore.REST_END_POINT + "/testModel4"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models==[]'"); + } + + @Test + public void loadModel_cohere_shouldLoadModelConfig() throws Exception { + loadModel("cohere-model.json"); + + final String modelName = "cohere-1"; + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/name=='" + modelName + "'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/baseUrl=='https://api.cohere.ai/v1/'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/apiKey=='apiKey-cohere'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/modelName=='embed-english-light-v3.0'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/inputType=='search_document'"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/timeout==60"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/logRequests==true"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/logResponses==true"); + + restTestHarness.delete(ManagedTextToVectorModelStore.REST_END_POINT + "/" + modelName); + } + + @Test + public void loadModel_openAi_shouldLoadModelConfig() throws Exception { + loadModel("openai-model.json"); + + final String modelName = "openai-1"; + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/name=='" + modelName + "'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/baseUrl=='https://api.openai.com/v1'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/apiKey=='apiKey-openAI'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/modelName=='text-embedding-3-small'"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/timeout==60"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/logRequests==true"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/logResponses==true"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/maxRetries==5"); + + restTestHarness.delete(ManagedTextToVectorModelStore.REST_END_POINT + "/" + modelName); + } + + @Test + public void loadModel_mistralAi_shouldLoadModelConfig() throws Exception { + loadModel("mistralai-model.json"); + + final String modelName = "mistralai-1"; + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/name=='" + modelName + "'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/baseUrl=='https://api.mistral.ai/v1'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/apiKey=='apiKey-mistralAI'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/modelName=='mistral-embed'"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/timeout==60"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/logRequests==true"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/logResponses==true"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/maxRetries==5"); + + restTestHarness.delete(ManagedTextToVectorModelStore.REST_END_POINT + "/" + modelName); + } + + @Test + public void loadModel_huggingface_shouldLoadModelConfig() throws Exception { + loadModel("huggingface-model.json"); + + final String modelName = "huggingface-1"; + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/name=='" + modelName + "'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/accessToken=='apiKey-huggingface'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/modelId=='sentence-transformers/all-MiniLM-L6-v2'"); + + restTestHarness.delete(ManagedTextToVectorModelStore.REST_END_POINT + "/" + modelName); + } + + @Test + public void loadModel_dummyUnsupportedParam_shouldRaiseError() throws Exception { + loadModel("dummy-model-unsupported.json", "400"); + } + + @Test + public void loadModel_dummyAmbiguousParam_shouldDefaultToString() throws Exception { + loadModel("dummy-model-ambiguous.json"); + + final String modelName = "dummy-1"; + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/name=='" + modelName + "'"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/ambiguous==10"); + + restTestHarness.delete(ManagedTextToVectorModelStore.REST_END_POINT + "/" + modelName); + } +} diff --git a/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/store/rest/TestModelManagerPersistence.java b/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/store/rest/TestModelManagerPersistence.java new file mode 100644 index 00000000000..798e2f091b6 --- /dev/null +++ b/solr/modules/llm/src/test/org/apache/solr/llm/texttovector/store/rest/TestModelManagerPersistence.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.llm.texttovector.store.rest; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import org.apache.solr.common.util.Utils; +import org.apache.solr.llm.TestLlmBase; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class TestModelManagerPersistence extends TestLlmBase { + + @Before + public void init() throws Exception { + setupTest("solrconfig-llm.xml", "schema.xml", false, true); + } + + @After + public void cleanup() throws Exception { + afterTest(); + } + + @Test + public void testModelAreStoredCompact() throws Exception { + loadModel("cohere-model.json"); + + final String JSONOnDisk = Files.readString(embeddingModelStoreFile, StandardCharsets.UTF_8); + Object objectFromDisk = Utils.fromJSONString(JSONOnDisk); + assertEquals(new String(Utils.toJSON(objectFromDisk, -1), UTF_8), JSONOnDisk); + } + + @Test + public void testModelStorePersistence() throws Exception { + // check models are empty + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/==[]"); + + // load models and features from files + loadModel("cohere-model.json"); + + final String modelName = "cohere-1"; + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/name=='" + modelName + "'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/baseUrl=='https://api.cohere.ai/v1/'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/apiKey=='apiKey-cohere'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/modelName=='embed-english-light-v3.0'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/inputType=='search_document'"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/timeout==60"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/logRequests==true"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/logResponses==true"); + + // check persistence after reload + restTestHarness.reload(); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/name=='" + modelName + "'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/baseUrl=='https://api.cohere.ai/v1/'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/apiKey=='apiKey-cohere'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/modelName=='embed-english-light-v3.0'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/inputType=='search_document'"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/timeout==60"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/logRequests==true"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/logResponses==true"); + + // check persistence after restart + getJetty().stop(); + getJetty().start(); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/name=='" + modelName + "'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/baseUrl=='https://api.cohere.ai/v1/'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/apiKey=='apiKey-cohere'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/modelName=='embed-english-light-v3.0'"); + assertJQ( + ManagedTextToVectorModelStore.REST_END_POINT, + "/models/[0]/params/inputType=='search_document'"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/timeout==60"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/logRequests==true"); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/[0]/params/logResponses==true"); + + // delete loaded models and features + restTestHarness.delete(ManagedTextToVectorModelStore.REST_END_POINT + "/" + modelName); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/==[]"); + + // check persistence after reload + restTestHarness.reload(); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/==[]"); + + // check persistence after restart + getJetty().stop(); + getJetty().start(); + assertJQ(ManagedTextToVectorModelStore.REST_END_POINT, "/models/==[]"); + } +} diff --git a/solr/modules/ltr/build.gradle b/solr/modules/ltr/build.gradle index 022cf99778b..61e02bb645d 100644 --- a/solr/modules/ltr/build.gradle +++ b/solr/modules/ltr/build.gradle @@ -32,9 +32,9 @@ dependencies { implementation project(':solr:core') implementation project(':solr:solrj') - implementation 'org.apache.lucene:lucene-core' + implementation libs.apache.lucene.core - implementation 'org.slf4j:slf4j-api' + implementation libs.slf4j.api // Used by example scripts generatedPythonClient project(path: ":solr:api", configuration: "pythonClient") @@ -42,20 +42,20 @@ dependencies { builtBy "copyPythonClientToExample" } - testImplementation('org.mockito:mockito-core', { + testImplementation(libs.mockito.core, { exclude group: "net.bytebuddy", module: "byte-buddy-agent" }) - testRuntimeOnly('org.mockito:mockito-subclass', { + testRuntimeOnly(libs.mockito.subclass, { exclude group: "net.bytebuddy", module: "byte-buddy-agent" }) testImplementation project(':solr:test-framework') - testImplementation 'org.apache.lucene:lucene-test-framework' - testImplementation 'com.carrotsearch.randomizedtesting:randomizedtesting-runner' - testImplementation 'junit:junit' - testImplementation 'org.hamcrest:hamcrest' + testImplementation libs.apache.lucene.testframework + testImplementation libs.carrotsearch.randomizedtesting.runner + testImplementation libs.junit.junit + testImplementation libs.hamcrest.hamcrest - testImplementation 'commons-io:commons-io' + testImplementation libs.commonsio.commonsio } task copyPythonClientToExample(type: Sync) { diff --git a/solr/modules/ltr/src/java/org/apache/solr/ltr/feature/Feature.java b/solr/modules/ltr/src/java/org/apache/solr/ltr/feature/Feature.java index f07f624f3f1..f3fe61c3fb5 100644 --- a/solr/modules/ltr/src/java/org/apache/solr/ltr/feature/Feature.java +++ b/solr/modules/ltr/src/java/org/apache/solr/ltr/feature/Feature.java @@ -302,7 +302,7 @@ public String toString() { } /** A 'recipe' for computing a feature */ - public abstract class FeatureScorer extends Scorer { + public abstract static class FeatureScorer extends Scorer { protected final String name; private DocInfo docInfo; @@ -342,7 +342,7 @@ public DocIdSetIterator iterator() { * A FeatureScorer that contains a Scorer, which it delegates to where * appropriate. */ - public abstract class FilterFeatureScorer extends FeatureScorer { + public abstract static class FilterFeatureScorer extends FeatureScorer { protected final Scorer in; @@ -380,7 +380,7 @@ public float getMaxScore(int upTo) throws IOException { * Default FeatureScorer class that returns the score passed in. Can be used as a simple * ValueFeature, or to return a default scorer in case an underlying feature's scorer is null. */ - public class ValueFeatureScorer extends FeatureScorer { + public static class ValueFeatureScorer extends FeatureScorer { float constScore; public ValueFeatureScorer(FeatureWeight weight, float constScore, DocIdSetIterator itr) { diff --git a/solr/modules/ltr/src/java/org/apache/solr/ltr/feature/OriginalScoreFeature.java b/solr/modules/ltr/src/java/org/apache/solr/ltr/feature/OriginalScoreFeature.java index a74e930641b..e2b4e8d93ad 100644 --- a/solr/modules/ltr/src/java/org/apache/solr/ltr/feature/OriginalScoreFeature.java +++ b/solr/modules/ltr/src/java/org/apache/solr/ltr/feature/OriginalScoreFeature.java @@ -91,7 +91,7 @@ public FeatureScorer scorer(LeafReaderContext context) throws IOException { return new OriginalScoreScorer(this, originalScorer); } - public class OriginalScoreScorer extends FilterFeatureScorer { + public static class OriginalScoreScorer extends FilterFeatureScorer { public OriginalScoreScorer(FeatureWeight weight, Scorer originalScorer) { super(weight, originalScorer); diff --git a/solr/modules/ltr/src/java/org/apache/solr/ltr/feature/SolrFeature.java b/solr/modules/ltr/src/java/org/apache/solr/ltr/feature/SolrFeature.java index 2162dfd3ecb..5c696b6f96f 100644 --- a/solr/modules/ltr/src/java/org/apache/solr/ltr/feature/SolrFeature.java +++ b/solr/modules/ltr/src/java/org/apache/solr/ltr/feature/SolrFeature.java @@ -234,7 +234,7 @@ public FeatureScorer scorer(LeafReaderContext context) throws IOException { } /** Scorer for a SolrFeature */ - public class SolrFeatureScorer extends FilterFeatureScorer { + public static class SolrFeatureScorer extends FilterFeatureScorer { public SolrFeatureScorer(FeatureWeight weight, Scorer solrScorer) { super(weight, solrScorer); diff --git a/solr/modules/ltr/src/java/org/apache/solr/ltr/model/LTRScoringModel.java b/solr/modules/ltr/src/java/org/apache/solr/ltr/model/LTRScoringModel.java index 8af64bbec5b..21cca228858 100644 --- a/solr/modules/ltr/src/java/org/apache/solr/ltr/model/LTRScoringModel.java +++ b/solr/modules/ltr/src/java/org/apache/solr/ltr/model/LTRScoringModel.java @@ -215,8 +215,7 @@ private final int calculateHashCode() { @Override public boolean equals(Object obj) { if (this == obj) return true; - if (!(obj instanceof LTRScoringModel)) return false; - final LTRScoringModel other = (LTRScoringModel) obj; + if (!(obj instanceof LTRScoringModel other)) return false; return Objects.equals(features, other.features) && Objects.equals(norms, other.norms) && Objects.equals(name, other.name) diff --git a/solr/modules/ltr/src/java/org/apache/solr/ltr/model/WrapperModel.java b/solr/modules/ltr/src/java/org/apache/solr/ltr/model/WrapperModel.java index 86eb8c45564..95aef5006f2 100644 --- a/solr/modules/ltr/src/java/org/apache/solr/ltr/model/WrapperModel.java +++ b/solr/modules/ltr/src/java/org/apache/solr/ltr/model/WrapperModel.java @@ -69,8 +69,7 @@ public int hashCode() { public boolean equals(Object obj) { if (this == obj) return true; if (!super.equals(obj)) return false; - if (!(obj instanceof WrapperModel)) return false; - WrapperModel other = (WrapperModel) obj; + if (!(obj instanceof WrapperModel other)) return false; return Objects.equals(model, other.model) && Objects.equals(solrResourceLoader, other.solrResourceLoader); } diff --git a/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr.xml b/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr.xml index b863d61728c..c20ee2026f6 100644 --- a/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr.xml +++ b/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr.xml @@ -16,9 +16,6 @@ - - - diff --git a/solr/modules/opentelemetry/build.gradle b/solr/modules/opentelemetry/build.gradle index 4426f67649e..ee9b3564486 100644 --- a/solr/modules/opentelemetry/build.gradle +++ b/solr/modules/opentelemetry/build.gradle @@ -23,27 +23,29 @@ dependencies { implementation project(':solr:core') implementation project(':solr:solrj') - implementation platform('io.opentelemetry:opentelemetry-bom') - implementation 'org.slf4j:slf4j-api' + implementation platform(libs.netty.bom) + implementation platform(libs.opentelemetry.bom) - implementation 'io.opentelemetry:opentelemetry-api' - implementation 'io.opentelemetry:opentelemetry-sdk-extension-autoconfigure' - runtimeOnly 'io.opentelemetry:opentelemetry-exporter-otlp' + implementation libs.slf4j.api + + implementation libs.opentelemetry.api + implementation libs.opentelemetry.sdkextension.autoconfigure + runtimeOnly libs.opentelemetry.exporter.otlp // End users must recompile with jaeger exporter and/or zipkin exporter if they need these // NOTE: sdk-autoconfigure needs both opentelemetry-sdk-metrics and opentelemetry-sdk-logs even if we don't use them // gRPC transport via netty - since we already ship netty this is more lightweight than netty-shaded - runtimeOnly 'io.grpc:grpc-netty' - runtimeOnly 'io.grpc:grpc-protobuf' - runtimeOnly 'io.grpc:grpc-stub' - runtimeOnly 'io.grpc:grpc-context' + runtimeOnly libs.grpc.netty + runtimeOnly libs.grpc.protobuf + runtimeOnly libs.grpc.stub + runtimeOnly libs.grpc.context // See https://issues.apache.org/jira/browse/LOG4J2-3609 due to needing these annotations - compileOnly 'org.apache.tomcat:annotations-api' + compileOnly libs.apache.tomcat.annotationsapi testImplementation project(':solr:test-framework') - testImplementation 'junit:junit' - testImplementation 'io.opentelemetry:opentelemetry-sdk' - testImplementation 'io.opentelemetry:opentelemetry-sdk-trace' - testImplementation 'io.opentelemetry:opentelemetry-sdk-testing' + testImplementation libs.junit.junit + testImplementation libs.opentelemetry.sdk + testImplementation libs.opentelemetry.sdktrace + testImplementation libs.opentelemetry.sdktesting } diff --git a/solr/modules/s3-repository/build.gradle b/solr/modules/s3-repository/build.gradle index 431df0820a1..15227d57f47 100644 --- a/solr/modules/s3-repository/build.gradle +++ b/solr/modules/s3-repository/build.gradle @@ -23,37 +23,37 @@ dependencies { api project(':solr:core') implementation project(':solr:solrj') - implementation 'org.apache.lucene:lucene-core' - - implementation platform(group: 'software.amazon.awssdk', name: 'bom') - implementation(group: 'software.amazon.awssdk', name: 'auth') - implementation(group: 'software.amazon.awssdk', name: 'apache-client') - implementation(group: 'software.amazon.awssdk', name: 'aws-core') - implementation(group: 'software.amazon.awssdk', name: 'regions') - implementation(group: 'software.amazon.awssdk', name: 'http-client-spi') - implementation(group: 'software.amazon.awssdk', name: 's3') { + implementation libs.apache.lucene.core + + implementation platform(libs.amazon.awssdk.bom) + implementation libs.amazon.awssdk.auth + implementation libs.amazon.awssdk.apacheclient + implementation libs.amazon.awssdk.awscore + implementation libs.amazon.awssdk.regions + implementation libs.amazon.awssdk.httpclient.spi + implementation(libs.amazon.awssdk.s3) { exclude group: 'software.amazon.awssdk', module: 'netty-nio-client' } - implementation(group: 'software.amazon.awssdk', name: 'sdk-core') - runtimeOnly(group: 'software.amazon.awssdk', name: 'sts') { + implementation libs.amazon.awssdk.sdkcore + runtimeOnly(libs.amazon.awssdk.sts) { exclude group: 'software.amazon.awssdk', module: 'netty-nio-client' } - implementation 'com.google.guava:guava' - implementation 'org.slf4j:slf4j-api' + implementation libs.google.guava + implementation libs.slf4j.api - runtimeOnly(group: 'com.fasterxml.woodstox', name: 'woodstox-core') - runtimeOnly(group: 'org.codehaus.woodstox', name: 'stax2-api') + runtimeOnly libs.fasterxml.woodstox.core + runtimeOnly libs.codehaus.woodstox.stax2api testImplementation project(':solr:test-framework') - testImplementation 'org.apache.lucene:lucene-test-framework' - testImplementation 'com.carrotsearch.randomizedtesting:randomizedtesting-runner' - testImplementation 'junit:junit' - testImplementation 'software.amazon.awssdk:profiles' + testImplementation libs.apache.lucene.testframework + testImplementation libs.carrotsearch.randomizedtesting.runner + testImplementation libs.junit.junit + testImplementation libs.amazon.awssdk.profiles - testImplementation 'org.hamcrest:hamcrest' + testImplementation libs.hamcrest.hamcrest - testImplementation('com.adobe.testing:s3mock-junit4') { + testImplementation(libs.adobe.testing.s3mock.junit4) { // Don't pull in separate versions of these libs, just use what Solr already has exclude group: 'org.apache.logging.log4j', module: 'log4j-to-slf4j' exclude group: 'ch.qos.logback', module: 'logback-classic' @@ -65,7 +65,7 @@ dependencies { exclude group: 'software.amazon.awssdk', module: 'netty-nio-client' exclude group: 'org.codehaus.woodstox', module: 'stax2-api' } - testImplementation('com.adobe.testing:s3mock-testsupport-common') { + testImplementation(libs.adobe.testing.s3mock.testsupportcommon) { // Don't pull in separate versions of these libs, just use what Solr already has exclude group: 'org.apache.logging.log4j', module: 'log4j-to-slf4j' exclude group: 'ch.qos.logback', module: 'logback-classic' @@ -78,14 +78,14 @@ dependencies { exclude group: 'org.codehaus.woodstox', module: 'stax2-api' } - testImplementation 'commons-io:commons-io' + testImplementation libs.commonsio.commonsio - testRuntimeOnly 'org.eclipse.jetty:jetty-webapp' + testRuntimeOnly libs.eclipse.jetty.webapp - testImplementation('org.mockito:mockito-core', { + testImplementation(libs.mockito.core, { exclude group: "net.bytebuddy", module: "byte-buddy-agent" }) - testRuntimeOnly('org.mockito:mockito-subclass', { + testRuntimeOnly(libs.mockito.subclass, { exclude group: "net.bytebuddy", module: "byte-buddy-agent" }) } diff --git a/solr/modules/s3-repository/src/java/org/apache/solr/s3/S3StorageClient.java b/solr/modules/s3-repository/src/java/org/apache/solr/s3/S3StorageClient.java index 3d600899647..527a951e71e 100644 --- a/solr/modules/s3-repository/src/java/org/apache/solr/s3/S3StorageClient.java +++ b/solr/modules/s3-repository/src/java/org/apache/solr/s3/S3StorageClient.java @@ -582,8 +582,7 @@ String sanitizedDirPath(String path) throws S3Exception { */ static S3Exception handleAmazonException(SdkException sdke) { - if (sdke instanceof AwsServiceException) { - AwsServiceException ase = (AwsServiceException) sdke; + if (sdke instanceof AwsServiceException ase) { String errMessage = String.format( Locale.ROOT, diff --git a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java index d10322253cd..7825c33afc1 100644 --- a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java +++ b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java @@ -95,7 +95,7 @@ public static void setupClass() throws Exception { AbstractS3ClientTest.setS3ConfFile(); configureCluster(NUM_NODES) // nodes - .addConfig("conf1", getFile("conf/solrconfig.xml").getParentFile().toPath()) + .addConfig("conf1", getFile("conf/solrconfig.xml").getParent()) .withSolrXml( SOLR_XML .replace("BUCKET", BUCKET_NAME) diff --git a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3InstallShardTest.java b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3InstallShardTest.java index ea21cb58877..19c88caa089 100644 --- a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3InstallShardTest.java +++ b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3InstallShardTest.java @@ -69,7 +69,7 @@ public static void setupClass() throws Exception { AbstractS3ClientTest.setS3ConfFile(); configureCluster(1) // nodes - .addConfig("conf1", getFile("conf/solrconfig.xml").getParentFile().toPath()) + .addConfig("conf1", getFile("conf/solrconfig.xml").getParent()) .withSolrXml( SOLR_XML .replace("BUCKET", BUCKET_NAME) diff --git a/solr/modules/scripting/build.gradle b/solr/modules/scripting/build.gradle index a177f26132a..b035e016e4d 100644 --- a/solr/modules/scripting/build.gradle +++ b/solr/modules/scripting/build.gradle @@ -23,13 +23,13 @@ dependencies { implementation project(':solr:core') implementation project(':solr:solrj') - implementation 'org.apache.lucene:lucene-core' + implementation libs.apache.lucene.core - implementation 'org.slf4j:slf4j-api' - implementation 'commons-io:commons-io' - implementation 'com.google.guava:guava' + implementation libs.slf4j.api + implementation libs.commonsio.commonsio + implementation libs.google.guava testImplementation project(':solr:test-framework') - testImplementation 'org.apache.lucene:lucene-test-framework' - testImplementation 'junit:junit' + testImplementation libs.apache.lucene.testframework + testImplementation libs.junit.junit } diff --git a/solr/modules/scripting/src/test/org/apache/solr/scripting/update/ScriptUpdateProcessorFactoryTest.java b/solr/modules/scripting/src/test/org/apache/solr/scripting/update/ScriptUpdateProcessorFactoryTest.java index 3e8bfa9a055..0433440cb25 100644 --- a/solr/modules/scripting/src/test/org/apache/solr/scripting/update/ScriptUpdateProcessorFactoryTest.java +++ b/solr/modules/scripting/src/test/org/apache/solr/scripting/update/ScriptUpdateProcessorFactoryTest.java @@ -44,7 +44,7 @@ public static void beforeClass() throws Exception { initCore( "solrconfig-script-updateprocessor.xml", "schema.xml", - getFile("scripting/solr").getAbsolutePath()); + getFile("scripting/solr").toAbsolutePath().toString()); } /** diff --git a/solr/modules/scripting/src/test/org/apache/solr/scripting/update/TestBadScriptingUpdateProcessorConfig.java b/solr/modules/scripting/src/test/org/apache/solr/scripting/update/TestBadScriptingUpdateProcessorConfig.java index 750ae4094bb..c20ef78b620 100644 --- a/solr/modules/scripting/src/test/org/apache/solr/scripting/update/TestBadScriptingUpdateProcessorConfig.java +++ b/solr/modules/scripting/src/test/org/apache/solr/scripting/update/TestBadScriptingUpdateProcessorConfig.java @@ -32,7 +32,7 @@ public void testBogusScriptEngine() throws Exception { assertConfigs( "bad-solrconfig-bogus-scriptengine-name.xml", "schema.xml", - getFile("scripting/solr/collection1").getParent(), + getFile("scripting/solr/collection1").getParent().toString(), "giberish"); } @@ -42,7 +42,7 @@ public void testMissingScriptFile() throws Exception { assertConfigs( "bad-solrconfig-missing-scriptfile.xml", "schema.xml", - getFile("scripting/solr/collection1").getParent(), + getFile("scripting/solr/collection1").getParent().toString(), "a-file-name-that-does-not-exist.js"); } @@ -52,7 +52,7 @@ public void testInvalidScriptFile() throws Exception { assertConfigs( "bad-solrconfig-invalid-scriptfile.xml", "schema.xml", - getFile("scripting/solr/collection1").getParent(), + getFile("scripting/solr/collection1").getParent().toString(), "invalid.script.xml"); } diff --git a/solr/modules/scripting/src/test/org/apache/solr/scripting/xslt/XSLTOutputWriterTest.java b/solr/modules/scripting/src/test/org/apache/solr/scripting/xslt/XSLTOutputWriterTest.java index 06122b2f212..f0dd2d85d9d 100644 --- a/solr/modules/scripting/src/test/org/apache/solr/scripting/xslt/XSLTOutputWriterTest.java +++ b/solr/modules/scripting/src/test/org/apache/solr/scripting/xslt/XSLTOutputWriterTest.java @@ -30,7 +30,7 @@ public class XSLTOutputWriterTest extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() throws Exception { - initCore("solrconfig.xml", "schema.xml", getFile("scripting/solr").getAbsolutePath()); + initCore("solrconfig.xml", "schema.xml", getFile("scripting/solr").toAbsolutePath().toString()); } @Test diff --git a/solr/modules/scripting/src/test/org/apache/solr/scripting/xslt/XSLTUpdateRequestHandlerTest.java b/solr/modules/scripting/src/test/org/apache/solr/scripting/xslt/XSLTUpdateRequestHandlerTest.java index c6a2a876b4c..af61f99e860 100644 --- a/solr/modules/scripting/src/test/org/apache/solr/scripting/xslt/XSLTUpdateRequestHandlerTest.java +++ b/solr/modules/scripting/src/test/org/apache/solr/scripting/xslt/XSLTUpdateRequestHandlerTest.java @@ -42,7 +42,7 @@ public class XSLTUpdateRequestHandlerTest extends SolrTestCaseJ4 { @BeforeClass public static void beforeTests() throws Exception { - initCore("solrconfig.xml", "schema.xml", getFile("scripting/solr").getAbsolutePath()); + initCore("solrconfig.xml", "schema.xml", getFile("scripting/solr").toAbsolutePath().toString()); } @Override @@ -98,7 +98,7 @@ public void testUpdate() throws Exception { @Test public void testEntities() throws Exception { // use a binary file, so when it's loaded fail with XML error: - String file = getFile("mailing_lists.pdf").toURI().toASCIIString(); + String file = getFile("mailing_lists.pdf").toUri().toASCIIString(); String xml = "" + " operands = call.getOperands(); if (operands.size() != 1) { throw new AssertionError("expected 1 operand for " + node); @@ -531,10 +530,9 @@ private String toSolrTimestamp(final String ts) { protected Pair, Character> getFieldValuePairWithEscapeCharacter( RexNode node) { - if (!(node instanceof RexCall)) { + if (!(node instanceof RexCall call)) { throw new AssertionError("expected RexCall for predicate but found: " + node); } - RexCall call = (RexCall) node; if (call.getOperands().size() == 3) { RexNode escapeNode = call.getOperands().get(2); Character escapeChar = null; @@ -552,11 +550,10 @@ protected Pair, Character> getFieldValuePairWithEscapeC } protected Pair getFieldValuePair(RexNode node) { - if (!(node instanceof RexCall)) { + if (!(node instanceof RexCall call)) { throw new AssertionError("expected RexCall for predicate but found: " + node); } - RexCall call = (RexCall) node; Pair binaryTranslated = call.getOperands().size() == 2 ? translateBinary(call) : null; if (binaryTranslated == null) { @@ -654,15 +651,13 @@ protected String translateSearch(RexNode condition) { !expanded.getOperands().isEmpty() ? expanded.getOperands().get(0) : null; if (expanded.op.kind == SqlKind.AND) { // See if NOT IN was translated into a big AND not - if (peekAt0 instanceof RexCall) { - RexCall op0 = (RexCall) peekAt0; + if (peekAt0 instanceof RexCall op0) { if (op0.op.kind == SqlKind.NOT_EQUALS) { return "*:* -" + fieldName + ":" + toOrSetOnSameField(fieldName, expanded); } } } else if (expanded.op.kind == SqlKind.OR) { - if (peekAt0 instanceof RexCall) { - RexCall op0 = (RexCall) peekAt0; + if (peekAt0 instanceof RexCall op0) { if (op0.op.kind == SqlKind.EQUALS) { return fieldName + ":" + toOrSetOnSameField(fieldName, expanded); } diff --git a/solr/packaging/build.gradle b/solr/packaging/build.gradle index f0e17595dcc..913aab3a6c0 100644 --- a/solr/packaging/build.gradle +++ b/solr/packaging/build.gradle @@ -229,9 +229,9 @@ artifacts { task downloadBats(type: NpmTask) { group = 'Build Dependency Download' - args = ["install", "https://github.com/bats-core/bats-core#v1.8.2", - "https://github.com/bats-core/bats-assert#v2.0.0", - "https://github.com/bats-core/bats-file#v0.3.0", + args = ["install", "https://github.com/bats-core/bats-core#v${libs.versions.bats.core.get()}", + "https://github.com/bats-core/bats-assert#v${libs.versions.bats.assert.get()}", + "https://github.com/bats-core/bats-file#v${libs.versions.bats.file.get()}", ] inputs.files("${project.ext.nodeProjectDir}/package.json") diff --git a/solr/packaging/test/bats_helper.bash b/solr/packaging/test/bats_helper.bash index 68c29686a99..2d8fbb18375 100644 --- a/solr/packaging/test/bats_helper.bash +++ b/solr/packaging/test/bats_helper.bash @@ -71,7 +71,7 @@ delete_all_collections() { local collection_list="$(solr zk ls /collections -z localhost:${ZK_PORT})" for collection in $collection_list; do if [[ -n $collection ]]; then - solr delete -c $collection >/dev/null 2>&1 + solr delete -c $collection --delete-config >/dev/null 2>&1 fi done } diff --git a/solr/packaging/test/test_basic_auth.bats b/solr/packaging/test/test_basic_auth.bats index 333d5cf73d3..a84d9d15d4a 100644 --- a/solr/packaging/test/test_basic_auth.bats +++ b/solr/packaging/test/test_basic_auth.bats @@ -57,9 +57,10 @@ teardown() { assert_output --partial '"numFound":0' # Test delete - run solr delete --credentials name:password -c COLL_NAME -z localhost:${ZK_PORT} --verbose + run solr delete --credentials name:password -c COLL_NAME -z localhost:${ZK_PORT} --delete-config --verbose assert_output --partial "Deleted collection 'COLL_NAME'" refute collection_exists "COLL_NAME" + refute config_exists "COLL_NAME" } diff --git a/solr/packaging/test/test_delete_collection.bats b/solr/packaging/test/test_delete_collection.bats index d5db4263e70..0ff455d1b27 100644 --- a/solr/packaging/test/test_delete_collection.bats +++ b/solr/packaging/test/test_delete_collection.bats @@ -44,6 +44,7 @@ teardown() { solr delete -c "COLL_NAME" refute collection_exists "COLL_NAME" + assert config_exists "COLL_NAME" } @test "can delete collections with solr-url" { @@ -52,13 +53,14 @@ teardown() { solr delete -c "COLL_NAME" --solr-url http://localhost:${SOLR_PORT} refute collection_exists "COLL_NAME" + assert config_exists "COLL_NAME" } @test "collection delete also deletes zk config" { solr create -c "COLL_NAME" assert config_exists "COLL_NAME" - solr delete -c "COLL_NAME" + solr delete -c "COLL_NAME" --delete-config refute config_exists "COLL_NAME" } @@ -66,7 +68,7 @@ teardown() { solr create -c "COLL_NAME" -n "NONDEFAULT_CONFIG_NAME" assert config_exists "NONDEFAULT_CONFIG_NAME" - solr delete -c "COLL_NAME" + solr delete -c "COLL_NAME" --delete-config refute config_exists "NONDEFAULT_CONFIG_NAME" } @@ -74,6 +76,6 @@ teardown() { solr create -c "COLL_NAME" assert config_exists "COLL_NAME" - solr delete -c "COLL_NAME" --delete-config false + solr delete -c "COLL_NAME" assert config_exists "COLL_NAME" } diff --git a/solr/packaging/test/test_modules.bats b/solr/packaging/test/test_modules.bats index 772e04344c4..125e9ffe5b1 100644 --- a/solr/packaging/test/test_modules.bats +++ b/solr/packaging/test/test_modules.bats @@ -39,27 +39,6 @@ teardown() { refute_output --partial '"EXCEPTION"' } -@test "Hadoop-Auth Module: KerberosPlugin Classloading" { - # Write a security.json that uses the KerberosPlugin - local security_json="${BATS_TEST_TMPDIR}/kerberos-security.json" - echo '{"authentication": {"class": "solr.KerberosPlugin"}}' > "${security_json}" - - # Start Solr - export SOLR_MODULES=hadoop-auth - solr start \ - -Dsolr.kerberos.principal=test \ - -Dsolr.kerberos.keytab=test \ - -Dsolr.kerberos.cookie.domain=test - - # Upload the custom security.json and wait for Solr to try to load it - solr zk cp "${security_json}" zk:security.json -z localhost:${ZK_PORT} - sleep 1 - - run cat "${SOLR_LOGS_DIR}/solr.log" - assert_output --partial "Initializing authentication plugin: solr.KerberosPlugin" - refute_output --partial "java.lang.ClassNotFoundException" -} - @test "icu collation in analysis-extras module" { local solr_include_file="${BATS_TEST_TMPDIR}/solr.include" echo "SOLR_MODULES=analysis-extras" > "${solr_include_file}" diff --git a/solr/packaging/test/test_stream.bats b/solr/packaging/test/test_stream.bats new file mode 100644 index 00000000000..2043d9c0c32 --- /dev/null +++ b/solr/packaging/test/test_stream.bats @@ -0,0 +1,100 @@ +#!/usr/bin/env bats + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load bats_helper + +setup_file() { + common_clean_setup + solr start -e techproducts + solr auth enable --type basicAuth --credentials name:password --solr-include-file /force/credentials/to/be/supplied +} + +teardown_file() { + common_setup + solr stop --all +} + +setup() { + common_setup +} + +teardown() { + # save a snapshot of SOLR_HOME for failed tests + save_home_on_failure +} + +@test "searching solr via locally executed streaming expression" { + + local solr_stream_file="${BATS_TEST_TMPDIR}/search.expr" + echo 'search(techproducts,' > "${solr_stream_file}" + echo 'q="name:memory",' >> "${solr_stream_file}" + echo 'fl="name,price",' >> "${solr_stream_file}" + echo 'sort="price desc"' >> "${solr_stream_file}" + echo ')' >> "${solr_stream_file}" + + run solr stream --execution local --header --credentials name:password ${solr_stream_file} + + assert_output --partial 'name price' + assert_output --partial 'CORSAIR XMS' + refute_output --partial 'ERROR' +} + +@test "searching solr via remotely executed streaming expression" { + + local solr_stream_file="${BATS_TEST_TMPDIR}/search.expr" + echo 'search(techproducts,' > "${solr_stream_file}" + echo 'q="name:memory",' >> "${solr_stream_file}" + echo 'fl="name,price",' >> "${solr_stream_file}" + echo 'sort="price desc"' >> "${solr_stream_file}" + echo ')' >> "${solr_stream_file}" + + run solr stream --name techproducts --solr-url http://localhost:${SOLR_PORT} --header --credentials name:password ${solr_stream_file} + + assert_output --partial 'name price' + assert_output --partial 'CORSAIR XMS' + refute_output --partial 'ERROR' +} + +@test "variable interpolation" { + + local solr_stream_file="${BATS_TEST_TMPDIR}/search.expr" + echo 'search(techproducts,' > "${solr_stream_file}" + echo 'q="name:$1",' >> "${solr_stream_file}" + echo 'fl="name,price",' >> "${solr_stream_file}" + echo 'sort="price $2"' >> "${solr_stream_file}" + echo ')' >> "${solr_stream_file}" + + run solr stream --execution local --header --credentials name:password ${solr_stream_file} apple asc + + assert_output --partial 'name price' + assert_output --partial 'Apple 60 GB iPod' + refute_output --partial 'ERROR' +} + +@test "searching solr without credentials fails with error" { + + local solr_stream_file="${BATS_TEST_TMPDIR}/search.expr" + echo 'search(techproducts,' > "${solr_stream_file}" + echo 'q="name:memory",' >> "${solr_stream_file}" + echo 'fl="name,price",' >> "${solr_stream_file}" + echo 'sort="price desc"' >> "${solr_stream_file}" + echo ')' >> "${solr_stream_file}" + + run ! solr stream --execution local --header ${solr_stream_file} + + assert_output --partial 'ERROR' +} diff --git a/solr/prometheus-exporter/build.gradle b/solr/prometheus-exporter/build.gradle index 1f257d5a5ee..b30a5c0a306 100644 --- a/solr/prometheus-exporter/build.gradle +++ b/solr/prometheus-exporter/build.gradle @@ -24,43 +24,43 @@ dependencies { implementation project(':solr:solrj') runtimeOnly project(':solr:solrj-zookeeper') // ideally remove ZK dep - implementation('org.apache.zookeeper:zookeeper', { + implementation(libs.apache.zookeeper.zookeeper, { exclude group: "org.apache.yetus", module: "audience-annotations" }) - implementation ('io.prometheus:simpleclient', { + implementation(libs.prometheus.simpleclient, { exclude group: "io.prometheus", module: "simpleclient_tracer_common" exclude group: "io.prometheus", module: "simpleclient_tracer_otel" exclude group: "io.prometheus", module: "simpleclient_tracer_otel_agent" }) - implementation ('io.prometheus:simpleclient_httpserver', { + implementation(libs.prometheus.simpleclient.httpserver, { exclude group: "io.prometheus", module: "simpleclient_tracer_common" exclude group: "io.prometheus", module: "simpleclient_tracer_otel" exclude group: "io.prometheus", module: "simpleclient_tracer_otel_agent" }) - implementation ('net.thisptr:jackson-jq', { + implementation(libs.thisptr.jacksonjq, { exclude group: "org.jruby.joni", module: "joni" exclude group: "com.fasterxml.jackson.core", module: "jackson-databind" }) - implementation ('com.fasterxml.jackson.core:jackson-databind') - implementation 'commons-cli:commons-cli' - implementation ('com.github.ben-manes.caffeine:caffeine') { transitive = false } - implementation 'org.slf4j:slf4j-api' - implementation 'commons-codec:commons-codec' + implementation libs.fasterxml.jackson.core.databind + implementation libs.commonscli.commonscli + implementation(libs.benmanes.caffeine) { transitive = false } + implementation libs.slf4j.api + implementation libs.commonscodec.commonscodec - runtimeOnly 'org.apache.logging.log4j:log4j-api' - runtimeOnly 'org.apache.logging.log4j:log4j-core' - runtimeOnly 'org.apache.logging.log4j:log4j-slf4j2-impl' - runtimeOnly 'com.lmax:disruptor' + runtimeOnly libs.apache.log4j.api + runtimeOnly libs.apache.log4j.core + runtimeOnly libs.apache.log4j.slf4j2impl + runtimeOnly libs.lmax.disruptor testImplementation project(':solr:test-framework') testImplementation project(':solr:solrj-zookeeper') - testImplementation 'com.carrotsearch.randomizedtesting:randomizedtesting-runner' - testImplementation 'junit:junit' - testImplementation 'org.apache.lucene:lucene-test-framework' + testImplementation libs.carrotsearch.randomizedtesting.runner + testImplementation libs.junit.junit + testImplementation libs.apache.lucene.testframework - testImplementation 'org.apache.httpcomponents:httpclient' - testImplementation 'org.apache.httpcomponents:httpcore' + testImplementation libs.apache.httpcomponents.httpclient + testImplementation libs.apache.httpcomponents.httpcore } ext { diff --git a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/MetricsQueryTemplate.java b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/MetricsQueryTemplate.java index 48d02ca3a3a..7d0e28ce001 100644 --- a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/MetricsQueryTemplate.java +++ b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/MetricsQueryTemplate.java @@ -116,8 +116,7 @@ public String applyTemplate(final Matcher matched) { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof MetricsQueryTemplate)) return false; - MetricsQueryTemplate that = (MetricsQueryTemplate) o; + if (!(o instanceof MetricsQueryTemplate that)) return false; return name.equals(that.name) && Objects.equals(defaultType, that.defaultType) && template.equals(that.template); diff --git a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java index e333076dbe2..79c3e46ae2c 100644 --- a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java +++ b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java @@ -57,10 +57,10 @@ public SolrCloudScraper( public Map pingAllCores(MetricsQuery query) throws IOException { Map httpSolrClients = createHttpSolrClients(); - Map collectionState = solrClient.getClusterState().getCollectionsMap(); - List replicas = - collectionState.values().stream() + solrClient + .getClusterState() + .collectionStream() .map(DocCollection::getReplicas) .flatMap(List::stream) .collect(Collectors.toList()); @@ -131,7 +131,9 @@ public MetricSamples collections(MetricsQuery metricsQuery) throws IOException { } private Set getBaseUrls() throws IOException { - return solrClient.getClusterState().getCollectionsMap().values().stream() + return solrClient + .getClusterState() + .collectionStream() .map(DocCollection::getReplicas) .flatMap(List::stream) .map(Replica::getBaseUrl) diff --git a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/PrometheusExporterTestBase.java b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/PrometheusExporterTestBase.java index 0b0e826c31d..93ab8b347b8 100644 --- a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/PrometheusExporterTestBase.java +++ b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/PrometheusExporterTestBase.java @@ -28,7 +28,8 @@ public class PrometheusExporterTestBase extends SolrCloudTestCase { public static final String COLLECTION = "collection1"; public static final String CONF_NAME = COLLECTION + "_config"; - public static final String CONF_DIR = getFile("solr/" + COLLECTION + "/conf").getAbsolutePath(); + public static final String CONF_DIR = + getFile("solr/" + COLLECTION + "/conf").toAbsolutePath().toString(); public static final int NUM_SHARDS = 2; public static final int NUM_REPLICAS = 2; public static final int NUM_NODES = NUM_SHARDS * NUM_REPLICAS; @@ -60,7 +61,7 @@ public void tearDown() throws Exception { @BeforeClass public static void setupCluster() throws Exception { System.setProperty("metricsEnabled", "true"); - configureCluster(NUM_NODES).addConfig(CONF_NAME, getFile(CONF_DIR).toPath()).configure(); + configureCluster(NUM_NODES).addConfig(CONF_NAME, getFile(CONF_DIR)).configure(); CollectionAdminRequest.createCollection(COLLECTION, CONF_NAME, NUM_SHARDS, NUM_REPLICAS) .process(cluster.getSolrClient()); diff --git a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/exporter/MetricsQueryTemplateTest.java b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/exporter/MetricsQueryTemplateTest.java index 9a23a1f19d8..115ae677eb2 100644 --- a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/exporter/MetricsQueryTemplateTest.java +++ b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/exporter/MetricsQueryTemplateTest.java @@ -83,7 +83,7 @@ public void testQueryMetricTemplate() throws Exception { Document config = DocumentBuilderFactory.newInstance() .newDocumentBuilder() - .parse(SolrTestCaseJ4.getFile("conf/test-config-with-templates.xml")); + .parse(SolrTestCaseJ4.getFile("conf/test-config-with-templates.xml").toFile()); NodeList jqTemplates = (NodeList) (xpathFactory.newXPath()) @@ -95,7 +95,8 @@ public void testQueryMetricTemplate() throws Exception { assertNotNull(coreQueryTemplate); ObjectMapper objectMapper = new ObjectMapper(); - JsonNode parsedMetrics = objectMapper.readTree(SolrTestCaseJ4.getFile("query-metrics.json")); + JsonNode parsedMetrics = + objectMapper.readTree(SolrTestCaseJ4.getFile("query-metrics.json").toFile()); final String[] queryMetrics = new String[] { "$jq:core-query(1minRate, select(.key | endswith(\".distrib.requestTimes\")), 1minRate)", diff --git a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrCloudScraperTest.java b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrCloudScraperTest.java index 2ebc3752cae..386b1252e04 100644 --- a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrCloudScraperTest.java +++ b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrCloudScraperTest.java @@ -115,17 +115,15 @@ public void pingCores() throws Exception { Map allCoreMetrics = solrCloudScraper.pingAllCores(configuration.getPingConfiguration().get(0)); - Map collectionStates = getClusterState().getCollectionsMap(); + final List collectionStates = getClusterState().collectionStream().toList(); long coreCount = - collectionStates.entrySet().stream() - .mapToInt(entry -> entry.getValue().getReplicas().size()) - .sum(); + collectionStates.stream().mapToInt(docColl -> docColl.getReplicas().size()).sum(); assertEquals(coreCount, allCoreMetrics.size()); - for (Map.Entry entry : collectionStates.entrySet()) { - String coreName = entry.getValue().getReplicas().get(0).getCoreName(); + for (DocCollection docColl : collectionStates) { + String coreName = docColl.getReplicas().get(0).getCoreName(); assertTrue(allCoreMetrics.containsKey(coreName)); List coreMetrics = allCoreMetrics.get(coreName).asList(); assertEquals(1, coreMetrics.size()); diff --git a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/utils/Helpers.java b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/utils/Helpers.java index b0acf4ef7a8..6c78149f817 100644 --- a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/utils/Helpers.java +++ b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/utils/Helpers.java @@ -31,16 +31,17 @@ public class Helpers { public static MetricsConfiguration loadConfiguration(String pathRsrc) throws Exception { - return MetricsConfiguration.from(SolrTestCaseJ4.getFile(pathRsrc).getPath()); + return MetricsConfiguration.from(SolrTestCaseJ4.getFile(pathRsrc).toString()); } public static void indexAllDocs(SolrClient client) throws IOException, SolrServerException { - File exampleDocsDir = new File(SolrTestCaseJ4.getFile("exampledocs").getAbsolutePath()); + File exampleDocsDir = + new File(SolrTestCaseJ4.getFile("exampledocs").toAbsolutePath().toString()); File[] xmlFiles = Objects.requireNonNull(exampleDocsDir.listFiles((dir, name) -> name.endsWith(".xml"))); for (File xml : xmlFiles) { ContentStreamUpdateRequest req = new ContentStreamUpdateRequest("/update"); - req.addFile(xml, "application/xml"); + req.addFile(xml.toPath(), "application/xml"); client.request(req, PrometheusExporterTestBase.COLLECTION); } client.commit(PrometheusExporterTestBase.COLLECTION); diff --git a/solr/server/build.gradle b/solr/server/build.gradle index 93b1196b95b..a7055de6539 100644 --- a/solr/server/build.gradle +++ b/solr/server/build.gradle @@ -37,59 +37,59 @@ configurations { } dependencies { - serverLib('org.eclipse.jetty:jetty-deploy', { + serverLib(libs.eclipse.jetty.deploy, { exclude group: "org.awaitility", module: "awaitility" }) - serverLib 'org.eclipse.jetty:jetty-http' - serverLib 'org.eclipse.jetty:jetty-io' - serverLib 'org.eclipse.jetty:jetty-jmx' - serverLib 'org.eclipse.jetty:jetty-rewrite' - serverLib 'org.eclipse.jetty:jetty-security' - serverLib 'org.eclipse.jetty:jetty-server' - serverLib 'org.eclipse.jetty:jetty-servlet' - serverLib 'org.eclipse.jetty:jetty-servlets' - serverLib 'org.eclipse.jetty:jetty-util' - serverLib 'org.eclipse.jetty:jetty-webapp' - serverLib 'org.eclipse.jetty:jetty-xml' - serverLib 'org.eclipse.jetty:jetty-alpn-server' - serverLib('org.eclipse.jetty:jetty-alpn-java-server', { + serverLib libs.eclipse.jetty.http + serverLib libs.eclipse.jetty.io + serverLib libs.eclipse.jetty.jmx + serverLib libs.eclipse.jetty.rewrite + serverLib libs.eclipse.jetty.security + serverLib libs.eclipse.jetty.server + serverLib libs.eclipse.jetty.servlet + serverLib libs.eclipse.jetty.servlets + serverLib libs.eclipse.jetty.util + serverLib libs.eclipse.jetty.webapp + serverLib libs.eclipse.jetty.xml + serverLib libs.eclipse.jetty.alpnserver + serverLib(libs.eclipse.jetty.alpnjavaserver, { exclude group: "org.eclipse.jetty.alpn", module: "alpn-api" }) - serverLib 'org.eclipse.jetty.http2:http2-server' - serverLib 'org.eclipse.jetty.http2:http2-common' - serverLib 'org.eclipse.jetty.http2:http2-hpack' + serverLib libs.eclipse.jetty.http2.server + serverLib libs.eclipse.jetty.http2.common + serverLib libs.eclipse.jetty.http2.hpack - serverLib 'org.eclipse.jetty.toolchain:jetty-servlet-api' + serverLib libs.eclipse.jetty.toolchain.servletapi - libExt 'com.lmax:disruptor' - libExt 'org.slf4j:jcl-over-slf4j' - libExt 'org.slf4j:jul-to-slf4j' - libExt 'org.slf4j:slf4j-api' - libExt 'org.apache.logging.log4j:log4j-1.2-api' - libExt 'org.apache.logging.log4j:log4j-api' - libExt 'org.apache.logging.log4j:log4j-core' - libExt 'org.apache.logging.log4j:log4j-layout-template-json' - libExt 'org.apache.logging.log4j:log4j-slf4j2-impl' - libExt 'org.apache.logging.log4j:log4j-web' + libExt libs.lmax.disruptor + libExt libs.slf4j.jcloverslf4j + libExt libs.slf4j.jultoslf4j + libExt libs.slf4j.api + libExt libs.apache.log4j1.api + libExt libs.apache.log4j.api + libExt libs.apache.log4j.core + libExt libs.apache.log4j.layout.templatejson + libExt libs.apache.log4j.slf4j2impl + libExt libs.apache.log4j.web - libExt('io.dropwizard.metrics:metrics-core', { + libExt(libs.dropwizard.metrics.core, { exclude group: "com.rabbitmq", module: "amqp-client" }) - libExt('io.dropwizard.metrics:metrics-graphite', { + libExt(libs.dropwizard.metrics.graphite, { exclude group: "com.rabbitmq", module: "amqp-client" }) - libExt('io.dropwizard.metrics:metrics-jetty10', { + libExt(libs.dropwizard.metrics.jetty10, { exclude group: "org.eclipse.jetty", module: "*" exclude group: "org.eclipse.jetty.http2", module: "*" exclude group: "org.eclipse.jetty.toolchain", module: "*" }) - libExt 'io.dropwizard.metrics:metrics-jvm' - libExt 'io.dropwizard.metrics:metrics-jmx' + libExt libs.dropwizard.metrics.jvm + libExt libs.dropwizard.metrics.jmx webapp project(path: ":solr:webapp", configuration: "war") - startJar('org.eclipse.jetty:jetty-start::shaded', { + startJar(variantOf(libs.eclipse.jetty.start) { classifier 'shaded' }, { transitive false }) diff --git a/solr/server/etc/security.policy b/solr/server/etc/security.policy index c898ac8dbfe..d9f8791486d 100644 --- a/solr/server/etc/security.policy +++ b/solr/server/etc/security.policy @@ -60,6 +60,15 @@ grant { // needed by randomizedtesting runner to identify test methods. permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; + permission java.lang.reflect.ReflectPermission "newProxyInPackage.dev.langchain4j.model.cohere"; + permission java.lang.reflect.ReflectPermission "newProxyInPackage.dev.ai4j.openai4j"; + permission java.lang.reflect.ReflectPermission "newProxyInPackage.dev.langchain4j.model.huggingface"; + permission java.net.SocketPermission "api.cohere.ai", "accept,listen,connect,resolve"; + permission java.net.SocketPermission "api.openai.com", "accept,listen,connect,resolve"; + permission java.net.SocketPermission "api.mistral.ai", "accept,listen,connect,resolve"; + permission java.net.SocketPermission "api-inference.huggingface.co", "accept,listen,connect,resolve"; + permission java.net.SocketPermission "langchain4j.dev", "accept,listen,connect,resolve"; + permission java.lang.RuntimePermission "accessDeclaredMembers"; // needed by certain tests to redirect sysout/syserr: permission java.lang.RuntimePermission "setIO"; @@ -95,7 +104,7 @@ grant { permission java.lang.RuntimePermission "closeClassLoader"; // needed by HttpSolrClient permission java.lang.RuntimePermission "getFileSystemAttributes"; - // needed by hadoop auth (TODO: there is a cleaner way to handle this) + // needed by hadoop hdfs (TODO: there is a cleaner way to handle this) permission java.lang.RuntimePermission "loadLibrary.jaas"; permission java.lang.RuntimePermission "loadLibrary.jaas_unix"; permission java.lang.RuntimePermission "loadLibrary.jaas_nt"; @@ -108,7 +117,6 @@ grant { permission java.lang.RuntimePermission "getProtectionDomain"; // needed by aws s3 sdk (Apache HTTP Client) permission java.lang.RuntimePermission "accessClassInPackage.jdk.internal.reflect"; - // These two *have* to be spelled out a separate permission java.lang.management.ManagementPermission "control"; permission java.lang.management.ManagementPermission "monitor"; @@ -131,16 +139,18 @@ grant { permission javax.management.MBeanServerPermission "releaseMBeanServer"; permission javax.management.MBeanTrustPermission "register"; - // needed by hadoop auth + // needed by hadoop hdfs permission javax.security.auth.AuthPermission "getSubject"; permission javax.security.auth.AuthPermission "modifyPrincipals"; permission javax.security.auth.AuthPermission "doAs"; - permission javax.security.auth.AuthPermission "getLoginConfiguration"; - permission javax.security.auth.AuthPermission "setLoginConfiguration"; permission javax.security.auth.AuthPermission "modifyPrivateCredentials"; permission javax.security.auth.AuthPermission "modifyPublicCredentials"; permission javax.security.auth.PrivateCredentialPermission "org.apache.hadoop.security.Credentials * \"*\"", "read"; + // needed by crossdc + permission javax.security.auth.AuthPermission "getLoginConfiguration"; + permission javax.security.auth.AuthPermission "setLoginConfiguration"; + // needed by hadoop security permission java.security.SecurityPermission "putProviderProperty.SaslPlainServer"; permission java.security.SecurityPermission "insertProvider"; diff --git a/solr/server/resources/log4j2-console.xml b/solr/server/resources/log4j2-console.xml index c33c93c9327..2d5b8690bf6 100644 --- a/solr/server/resources/log4j2-console.xml +++ b/solr/server/resources/log4j2-console.xml @@ -16,9 +16,9 @@ limitations under the License. --> - - + diff --git a/solr/server/resources/log4j2.xml b/solr/server/resources/log4j2.xml index 006de0c965c..f5b373338b7 100644 --- a/solr/server/resources/log4j2.xml +++ b/solr/server/resources/log4j2.xml @@ -16,7 +16,7 @@ limitations under the License. --> - + @@ -62,7 +62,7 @@ - + diff --git a/solr/server/solr/configsets/_default/conf/solrconfig.xml b/solr/server/solr/configsets/_default/conf/solrconfig.xml index 5bc2cf13415..3cec6a4e578 100644 --- a/solr/server/solr/configsets/_default/conf/solrconfig.xml +++ b/solr/server/solr/configsets/_default/conf/solrconfig.xml @@ -37,51 +37,6 @@ --> 9.11 - - - - - - - - 9.11 - - - - - - - - - - - - - - - -