diff --git a/.gitignore b/.gitignore
index 7514d55cc3c9a..291a63cdeef92 100644
--- a/.gitignore
+++ b/.gitignore
@@ -64,4 +64,5 @@ testfixtures_shared/
.ci/jobs/
# build files generated
-doc-tools/missing-doclet/bin/
\ No newline at end of file
+doc-tools/missing-doclet/bin/
+server/src/main/java/org/opensearch/indices/KLSPerformanceTest.java
diff --git a/.idea/runConfigurations/Debug_OpenSearch.xml b/.idea/runConfigurations/Debug_OpenSearch.xml
index 0d8bf59823acf..c18046f873477 100644
--- a/.idea/runConfigurations/Debug_OpenSearch.xml
+++ b/.idea/runConfigurations/Debug_OpenSearch.xml
@@ -6,6 +6,10 @@
+
+
+
+
-
+
\ No newline at end of file
diff --git a/distribution/src/config/opensearch.yml b/distribution/src/config/opensearch.yml
index b7ab2e1c2309b..78d719edd599a 100644
--- a/distribution/src/config/opensearch.yml
+++ b/distribution/src/config/opensearch.yml
@@ -127,3 +127,7 @@ ${path.logs}
# Once there is no observed impact on performance, this feature flag can be removed.
#
#opensearch.experimental.optimization.datetime_formatter_caching.enabled: false
+#
+#
+# Gates the tiered caching feature. This feature adds a disk tier to the IndicesRequestCache.
+# opensearch.experimental.feature.tiered_caching.enabled: false
diff --git a/server/build.gradle b/server/build.gradle
index fd15367553fe9..de8d2c53af3d3 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -161,6 +161,9 @@ dependencies {
api "org.ehcache:ehcache:${versions.ehcache}"
api "org.slf4j:slf4j-api:${versions.slf4j}"
+ // roaring bitmaps
+ api 'org.roaringbitmap:RoaringBitmap:0.9.49'
+ runtimeOnly 'org.roaringbitmap:shims:0.9.49'
testImplementation(project(":test:framework")) {
// tests use the locally compiled version of server
diff --git a/server/licenses/RoaringBitmap-0.9.49.jar.sha1 b/server/licenses/RoaringBitmap-0.9.49.jar.sha1
new file mode 100644
index 0000000000000..919a73c074b6a
--- /dev/null
+++ b/server/licenses/RoaringBitmap-0.9.49.jar.sha1
@@ -0,0 +1 @@
+b45b49c1ec5c5fc48580412d0ca635e1833110ea
\ No newline at end of file
diff --git a/server/licenses/RoaringBitmap-LICENSE.txt b/server/licenses/RoaringBitmap-LICENSE.txt
new file mode 100644
index 0000000000000..a890d4a062fad
--- /dev/null
+++ b/server/licenses/RoaringBitmap-LICENSE.txt
@@ -0,0 +1,191 @@
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction, and
+distribution as defined by Sections 1 through 9 of this document.
+
+"Licensor" shall mean the copyright owner or entity authorized by the copyright
+owner that is granting the License.
+
+"Legal Entity" shall mean the union of the acting entity and all other entities
+that control, are controlled by, or are under common control with that entity.
+For the purposes of this definition, "control" means (i) the power, direct or
+indirect, to cause the direction or management of such entity, whether by
+contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the
+outstanding shares, or (iii) beneficial ownership of such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity exercising
+permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications, including
+but not limited to software source code, documentation source, and configuration
+files.
+
+"Object" form shall mean any form resulting from mechanical transformation or
+translation of a Source form, including but not limited to compiled object code,
+generated documentation, and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or Object form, made
+available under the License, as indicated by a copyright notice that is included
+in or attached to the work (an example is provided in the Appendix below).
+
+"Derivative Works" shall mean any work, whether in Source or Object form, that
+is based on (or derived from) the Work and for which the editorial revisions,
+annotations, elaborations, or other modifications represent, as a whole, an
+original work of authorship. For the purposes of this License, Derivative Works
+shall not include works that remain separable from, or merely link (or bind by
+name) to the interfaces of, the Work and Derivative Works thereof.
+
+"Contribution" shall mean any work of authorship, including the original version
+of the Work and any modifications or additions to that Work or Derivative Works
+thereof, that is intentionally submitted to Licensor for inclusion in the Work
+by the copyright owner or by an individual or Legal Entity authorized to submit
+on behalf of the copyright owner. For the purposes of this definition,
+"submitted" means any form of electronic, verbal, or written communication sent
+to the Licensor or its representatives, including but not limited to
+communication on electronic mailing lists, source code control systems, and
+issue tracking systems that are managed by, or on behalf of, the Licensor for
+the purpose of discussing and improving the Work, but excluding communication
+that is conspicuously marked or otherwise designated in writing by the copyright
+owner as "Not a Contribution."
+
+"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
+of whom a Contribution has been received by Licensor and subsequently
+incorporated within the Work.
+
+2. Grant of Copyright License.
+
+Subject to the terms and conditions of this License, each Contributor hereby
+grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
+irrevocable copyright license to reproduce, prepare Derivative Works of,
+publicly display, publicly perform, sublicense, and distribute the Work and such
+Derivative Works in Source or Object form.
+
+3. Grant of Patent License.
+
+Subject to the terms and conditions of this License, each Contributor hereby
+grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
+irrevocable (except as stated in this section) patent license to make, have
+made, use, offer to sell, sell, import, and otherwise transfer the Work, where
+such license applies only to those patent claims licensable by such Contributor
+that are necessarily infringed by their Contribution(s) alone or by combination
+of their Contribution(s) with the Work to which such Contribution(s) was
+submitted. If You institute patent litigation against any entity (including a
+cross-claim or counterclaim in a lawsuit) alleging that the Work or a
+Contribution incorporated within the Work constitutes direct or contributory
+patent infringement, then any patent licenses granted to You under this License
+for that Work shall terminate as of the date such litigation is filed.
+
+4. Redistribution.
+
+You may reproduce and distribute copies of the Work or Derivative Works thereof
+in any medium, with or without modifications, and in Source or Object form,
+provided that You meet the following conditions:
+
+You must give any other recipients of the Work or Derivative Works a copy of
+this License; and
+You must cause any modified files to carry prominent notices stating that You
+changed the files; and
+You must retain, in the Source form of any Derivative Works that You distribute,
+all copyright, patent, trademark, and attribution notices from the Source form
+of the Work, excluding those notices that do not pertain to any part of the
+Derivative Works; and
+If the Work includes a "NOTICE" text file as part of its distribution, then any
+Derivative Works that You distribute must include a readable copy of the
+attribution notices contained within such NOTICE file, excluding those notices
+that do not pertain to any part of the Derivative Works, in at least one of the
+following places: within a NOTICE text file distributed as part of the
+Derivative Works; within the Source form or documentation, if provided along
+with the Derivative Works; or, within a display generated by the Derivative
+Works, if and wherever such third-party notices normally appear. The contents of
+the NOTICE file are for informational purposes only and do not modify the
+License. You may add Your own attribution notices within Derivative Works that
+You distribute, alongside or as an addendum to the NOTICE text from the Work,
+provided that such additional attribution notices cannot be construed as
+modifying the License.
+You may add Your own copyright statement to Your modifications and may provide
+additional or different license terms and conditions for use, reproduction, or
+distribution of Your modifications, or for any such Derivative Works as a whole,
+provided Your use, reproduction, and distribution of the Work otherwise complies
+with the conditions stated in this License.
+
+5. Submission of Contributions.
+
+Unless You explicitly state otherwise, any Contribution intentionally submitted
+for inclusion in the Work by You to the Licensor shall be under the terms and
+conditions of this License, without any additional terms or conditions.
+Notwithstanding the above, nothing herein shall supersede or modify the terms of
+any separate license agreement you may have executed with Licensor regarding
+such Contributions.
+
+6. Trademarks.
+
+This License does not grant permission to use the trade names, trademarks,
+service marks, or product names of the Licensor, except as required for
+reasonable and customary use in describing the origin of the Work and
+reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty.
+
+Unless required by applicable law or agreed to in writing, Licensor provides the
+Work (and each Contributor provides its Contributions) on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
+including, without limitation, any warranties or conditions of TITLE,
+NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are
+solely responsible for determining the appropriateness of using or
+redistributing the Work and assume any risks associated with Your exercise of
+permissions under this License.
+
+8. Limitation of Liability.
+
+In no event and under no legal theory, whether in tort (including negligence),
+contract, or otherwise, unless required by applicable law (such as deliberate
+and grossly negligent acts) or agreed to in writing, shall any Contributor be
+liable to You for damages, including any direct, indirect, special, incidental,
+or consequential damages of any character arising as a result of this License or
+out of the use or inability to use the Work (including but not limited to
+damages for loss of goodwill, work stoppage, computer failure or malfunction, or
+any and all other commercial damages or losses), even if such Contributor has
+been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability.
+
+While redistributing the Work or Derivative Works thereof, You may choose to
+offer, and charge a fee for, acceptance of support, warranty, indemnity, or
+other liability obligations and/or rights consistent with this License. However,
+in accepting such obligations, You may act only on Your own behalf and on Your
+sole responsibility, not on behalf of any other Contributor, and only if You
+agree to indemnify, defend, and hold each Contributor harmless for any liability
+incurred by, or claims asserted against, such Contributor by reason of your
+accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work
+
+To apply the Apache License to your work, attach the following boilerplate
+notice, with the fields enclosed by brackets "[]" replaced with your own
+identifying information. (Don't include the brackets!) The text should be
+enclosed in the appropriate comment syntax for the file format. We also
+recommend that a file or class name and description of purpose be included on
+the same "printed page" as the copyright notice for easier identification within
+third-party archives.
+
+ Copyright 2013-2016 the RoaringBitmap authors
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/server/licenses/RoaringBitmap-NOTICE.txt b/server/licenses/RoaringBitmap-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/server/licenses/shims-0.9.49.jar.sha1 b/server/licenses/shims-0.9.49.jar.sha1
new file mode 100644
index 0000000000000..9e76614ca5207
--- /dev/null
+++ b/server/licenses/shims-0.9.49.jar.sha1
@@ -0,0 +1 @@
+8bd7794fbdaa9536354dd2d8d961d9503beb9460
\ No newline at end of file
diff --git a/server/licenses/shims-LICENSE.txt b/server/licenses/shims-LICENSE.txt
new file mode 100644
index 0000000000000..a890d4a062fad
--- /dev/null
+++ b/server/licenses/shims-LICENSE.txt
@@ -0,0 +1,191 @@
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction, and
+distribution as defined by Sections 1 through 9 of this document.
+
+"Licensor" shall mean the copyright owner or entity authorized by the copyright
+owner that is granting the License.
+
+"Legal Entity" shall mean the union of the acting entity and all other entities
+that control, are controlled by, or are under common control with that entity.
+For the purposes of this definition, "control" means (i) the power, direct or
+indirect, to cause the direction or management of such entity, whether by
+contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the
+outstanding shares, or (iii) beneficial ownership of such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity exercising
+permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications, including
+but not limited to software source code, documentation source, and configuration
+files.
+
+"Object" form shall mean any form resulting from mechanical transformation or
+translation of a Source form, including but not limited to compiled object code,
+generated documentation, and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or Object form, made
+available under the License, as indicated by a copyright notice that is included
+in or attached to the work (an example is provided in the Appendix below).
+
+"Derivative Works" shall mean any work, whether in Source or Object form, that
+is based on (or derived from) the Work and for which the editorial revisions,
+annotations, elaborations, or other modifications represent, as a whole, an
+original work of authorship. For the purposes of this License, Derivative Works
+shall not include works that remain separable from, or merely link (or bind by
+name) to the interfaces of, the Work and Derivative Works thereof.
+
+"Contribution" shall mean any work of authorship, including the original version
+of the Work and any modifications or additions to that Work or Derivative Works
+thereof, that is intentionally submitted to Licensor for inclusion in the Work
+by the copyright owner or by an individual or Legal Entity authorized to submit
+on behalf of the copyright owner. For the purposes of this definition,
+"submitted" means any form of electronic, verbal, or written communication sent
+to the Licensor or its representatives, including but not limited to
+communication on electronic mailing lists, source code control systems, and
+issue tracking systems that are managed by, or on behalf of, the Licensor for
+the purpose of discussing and improving the Work, but excluding communication
+that is conspicuously marked or otherwise designated in writing by the copyright
+owner as "Not a Contribution."
+
+"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
+of whom a Contribution has been received by Licensor and subsequently
+incorporated within the Work.
+
+2. Grant of Copyright License.
+
+Subject to the terms and conditions of this License, each Contributor hereby
+grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
+irrevocable copyright license to reproduce, prepare Derivative Works of,
+publicly display, publicly perform, sublicense, and distribute the Work and such
+Derivative Works in Source or Object form.
+
+3. Grant of Patent License.
+
+Subject to the terms and conditions of this License, each Contributor hereby
+grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
+irrevocable (except as stated in this section) patent license to make, have
+made, use, offer to sell, sell, import, and otherwise transfer the Work, where
+such license applies only to those patent claims licensable by such Contributor
+that are necessarily infringed by their Contribution(s) alone or by combination
+of their Contribution(s) with the Work to which such Contribution(s) was
+submitted. If You institute patent litigation against any entity (including a
+cross-claim or counterclaim in a lawsuit) alleging that the Work or a
+Contribution incorporated within the Work constitutes direct or contributory
+patent infringement, then any patent licenses granted to You under this License
+for that Work shall terminate as of the date such litigation is filed.
+
+4. Redistribution.
+
+You may reproduce and distribute copies of the Work or Derivative Works thereof
+in any medium, with or without modifications, and in Source or Object form,
+provided that You meet the following conditions:
+
+You must give any other recipients of the Work or Derivative Works a copy of
+this License; and
+You must cause any modified files to carry prominent notices stating that You
+changed the files; and
+You must retain, in the Source form of any Derivative Works that You distribute,
+all copyright, patent, trademark, and attribution notices from the Source form
+of the Work, excluding those notices that do not pertain to any part of the
+Derivative Works; and
+If the Work includes a "NOTICE" text file as part of its distribution, then any
+Derivative Works that You distribute must include a readable copy of the
+attribution notices contained within such NOTICE file, excluding those notices
+that do not pertain to any part of the Derivative Works, in at least one of the
+following places: within a NOTICE text file distributed as part of the
+Derivative Works; within the Source form or documentation, if provided along
+with the Derivative Works; or, within a display generated by the Derivative
+Works, if and wherever such third-party notices normally appear. The contents of
+the NOTICE file are for informational purposes only and do not modify the
+License. You may add Your own attribution notices within Derivative Works that
+You distribute, alongside or as an addendum to the NOTICE text from the Work,
+provided that such additional attribution notices cannot be construed as
+modifying the License.
+You may add Your own copyright statement to Your modifications and may provide
+additional or different license terms and conditions for use, reproduction, or
+distribution of Your modifications, or for any such Derivative Works as a whole,
+provided Your use, reproduction, and distribution of the Work otherwise complies
+with the conditions stated in this License.
+
+5. Submission of Contributions.
+
+Unless You explicitly state otherwise, any Contribution intentionally submitted
+for inclusion in the Work by You to the Licensor shall be under the terms and
+conditions of this License, without any additional terms or conditions.
+Notwithstanding the above, nothing herein shall supersede or modify the terms of
+any separate license agreement you may have executed with Licensor regarding
+such Contributions.
+
+6. Trademarks.
+
+This License does not grant permission to use the trade names, trademarks,
+service marks, or product names of the Licensor, except as required for
+reasonable and customary use in describing the origin of the Work and
+reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty.
+
+Unless required by applicable law or agreed to in writing, Licensor provides the
+Work (and each Contributor provides its Contributions) on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
+including, without limitation, any warranties or conditions of TITLE,
+NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are
+solely responsible for determining the appropriateness of using or
+redistributing the Work and assume any risks associated with Your exercise of
+permissions under this License.
+
+8. Limitation of Liability.
+
+In no event and under no legal theory, whether in tort (including negligence),
+contract, or otherwise, unless required by applicable law (such as deliberate
+and grossly negligent acts) or agreed to in writing, shall any Contributor be
+liable to You for damages, including any direct, indirect, special, incidental,
+or consequential damages of any character arising as a result of this License or
+out of the use or inability to use the Work (including but not limited to
+damages for loss of goodwill, work stoppage, computer failure or malfunction, or
+any and all other commercial damages or losses), even if such Contributor has
+been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability.
+
+While redistributing the Work or Derivative Works thereof, You may choose to
+offer, and charge a fee for, acceptance of support, warranty, indemnity, or
+other liability obligations and/or rights consistent with this License. However,
+in accepting such obligations, You may act only on Your own behalf and on Your
+sole responsibility, not on behalf of any other Contributor, and only if You
+agree to indemnify, defend, and hold each Contributor harmless for any liability
+incurred by, or claims asserted against, such Contributor by reason of your
+accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work
+
+To apply the Apache License to your work, attach the following boilerplate
+notice, with the fields enclosed by brackets "[]" replaced with your own
+identifying information. (Don't include the brackets!) The text should be
+enclosed in the appropriate comment syntax for the file format. We also
+recommend that a file or class name and description of purpose be included on
+the same "printed page" as the copyright notice for easier identification within
+third-party archives.
+
+ Copyright 2013-2016 the RoaringBitmap authors
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/server/licenses/shims-NOTICE.txt b/server/licenses/shims-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/server/licenses/slf4j-api-LICENSE.txt b/server/licenses/slf4j-api-LICENSE.txt
index 2be7689435062..f687729a0b452 100644
--- a/server/licenses/slf4j-api-LICENSE.txt
+++ b/server/licenses/slf4j-api-LICENSE.txt
@@ -1,4 +1,4 @@
-Copyright (c) 2004-2022 QOS.ch
+Copyright (c) 2004-2022 QOS.ch Sarl (Switzerland)
All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining
@@ -18,4 +18,4 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
\ No newline at end of file
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
new file mode 100644
index 0000000000000..aa1b64e4bcae8
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheDiskTierIT.java
@@ -0,0 +1,438 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.indices;
+
+import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
+import org.opensearch.action.search.SearchResponse;
+import org.opensearch.client.Client;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.cache.tier.DiskTierTookTimePolicy;
+import org.opensearch.common.cache.tier.TierType;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.index.cache.request.RequestCacheStats;
+import org.opensearch.index.cache.request.ShardRequestCache;
+import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.test.OpenSearchIntegTestCase;
+
+import java.time.Duration;
+import java.time.Instant;
+
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse;
+
+// This is a separate file from IndicesRequestCacheIT because we only want to run our test
+// on a node with a maximum request cache size that we set.
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class IndicesRequestCacheDiskTierIT extends OpenSearchIntegTestCase {
+ @Override
+ protected Settings featureFlagSettings() {
+ return Settings.builder()
+ .put(super.featureFlagSettings())
+ .put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true")
+ .put(FeatureFlags.TIERED_CACHING, "true")
+ .build();
+ }
+
+ public void testDiskTierStats() throws Exception {
+ int heapSizeBytes = 9876;
+ String node = internalCluster().startNode(
+ Settings.builder()
+ .put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), new ByteSizeValue(heapSizeBytes))
+ .put(DiskTierTookTimePolicy.DISK_TOOKTIME_THRESHOLD_SETTING.getKey(), TimeValue.ZERO) // allow into disk cache regardless of took time
+ .put(IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED.getKey(), true)
+ );
+ Client client = client(node);
+ startIndex(client);
+ int requestSize = getRequestSize(client, "index", heapSizeBytes);
+
+ int numOnDisk = 5;
+ int numRequests = heapSizeBytes / requestSize + numOnDisk;
+ for (int i = 1; i < numRequests; i++) {
+ doHelloSearch(client, "index", i);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.ON_HEAP, false);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.DISK, false);
+ }
+
+ // So far, disk-specific stats should be 0, as keystore has prevented any actual disk reaches
+ long tookTimeSoFar = assertDiskTierSpecificStats(client, "index", 0, -1, 0);
+
+ // the first request, for "hello0", should have been evicted to the disk tier
+ doHelloSearch(client, "index", 0);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 0, numRequests + 1, TierType.ON_HEAP, false);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 1, numRequests, TierType.DISK, false);
+ tookTimeSoFar = assertDiskTierSpecificStats(client, "index", 1, 0, -1);
+
+ // We make another actual request that should be in the disk tier. Disk specific stats should keep incrementing
+ doHelloSearch(client, "index", 1);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 0, numRequests + 2, TierType.ON_HEAP, false);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 2, numRequests, TierType.DISK, false);
+ tookTimeSoFar = assertDiskTierSpecificStats(client, "index", 2, tookTimeSoFar, -1);
+
+ // A final request for something in neither tier shouldn't increment disk specific stats
+ doHelloSearch(client, "index", numRequests);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 0, numRequests + 3, TierType.ON_HEAP, false);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 2, numRequests + 1, TierType.DISK, false);
+ assertDiskTierSpecificStats(client, "index", 2, tookTimeSoFar, tookTimeSoFar);
+ }
+
+ public void testTogglingDiskTierSetting() throws Exception {
+ // For this test, feature flag tiered caching setting is on, but we toggle IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED
+ // and cover all possible transitions
+ // start with it off (default), then turn it on dynamically
+
+ // setup
+ int heapSizeBytes = 5000;
+ String node = internalCluster().startNode(
+ Settings.builder()
+ .put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), new ByteSizeValue(heapSizeBytes))
+ .put(DiskTierTookTimePolicy.DISK_TOOKTIME_THRESHOLD_SETTING.getKey(), TimeValue.ZERO) // allow into disk cache regardless of took time
+ );
+ Client client = client(node);
+ startIndex(client);
+ int requestSize = getRequestSize(client, "index", heapSizeBytes);
+
+ // Make more queries than fit in the heap cache, and confirm these do not spill over to a disk tier, as we haven't made one yet.
+
+ int numEvicted = 5;
+ int numRequests = heapSizeBytes / requestSize + numEvicted;
+ SearchResponse resp;
+ for (int i = 1; i < numRequests; i++) {
+ doHelloSearch(client, "index", i);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.ON_HEAP, false);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 0, 0, TierType.DISK, false);
+ }
+ IndicesRequestCacheIT.assertNumEvictions(client, "index", numEvicted, TierType.ON_HEAP);
+ // The heap tier should now contain requests # numEvicted through numRequests. 0 through numEvicted have been evicted and lost.
+
+ // Set IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED = false. Nothing should happen
+ setDiskCacheEnabledSetting(client, false);
+
+ // Set IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED = true, creating a new disk tier
+ setDiskCacheEnabledSetting(client, true);
+
+ // Make some new requests. These should cause evictions from the heap tier which spill over to the disk tier (requests between numEvicted and numEvicted + numOnDisk)
+ int numOnDisk = 3;
+ for (int i = numRequests; i < numRequests + numOnDisk; i++) {
+ doHelloSearch(client, "index", i);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.ON_HEAP, false);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 0, i - numRequests + 1, TierType.DISK, false);
+ }
+ IndicesRequestCacheIT.assertNumEvictions(client, "index", numEvicted + numOnDisk, TierType.ON_HEAP);
+ IndicesRequestCacheIT.assertNumCacheEntries(client, "index", heapSizeBytes / requestSize, TierType.ON_HEAP);
+ IndicesRequestCacheIT.assertNumCacheEntries(client, "index", numOnDisk, TierType.DISK);
+ // Confirm request # numEvicted + 1 is a disk tier hit
+ doHelloSearch(client, "index", numEvicted + 1);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 0, numRequests + numOnDisk + 1, TierType.ON_HEAP, false);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 1, numOnDisk, TierType.DISK, false);
+
+ // Set IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED = true. Nothing should happen
+ setDiskCacheEnabledSetting(client, true);
+
+ // Set IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED = false. The disk tier should be deactivated but not deleted
+ setDiskCacheEnabledSetting(client, false);
+ IndicesRequestCacheIT.assertNumCacheEntries(client, "index", numOnDisk, TierType.DISK);
+
+ // Make some new requests. These should cause evictions from the heap tier (numEvicted + numOnDisk through numEvicted + numOnDisk + numNewRequests), but not reach the deactivated disk tier
+ int numNewRequests = 5;
+ assertTrue(numNewRequests > 3); // necessary for checks later in the test
+ for (int i = numRequests + numOnDisk; i < numRequests + numOnDisk + numNewRequests; i++) {
+ doHelloSearch(client, "index", i);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 2, TierType.ON_HEAP, false);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 1, numOnDisk, TierType.DISK, false);
+ }
+ IndicesRequestCacheIT.assertNumEvictions(client, "index", numEvicted + numOnDisk + numNewRequests, TierType.ON_HEAP);
+ IndicesRequestCacheIT.assertNumCacheEntries(client, "index", heapSizeBytes / requestSize, TierType.ON_HEAP);
+ IndicesRequestCacheIT.assertNumCacheEntries(client, "index", numOnDisk, TierType.DISK);
+ // Now the heap tier contains requests numEvicted + numOnDisk + numNewRequests through numEvicted + numOnDisk + numNewRequests + num
+
+ // Check a request (# numEvicted) which was on the disk tier is now a cache miss. This evicts #
+ doHelloSearch(client, "index", numEvicted);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 0, numRequests + numOnDisk + numNewRequests + 2, TierType.ON_HEAP, false);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 1, numOnDisk, TierType.DISK, false);
+
+
+ // Re-enable the disk tier. We should get the same previously deactivated disk tier, not a new one
+ setDiskCacheEnabledSetting(client, true);
+ IndicesRequestCacheIT.assertNumCacheEntries(client, "index", numOnDisk, TierType.DISK);
+ // Request # numEvicted + 2 should be a disk tier hit
+ doHelloSearch(client, "index", numEvicted + 2);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 0, numRequests + numOnDisk + numNewRequests + 3, TierType.ON_HEAP, false);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 2, numOnDisk, TierType.DISK, false);
+ // New queries should cause spillover to the disk tier again
+ doHelloSearch(client, "index", numRequests + numOnDisk + numNewRequests);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 0, numRequests + numOnDisk + numNewRequests + 4, TierType.ON_HEAP, false);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 2, numOnDisk + 1, TierType.DISK, false);
+ IndicesRequestCacheIT.assertNumCacheEntries(client, "index", numOnDisk + 1, TierType.DISK);
+
+ // TODO: Once Kiran's cache clear PR is done, add a case where we deactivate, clear disk cache, reactivate, and see a new empty disk tier.
+ }
+
+ private void doHelloSearch(Client client, String index, int number) {
+ // make a search request of the form "hello" + number
+ SearchResponse resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + number)).get();
+ assertSearchResponse(resp);
+ }
+
+ private void startIndex(Client client) throws Exception {
+ Settings.Builder indicesSettingBuilder = Settings.builder()
+ .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+ .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+ .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
+
+ assertAcked(
+ client.admin().indices().prepareCreate("index").setMapping("k", "type=keyword").setSettings(indicesSettingBuilder).get()
+ );
+ indexRandom(true, client.prepareIndex("index").setSource("k", "hello"));
+ ensureSearchable("index");
+ }
+
+ private int getRequestSize(Client client, String index, int heapSizeBytes) throws Exception {
+ // Setup function to get the maximum number of entries of the form "hello" + i
+ // that will fit in the heap cache, by making a query and checking size afterwards
+ doHelloSearch(client, index, 0);
+ int requestSize = (int) getCacheSizeBytes(client, "index", TierType.ON_HEAP);
+ assertTrue(heapSizeBytes > requestSize);
+ // If this fails, increase heapSizeBytes! We can't adjust it after getting the size of one query
+ // as the cache size setting is not dynamic
+ return requestSize;
+ }
+
+ private void setDiskCacheEnabledSetting(Client client, boolean newSetting) {
+ ClusterUpdateSettingsRequest clusterSettingUpdate = new ClusterUpdateSettingsRequest();
+ clusterSettingUpdate.persistentSettings(Settings.builder().put(IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED.getKey(), newSetting));
+ assertAcked(client.admin().cluster().updateSettings(clusterSettingUpdate).actionGet());
+ }
+
+ public void testDiskTierInvalidationByCleanCacheAPI() throws Exception {
+ int cleanupIntervalInMillis = 10_000_000; // setting this intentionally high so that we don't get background cleanups
+ int heapSizeBytes = 9876;
+ String node = internalCluster().startNode(
+ Settings.builder()
+ .put(IndicesService.INDICES_REQUEST_CACHE_DISK_CLEAN_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(cleanupIntervalInMillis))
+ .put(IndicesService.INDICES_REQUEST_CACHE_DISK_CLEAN_THRESHOLD_SETTING.getKey(), "0%")
+ .put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), new ByteSizeValue(heapSizeBytes))
+ .put(DiskTierTookTimePolicy.DISK_TOOKTIME_THRESHOLD_SETTING.getKey(), TimeValue.ZERO) // allow into disk cache regardless of took time
+ );
+ Client client = client(node);
+
+ Settings.Builder indicesSettingBuilder = Settings.builder()
+ .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+ .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+ .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
+
+ assertAcked(
+ client.admin().indices().prepareCreate("index").setMapping("k", "type=keyword").setSettings(indicesSettingBuilder).get()
+ );
+
+ indexRandom(true, client.prepareIndex("index").setSource("k", "hello"));
+ ensureSearchable("index");
+ SearchResponse resp;
+
+ resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + 0)).get();
+ int requestSize = (int) getCacheSizeBytes(client, "index", TierType.ON_HEAP);
+ assertTrue(heapSizeBytes > requestSize);
+ // If this fails, increase heapSizeBytes! We can't adjust it after getting the size of one query
+ // as the cache size setting is not dynamic
+ int numOnDisk = 2;
+ int numRequests = heapSizeBytes / requestSize + numOnDisk;
+ for (int i = 1; i < numRequests; i++) {
+ resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + i)).get();
+ assertSearchResponse(resp);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.ON_HEAP, false);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.DISK, false);
+ }
+ IndicesRequestCacheIT.assertNumCacheEntries(client, "index", 2, TierType.DISK);
+
+ // call clear cache api
+ client.admin().indices().prepareClearCache().setIndices("index").setRequestCache(true).get();
+ // fetch the stats again
+ IndicesRequestCacheIT.assertNumCacheEntries(client, "index", 0, TierType.DISK);
+ }
+
+ // When entire disk tier is stale, test whether cache cleaner cleans up everything from disk
+ public void testDiskTierInvalidationByCacheCleanerEntireDiskTier() throws Exception {
+ int thresholdInMillis = 4_000;
+ Instant start = Instant.now();
+ int heapSizeBytes = 9876;
+ String node = internalCluster().startNode(
+ Settings.builder()
+ .put(IndicesService.INDICES_REQUEST_CACHE_DISK_CLEAN_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(thresholdInMillis))
+ .put(IndicesService.INDICES_REQUEST_CACHE_DISK_CLEAN_THRESHOLD_SETTING.getKey(), "1%")
+ .put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), new ByteSizeValue(heapSizeBytes))
+ .put(DiskTierTookTimePolicy.DISK_TOOKTIME_THRESHOLD_SETTING.getKey(), TimeValue.ZERO) // allow into disk cache regardless of took time
+ );
+ Client client = client(node);
+
+ Settings.Builder indicesSettingBuilder = Settings.builder()
+ .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+ .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+ .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
+
+ assertAcked(
+ client.admin().indices().prepareCreate("index").setMapping("k", "type=keyword").setSettings(indicesSettingBuilder).get()
+ );
+
+ indexRandom(true, client.prepareIndex("index").setSource("k", "hello"));
+ ensureSearchable("index");
+ SearchResponse resp;
+
+ resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + 0)).get();
+ int requestSize = (int) getCacheSizeBytes(client, "index", TierType.ON_HEAP);
+ assertTrue(heapSizeBytes > requestSize);
+
+ int numOnDisk = 2;
+ int numRequests = heapSizeBytes / requestSize + numOnDisk;
+ for (int i = 1; i < numRequests; i++) {
+ resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + i)).get();
+ assertSearchResponse(resp);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.ON_HEAP, false);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.DISK, false);
+ }
+
+ IndicesRequestCacheIT.assertNumCacheEntries(client, "index", 2, TierType.DISK);
+
+ // index a doc and force refresh so that the cache cleaner can clean the cache
+ indexRandom(true, client.prepareIndex("index").setSource("k", "hello"));
+ ensureSearchable("index");
+
+ // sleep for the threshold time, so that the cache cleaner can clean the cache
+ Instant end = Instant.now();
+ long elapsedTimeMillis = Duration.between(start, end).toMillis();
+ // if this test is flaky, increase the sleep time.
+ long sleepTime = (thresholdInMillis - elapsedTimeMillis) + 1_500;
+ Thread.sleep(sleepTime);
+
+ // by now cache cleaner would have run and cleaned up stale keys
+ // fetch the stats again
+ IndicesRequestCacheIT.assertNumCacheEntries(client, "index", 0, TierType.DISK);
+ }
+
+ // When part of disk tier is stale, test whether cache cleaner cleans up only stale items from disk
+ public void testDiskTierInvalidationByCacheCleanerPartOfDiskTier() throws Exception {
+ int thresholdInMillis = 4_000;
+ Instant start = Instant.now();
+ int heapSizeBytes = 987;
+ String node = internalCluster().startNode(
+ Settings.builder()
+ .put(IndicesService.INDICES_REQUEST_CACHE_DISK_CLEAN_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(thresholdInMillis))
+ .put(IndicesService.INDICES_REQUEST_CACHE_DISK_CLEAN_THRESHOLD_SETTING.getKey(), "1%")
+ .put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), new ByteSizeValue(heapSizeBytes))
+ .put(DiskTierTookTimePolicy.DISK_TOOKTIME_THRESHOLD_SETTING.getKey(), TimeValue.ZERO) // allow into disk cache regardless of took time
+ );
+ Client client = client(node);
+
+ Settings.Builder indicesSettingBuilder = Settings.builder()
+ .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+ .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+ .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
+
+ assertAcked(
+ client.admin().indices().prepareCreate("index").setMapping("k", "type=text").setSettings(indicesSettingBuilder).get()
+ );
+
+ indexRandom(true, client.prepareIndex("index").setSource("k", "hello"));
+ ensureSearchable("index");
+ SearchResponse resp;
+
+ resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + 0)).get();
+ int requestSize = (int) getCacheSizeBytes(client, "index", TierType.ON_HEAP);
+ assertTrue(heapSizeBytes > requestSize);
+
+ int numOnDisk = 2;
+ int numRequests = heapSizeBytes / requestSize + numOnDisk;
+ for (int i = 1; i < numRequests; i++) {
+ resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + i)).get();
+ assertSearchResponse(resp);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.ON_HEAP, false);
+ IndicesRequestCacheIT.assertCacheState(client, "index", 0, i + 1, TierType.DISK, false);
+ }
+
+ IndicesRequestCacheIT.assertNumCacheEntries(client, "index", 2, TierType.DISK);
+
+ // force refresh so that it creates stale keys in the cache for the cache cleaner to pick up.
+ flushAndRefresh("index");
+ client().prepareIndex("index").setId("1").setSource("k", "good bye");
+ ensureSearchable("index");
+
+ for (int i = 0; i < 6; i++) { // index 6 items with the new readerCacheKeyId
+ client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello" + i)).get();
+ }
+
+ // sleep for the threshold time, so that the cache cleaner can clean the cache
+ Instant end = Instant.now();
+ long elapsedTimeMillis = Duration.between(start, end).toMillis();
+ // if this test is flaky, increase the sleep time.
+ long sleepTime = (thresholdInMillis - elapsedTimeMillis) + 1_500;
+ Thread.sleep(sleepTime);
+
+ // make sure we have 5 entries in disk.
+ IndicesRequestCacheIT.assertNumCacheEntries(client, "index", 5, TierType.DISK);
+ }
+
+ private long getCacheSizeBytes(Client client, String index, TierType tierType) {
+ RequestCacheStats requestCacheStats = client.admin()
+ .indices()
+ .prepareStats(index)
+ .setRequestCache(true)
+ .get()
+ .getTotal()
+ .getRequestCache();
+ return requestCacheStats.getMemorySizeInBytes(tierType);
+ }
+
+ private long assertDiskTierSpecificStats(
+ Client client,
+ String index,
+ long totalDiskReaches,
+ long totalGetTimeLowerBound,
+ long totalGetTimeUpperBound
+ ) {
+ // set bounds to -1 to ignore them
+ RequestCacheStats requestCacheStats = client.admin()
+ .indices()
+ .prepareStats(index)
+ .setRequestCache(true)
+ .get()
+ .getTotal()
+ .getRequestCache();
+ ShardRequestCache.DiskStatsAccumulator specStats = requestCacheStats.getDiskSpecificStats();
+ assertEquals(totalDiskReaches, specStats.getTotalDiskReaches());
+ long tookTime = specStats.getTotalGetTime();
+ assertTrue(tookTime >= totalGetTimeLowerBound || totalGetTimeLowerBound < 0);
+ assertTrue(tookTime <= totalGetTimeUpperBound || totalGetTimeUpperBound < 0);
+ return tookTime; // Return for use in next check
+ }
+}
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
index a1815d9be2daf..02664c86d0967 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
@@ -40,6 +40,7 @@
import org.opensearch.action.search.SearchType;
import org.opensearch.client.Client;
import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.cache.tier.TierType;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.time.DateFormatter;
import org.opensearch.common.util.FeatureFlags;
@@ -84,7 +85,11 @@ public static Collection parameters() {
@Override
protected Settings featureFlagSettings() {
- return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build();
+ return Settings.builder()
+ .put(super.featureFlagSettings())
+ .put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true")
+ .put(FeatureFlags.TIERED_CACHING, "true")
+ .build();
}
// One of the primary purposes of the query cache is to cache aggs results
@@ -662,7 +667,8 @@ public void testCacheWithInvalidation() throws Exception {
resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello")).get();
assertSearchResponse(resp);
// Should expect hit as here as refresh didn't happen
- assertCacheState(client, "index", 1, 1);
+ assertCacheState(client, "index", 1, 1, TierType.ON_HEAP, false);
+ assertNumCacheEntries(client, "index", 1, TierType.ON_HEAP);
// Explicit refresh would invalidate cache
refresh();
@@ -670,10 +676,17 @@ public void testCacheWithInvalidation() throws Exception {
resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello")).get();
assertSearchResponse(resp);
// Should expect miss as key has changed due to change in IndexReader.CacheKey (due to refresh)
- assertCacheState(client, "index", 1, 2);
+ assertCacheState(client, "index", 1, 2, TierType.ON_HEAP, false);
}
- private static void assertCacheState(Client client, String index, long expectedHits, long expectedMisses) {
+ protected static void assertCacheState(
+ Client client,
+ String index,
+ long expectedHits,
+ long expectedMisses,
+ TierType tierType,
+ boolean enforceZeroEvictions
+ ) {
RequestCacheStats requestCacheStats = client.admin()
.indices()
.prepareStats(index)
@@ -683,11 +696,46 @@ private static void assertCacheState(Client client, String index, long expectedH
.getRequestCache();
// Check the hit count and miss count together so if they are not
// correct we can see both values
- assertEquals(
- Arrays.asList(expectedHits, expectedMisses, 0L),
- Arrays.asList(requestCacheStats.getHitCount(), requestCacheStats.getMissCount(), requestCacheStats.getEvictions())
- );
+ if (enforceZeroEvictions) {
+ assertEquals(
+ Arrays.asList(expectedHits, expectedMisses, 0L),
+ Arrays.asList(
+ requestCacheStats.getHitCount(tierType),
+ requestCacheStats.getMissCount(tierType),
+ requestCacheStats.getEvictions(tierType)
+ )
+ );
+ } else {
+ assertEquals(
+ Arrays.asList(expectedHits, expectedMisses),
+ Arrays.asList(requestCacheStats.getHitCount(tierType), requestCacheStats.getMissCount(tierType))
+ );
+ }
+ }
+
+ protected static void assertCacheState(Client client, String index, long expectedHits, long expectedMisses) {
+ assertCacheState(client, index, expectedHits, expectedMisses, TierType.ON_HEAP, true);
+ }
+ protected static void assertNumCacheEntries(Client client, String index, long expectedEntries, TierType tierType) {
+ RequestCacheStats requestCacheStats = client.admin()
+ .indices()
+ .prepareStats(index)
+ .setRequestCache(true)
+ .get()
+ .getTotal()
+ .getRequestCache();
+ assertEquals(expectedEntries, requestCacheStats.getEntries(tierType));
}
+ protected static void assertNumEvictions(Client client, String index, long expectedEvictions, TierType tierType) {
+ RequestCacheStats requestCacheStats = client.admin()
+ .indices()
+ .prepareStats(index)
+ .setRequestCache(true)
+ .get()
+ .getTotal()
+ .getRequestCache();
+ assertEquals(expectedEvictions, requestCacheStats.getEvictions(tierType));
+ }
}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/BytesReferenceSerializer.java b/server/src/main/java/org/opensearch/common/cache/tier/BytesReferenceSerializer.java
new file mode 100644
index 0000000000000..3ac30b09bddca
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/BytesReferenceSerializer.java
@@ -0,0 +1,42 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.tier;
+
+import org.opensearch.core.common.bytes.BytesArray;
+import org.opensearch.core.common.bytes.BytesReference;
+
+import java.util.Arrays;
+
+/**
+ * A serializer which transforms BytesReference to byte[].
+ * The type of BytesReference is NOT preserved after deserialization, but nothing in opensearch should care.
+ */
+public class BytesReferenceSerializer implements Serializer {
+ // This class does not get passed to ehcache itself, so it's not required that classes match after deserialization.
+
+ public BytesReferenceSerializer() {}
+
+ @Override
+ public byte[] serialize(BytesReference object) {
+ return BytesReference.toBytes(object);
+ }
+
+ @Override
+ public BytesReference deserialize(byte[] bytes) {
+ if (bytes == null) {
+ return null;
+ }
+ return new BytesArray(bytes);
+ }
+
+ @Override
+ public boolean equals(BytesReference object, byte[] bytes) {
+ return Arrays.equals(serialize(object), bytes);
+ }
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/CachePolicyInfoWrapper.java b/server/src/main/java/org/opensearch/common/cache/tier/CachePolicyInfoWrapper.java
new file mode 100644
index 0000000000000..ae7854850fed6
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/CachePolicyInfoWrapper.java
@@ -0,0 +1,47 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.common.cache.tier;
+
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+
+import java.io.IOException;
+
+/**
+ * A class containing information needed for all CacheTierPolicy objects to decide whether to admit
+ * a given BytesReference. This spares us from having to create an entire short-lived QuerySearchResult object
+ * just to read a few values.
+ */
+public class CachePolicyInfoWrapper implements Writeable {
+ private final Long tookTimeNanos;
+
+ public CachePolicyInfoWrapper(Long tookTimeNanos) {
+ this.tookTimeNanos = tookTimeNanos;
+ // Add more values here as they are needed for future cache tier policies
+ }
+
+ public CachePolicyInfoWrapper(StreamInput in) throws IOException {
+ this.tookTimeNanos = in.readOptionalLong();
+ }
+
+ public Long getTookTimeNanos() {
+ return tookTimeNanos;
+ }
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {
+ out.writeOptionalLong(tookTimeNanos);
+ }
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/CacheTierPolicy.java b/server/src/main/java/org/opensearch/common/cache/tier/CacheTierPolicy.java
new file mode 100644
index 0000000000000..ac5fbb230d173
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/CacheTierPolicy.java
@@ -0,0 +1,45 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.common.cache.tier;
+
+/**
+ * An interface for policies that inspect data of type T to decide whether they are admitted into a cache tier.
+ */
+public interface CacheTierPolicy {
+ /**
+ * Determines whether this policy allows the data into its cache tier.
+ * @param data The data to check
+ * @return true if accepted, otherwise false
+ */
+ boolean checkData(T data);
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/CacheValue.java b/server/src/main/java/org/opensearch/common/cache/tier/CacheValue.java
new file mode 100644
index 0000000000000..bb3735e48b181
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/CacheValue.java
@@ -0,0 +1,38 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.tier;
+
+/**
+ * Represents a cache value, with its associated tier type where it is stored,
+ * and tier-specific stats for an individual request stored in a map.
+ * @param Type of value.
+ */
+public class CacheValue {
+ V value;
+ TierType source;
+ TierRequestStats stats;
+
+ CacheValue(V value, TierType source, TierRequestStats stats) {
+ this.value = value;
+ this.source = source;
+ this.stats = stats;
+ }
+
+ public V getValue() {
+ return value;
+ }
+
+ public TierType getSource() {
+ return source;
+ }
+
+ public TierRequestStats getStats() {
+ return stats;
+ }
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/CachingTier.java b/server/src/main/java/org/opensearch/common/cache/tier/CachingTier.java
index 48fd5deadc111..2e94cacd0f40a 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/CachingTier.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/CachingTier.java
@@ -18,7 +18,7 @@
*/
public interface CachingTier {
- V get(K key);
+ CacheValue get(K key);
void put(K key, V value);
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/DiskTierProvider.java b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierProvider.java
new file mode 100644
index 0000000000000..ebb68bbc2ae63
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierProvider.java
@@ -0,0 +1,16 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.tier;
+
+/**
+ * An interface that provides new instances of a DiskTier.
+ */
+public interface DiskTierProvider {
+ DiskCachingTier createNewDiskTier();
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/DiskTierRequestStats.java b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierRequestStats.java
new file mode 100644
index 0000000000000..9c2d1c5d1a706
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierRequestStats.java
@@ -0,0 +1,36 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.tier;
+
+/**
+ * A class created by disk tier implementations containing disk-specific stats for a single request.
+ */
+public class DiskTierRequestStats implements TierRequestStats {
+
+ private final long requestGetTimeNanos;
+ private final boolean requestReachedDisk;
+
+ public DiskTierRequestStats(long requestGetTimeNanos, boolean requestReachedDisk) {
+ this.requestReachedDisk = requestReachedDisk;
+ this.requestGetTimeNanos = requestGetTimeNanos;
+ }
+
+ @Override
+ public TierType getTierType() {
+ return TierType.DISK;
+ }
+
+ public long getRequestGetTimeNanos() {
+ return requestGetTimeNanos;
+ }
+
+ public boolean getRequestReachedDisk() {
+ return requestReachedDisk;
+ }
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java
new file mode 100644
index 0000000000000..f5c52b5f0afd8
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/DiskTierTookTimePolicy.java
@@ -0,0 +1,77 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.common.cache.tier;
+
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.common.bytes.BytesReference;
+
+import java.util.function.Function;
+
+/**
+ * A cache tier policy which accepts queries whose took time is greater than some threshold,
+ * which is specified as a dynamic cluster-level setting. The threshold should be set to approximately
+ * the time it takes to get a result from the cache tier.
+ * The policy expects to be able to read a CachePolicyInfoWrapper from the start of the BytesReference.
+ */
+public class DiskTierTookTimePolicy implements CacheTierPolicy {
+ public static final Setting DISK_TOOKTIME_THRESHOLD_SETTING = Setting.positiveTimeSetting(
+ "indices.requests.cache.disk.tooktime.threshold",
+ TimeValue.ZERO,
+ Setting.Property.Dynamic,
+ Setting.Property.NodeScope
+ ); // Set this to TimeValue.ZERO to let all data through
+
+ private TimeValue threshold;
+ private final Function getPolicyInfoFn;
+
+ public DiskTierTookTimePolicy(
+ Settings settings,
+ ClusterSettings clusterSettings,
+ Function getPolicyInfoFn
+ ) {
+ this.threshold = DISK_TOOKTIME_THRESHOLD_SETTING.get(settings);
+ clusterSettings.addSettingsUpdateConsumer(DISK_TOOKTIME_THRESHOLD_SETTING, this::setThreshold);
+ this.getPolicyInfoFn = getPolicyInfoFn;
+ }
+
+ protected void setThreshold(TimeValue threshold) { // protected so that we can manually set value in unit test
+ this.threshold = threshold;
+ }
+
+ @Override
+ public boolean checkData(BytesReference data) {
+ if (threshold.equals(TimeValue.ZERO)) {
+ return true;
+ }
+ Long tookTimeNanos;
+ try {
+ tookTimeNanos = getPolicyInfoFn.apply(data).getTookTimeNanos();
+ } catch (Exception e) {
+ // If we can't retrieve the took time for whatever reason, admit the data to be safe
+ return true;
+ }
+ if (tookTimeNanos == null) {
+ // Received a null took time -> this QSR is from an old version which does not have took time, we should accept it
+ return true;
+ }
+ TimeValue tookTime = TimeValue.timeValueNanos(tookTimeNanos);
+ if (tookTime.compareTo(threshold) < 0) { // negative -> tookTime is shorter than threshold
+ return false;
+ }
+ return true;
+ }
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
index e7e52c7fbccbf..7b1e7ac9c724e 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java
@@ -8,16 +8,36 @@
package org.opensearch.common.cache.tier;
+import org.ehcache.Cache;
+import org.ehcache.CachePersistenceException;
+import org.ehcache.PersistentCacheManager;
+import org.ehcache.config.builders.CacheConfigurationBuilder;
+import org.ehcache.config.builders.CacheEventListenerConfigurationBuilder;
+import org.ehcache.config.builders.CacheManagerBuilder;
+import org.ehcache.config.builders.PooledExecutionServiceConfigurationBuilder;
+import org.ehcache.config.builders.ResourcePoolsBuilder;
+import org.ehcache.config.units.MemoryUnit;
+import org.ehcache.core.spi.service.FileBasedPersistenceContext;
+import org.ehcache.event.CacheEvent;
+import org.ehcache.event.CacheEventListener;
+import org.ehcache.event.EventType;
+import org.ehcache.expiry.ExpiryPolicy;
+import org.ehcache.impl.config.store.disk.OffHeapDiskStoreConfiguration;
+import org.ehcache.spi.serialization.SerializerException;
import org.opensearch.OpenSearchException;
import org.opensearch.common.cache.RemovalListener;
import org.opensearch.common.cache.RemovalNotification;
import org.opensearch.common.cache.RemovalReason;
+import org.opensearch.common.cache.tier.keystore.RBMIntKeyLookupStore;
import org.opensearch.common.metrics.CounterMetric;
+import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Setting.Property;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.TimeValue;
import java.io.File;
+import java.nio.ByteBuffer;
import java.time.Duration;
import java.util.Iterator;
import java.util.NoSuchElementException;
@@ -25,28 +45,59 @@
import java.util.Optional;
import java.util.function.Supplier;
-import org.ehcache.Cache;
-import org.ehcache.CachePersistenceException;
-import org.ehcache.PersistentCacheManager;
-import org.ehcache.config.builders.CacheConfigurationBuilder;
-import org.ehcache.config.builders.CacheEventListenerConfigurationBuilder;
-import org.ehcache.config.builders.CacheManagerBuilder;
-import org.ehcache.config.builders.PooledExecutionServiceConfigurationBuilder;
-import org.ehcache.config.builders.ResourcePoolsBuilder;
-import org.ehcache.config.units.MemoryUnit;
-import org.ehcache.event.CacheEvent;
-import org.ehcache.event.CacheEventListener;
-import org.ehcache.event.EventType;
-import org.ehcache.expiry.ExpiryPolicy;
-import org.ehcache.impl.config.store.disk.OffHeapDiskStoreConfiguration;
-
+/**
+ * An ehcache-based disk tier implementation.
+ * @param The key type of cache entries
+ * @param The value type of cache entries
+ */
public class EhCacheDiskCachingTier implements DiskCachingTier {
+ // Ehcache disk write minimum threads for its pool
+ // All number values in setting constructors are default value, min value, and max value
+ public static final Setting REQUEST_CACHE_DISK_MIN_THREADS = Setting.intSetting(
+ "indices.requests.cache.tiered.disk.ehcache.min_threads",
+ 2,
+ 1,
+ 5,
+ Property.NodeScope
+ );
+
+ // Ehcache disk write maximum threads for its pool
+ public static final Setting REQUEST_CACHE_DISK_MAX_THREADS = Setting.intSetting(
+ "indices.requests.cache.tiered.disk.ehcache.max_threads",
+ 2,
+ 1,
+ 20,
+ Property.NodeScope
+ );
+
+ // Not be to confused with number of disk segments, this is different. Defines
+ // distinct write queues created for disk store where a group of segments share a write queue. This is
+ // implemented with ehcache using a partitioned thread pool exectutor By default all segments share a single write
+ // queue ie write concurrency is 1. Check OffHeapDiskStoreConfiguration and DiskWriteThreadPool.
+ public static final Setting REQUEST_CACHE_DISK_WRITE_CONCURRENCY = Setting.intSetting(
+ "indices.requests.cache.tiered.disk.ehcache.write_concurrency",
+ 2,
+ 1,
+ 3,
+ Property.NodeScope
+ );
+
+ // Defines how many segments the disk cache is separated into. Higher number achieves greater concurrency but
+ // will hold that many file pointers.
+ public static final Setting REQUEST_CACHE_DISK_SEGMENTS = Setting.intSetting(
+ "indices.requests.cache.tiered.disk.ehcache.segments",
+ 16,
+ 1,
+ 32,
+ Property.NodeScope
+ );
+
// A Cache manager can create many caches.
- private final PersistentCacheManager cacheManager;
+ private static PersistentCacheManager cacheManager = null;
// Disk cache
- private Cache cache;
+ private Cache cache;
private final long maxWeightInBytes;
private final String storagePath;
@@ -59,8 +110,7 @@ public class EhCacheDiskCachingTier implements DiskCachingTier {
private final EhCacheEventListener ehCacheEventListener;
private final String threadPoolAlias;
-
- private final Settings settings;
+ private final ClusterSettings clusterSettings;
private CounterMetric count = new CounterMetric();
@@ -70,27 +120,18 @@ public class EhCacheDiskCachingTier implements DiskCachingTier {
private final static int MINIMUM_MAX_SIZE_IN_BYTES = 1024 * 100; // 100KB
- // Ehcache disk write minimum threads for its pool
- public final Setting DISK_WRITE_MINIMUM_THREADS;
+ private final RBMIntKeyLookupStore keystore;
- // Ehcache disk write maximum threads for its pool
- public final Setting DISK_WRITE_MAXIMUM_THREADS;
-
- // Not be to confused with number of disk segments, this is different. Defines
- // distinct write queues created for disk store where a group of segments share a write queue. This is
- // implemented with ehcache using a partitioned thread pool exectutor By default all segments share a single write
- // queue ie write concurrency is 1. Check OffHeapDiskStoreConfiguration and DiskWriteThreadPool.
- public final Setting DISK_WRITE_CONCURRENCY;
-
- // Defines how many segments the disk cache is separated into. Higher number achieves greater concurrency but
- // will hold that many file pointers.
- public final Setting DISK_SEGMENTS;
+ private final Serializer keySerializer;
+ private final Serializer valueSerializer;
private EhCacheDiskCachingTier(Builder builder) {
this.keyType = Objects.requireNonNull(builder.keyType, "Key type shouldn't be null");
this.valueType = Objects.requireNonNull(builder.valueType, "Value type shouldn't be null");
this.expireAfterAccess = Objects.requireNonNull(builder.expireAfterAcess, "ExpireAfterAccess value shouldn't " + "be null");
- this.ehCacheEventListener = new EhCacheEventListener();
+ this.keySerializer = Objects.requireNonNull(builder.keySerializer, "Key serializer shouldn't be null");
+ this.valueSerializer = Objects.requireNonNull(builder.valueSerializer, "Value serializer shouldn't be null");
+ this.ehCacheEventListener = new EhCacheEventListener(this.valueSerializer);
this.maxWeightInBytes = builder.maxWeightInBytes;
this.storagePath = Objects.requireNonNull(builder.storagePath, "Storage path shouldn't be null");
if (builder.threadPoolAlias == null || builder.threadPoolAlias.isBlank()) {
@@ -98,16 +139,15 @@ private EhCacheDiskCachingTier(Builder builder) {
} else {
this.threadPoolAlias = builder.threadPoolAlias;
}
- this.settings = Objects.requireNonNull(builder.settings, "Settings objects shouldn't be null");
- Objects.requireNonNull(builder.settingPrefix, "Setting prefix shouldn't be null");
- this.DISK_WRITE_MINIMUM_THREADS = Setting.intSetting(builder.settingPrefix + ".tiered.disk.ehcache.min_threads", 2, 1, 5);
- this.DISK_WRITE_MAXIMUM_THREADS = Setting.intSetting(builder.settingPrefix + ".tiered.disk.ehcache.max_threads", 2, 1, 20);
- // Default value is 1 within EhCache.
- this.DISK_WRITE_CONCURRENCY = Setting.intSetting(builder.settingPrefix + ".tiered.disk.ehcache.concurrency", 2, 1, 3);
- // Default value is 16 within Ehcache.
- this.DISK_SEGMENTS = Setting.intSetting(builder.settingPrefix + ".ehcache.disk.segments", 16, 1, 32);
- this.cacheManager = buildCacheManager();
+ this.clusterSettings = Objects.requireNonNull(builder.clusterSettings, "ClusterSettings object shouldn't be null");
+
+ // In test cases, there might be leftover cache managers and caches hanging around, from nodes created in the test case setup
+ // Destroy them before recreating them
+ close();
+ cacheManager = buildCacheManager();
this.cache = buildCache(Duration.ofMillis(expireAfterAccess.getMillis()), builder);
+
+ this.keystore = new RBMIntKeyLookupStore(clusterSettings);
}
private PersistentCacheManager buildCacheManager() {
@@ -118,32 +158,36 @@ private PersistentCacheManager buildCacheManager() {
PooledExecutionServiceConfigurationBuilder.newPooledExecutionServiceConfigurationBuilder()
.defaultPool(THREAD_POOL_ALIAS_PREFIX + "Default", 1, 3) // Default pool used for other tasks like
// event listeners
- .pool(this.threadPoolAlias, DISK_WRITE_MINIMUM_THREADS.get(settings), DISK_WRITE_MAXIMUM_THREADS.get(settings))
+ .pool(
+ this.threadPoolAlias,
+ clusterSettings.get(REQUEST_CACHE_DISK_MIN_THREADS),
+ clusterSettings.get(REQUEST_CACHE_DISK_MAX_THREADS)
+ )
.build()
)
.build(true);
}
- private Cache buildCache(Duration expireAfterAccess, Builder builder) {
- return this.cacheManager.createCache(
+ private Cache buildCache(Duration expireAfterAccess, Builder builder) {
+ return cacheManager.createCache(
DISK_CACHE_ALIAS,
CacheConfigurationBuilder.newCacheConfigurationBuilder(
- this.keyType,
- this.valueType,
+ keyType,
+ byte[].class,
ResourcePoolsBuilder.newResourcePoolsBuilder().disk(maxWeightInBytes, MemoryUnit.B)
- ).withExpiry(new ExpiryPolicy() {
+ ).withExpiry(new ExpiryPolicy() {
@Override
- public Duration getExpiryForCreation(K key, V value) {
+ public Duration getExpiryForCreation(K key, byte[] value) {
return INFINITE;
}
@Override
- public Duration getExpiryForAccess(K key, Supplier value) {
+ public Duration getExpiryForAccess(K key, Supplier value) {
return expireAfterAccess;
}
@Override
- public Duration getExpiryForUpdate(K key, Supplier oldValue, V newValue) {
+ public Duration getExpiryForUpdate(K key, Supplier oldValue, byte[] newValue) {
return INFINITE;
}
})
@@ -151,10 +195,11 @@ public Duration getExpiryForUpdate(K key, Supplier oldValue, V newV
.withService(
new OffHeapDiskStoreConfiguration(
this.threadPoolAlias,
- DISK_WRITE_CONCURRENCY.get(settings),
- DISK_SEGMENTS.get(settings)
+ clusterSettings.get(REQUEST_CACHE_DISK_WRITE_CONCURRENCY),
+ clusterSettings.get(REQUEST_CACHE_DISK_SEGMENTS)
)
)
+ .withKeySerializer(new KeySerializerWrapper(keySerializer))
);
}
@@ -175,14 +220,28 @@ private CacheEventListenerConfigurationBuilder getListenerConfiguration(Builder<
}
@Override
- public V get(K key) {
- // Optimize it by adding key store.
- return cache.get(key);
+ public CacheValue get(K key) {
+ boolean reachedDisk = false;
+ long now = System.nanoTime();
+
+ V value = null;
+ if (keystore.contains(key.hashCode()) || keystore.isFull()) { // Check in-memory store of key hashes to avoid unnecessary disk seek
+ value = valueSerializer.deserialize(cache.get(key));
+ reachedDisk = true;
+ }
+
+ long tookTime = -1L; // This value will be ignored by stats accumulator if reachedDisk is false anyway
+ if (reachedDisk) {
+ tookTime = System.nanoTime() - now;
+ }
+ DiskTierRequestStats stats = new DiskTierRequestStats(tookTime, reachedDisk);
+ return new CacheValue<>(value, TierType.DISK, stats);
}
@Override
public void put(K key, V value) {
- cache.put(key, value);
+ cache.put(key, valueSerializer.serialize(value));
+ keystore.add(key.hashCode());
}
@Override
@@ -193,8 +252,9 @@ public V computeIfAbsent(K key, TieredCacheLoader loader) throws Exception
@Override
public void invalidate(K key) {
- // There seems to be an thread leak issue while calling this and then closing cache.
+ // There seems to be a thread leak issue while calling this and then closing cache.
cache.remove(key);
+ keystore.remove(key.hashCode());
}
@Override
@@ -211,6 +271,7 @@ public void setRemovalListener(RemovalListener removalListener) {
@Override
public void invalidateAll() {
// Clear up files.
+ keystore.clear();
}
@Override
@@ -230,13 +291,13 @@ public TierType getTierType() {
@Override
public void close() {
- cacheManager.removeCache(DISK_CACHE_ALIAS);
- cacheManager.close();
try {
cacheManager.destroyCache(DISK_CACHE_ALIAS);
+ cacheManager.close();
+ cacheManager = null;
} catch (CachePersistenceException e) {
throw new OpenSearchException("Exception occurred while destroying ehcache and associated data", e);
- }
+ } catch (NullPointerException ignored) {} // Another test node has already destroyed the cache manager
}
/**
@@ -244,18 +305,23 @@ public void close() {
* @param Type of key
* @param Type of value
*/
- class EhCacheEventListener implements CacheEventListener {
+ class EhCacheEventListener implements CacheEventListener {
private Optional> removalListener;
+ // We need to pass the value serializer to this listener, as the removal listener is expecting
+ // values of type K, V, not K, byte[]
+ private Serializer valueSerializer;
- EhCacheEventListener() {}
+ EhCacheEventListener(Serializer valueSerializer) {
+ this.valueSerializer = valueSerializer;
+ }
public void setRemovalListener(RemovalListener removalListener) {
this.removalListener = Optional.ofNullable(removalListener);
}
@Override
- public void onEvent(CacheEvent event) {
+ public void onEvent(CacheEvent event) {
switch (event.getType()) {
case CREATED:
count.inc();
@@ -264,20 +330,40 @@ public void onEvent(CacheEvent event) {
case EVICTED:
this.removalListener.ifPresent(
listener -> listener.onRemoval(
- new RemovalNotification<>(event.getKey(), event.getOldValue(), RemovalReason.EVICTED)
+ new RemovalNotification<>(
+ event.getKey(),
+ valueSerializer.deserialize(event.getOldValue()),
+ RemovalReason.EVICTED,
+ TierType.DISK
+ )
)
);
count.dec();
assert event.getNewValue() == null;
break;
case REMOVED:
+ this.removalListener.ifPresent(
+ listener -> listener.onRemoval(
+ new RemovalNotification<>(
+ event.getKey(),
+ valueSerializer.deserialize(event.getOldValue()),
+ RemovalReason.INVALIDATED,
+ TierType.DISK
+ )
+ )
+ );
count.dec();
assert event.getNewValue() == null;
break;
case EXPIRED:
this.removalListener.ifPresent(
listener -> listener.onRemoval(
- new RemovalNotification<>(event.getKey(), event.getOldValue(), RemovalReason.INVALIDATED)
+ new RemovalNotification<>(
+ event.getKey(),
+ valueSerializer.deserialize(event.getOldValue()),
+ RemovalReason.INVALIDATED,
+ TierType.DISK
+ )
)
);
count.dec();
@@ -297,9 +383,9 @@ public void onEvent(CacheEvent event) {
*/
class EhCacheKeyIterator implements Iterator {
- Iterator> iterator;
+ Iterator> iterator;
- EhCacheKeyIterator(Iterator> iterator) {
+ EhCacheKeyIterator(Iterator> iterator) {
this.iterator = iterator;
}
@@ -317,6 +403,42 @@ public K next() {
}
}
+ /**
+ * The wrapper for the key serializer which is passed directly to Ehcache.
+ * Required because we cannot directly use a byte[] as an ehcache key, possibly due to an ehcache bug.
+ */
+ private class KeySerializerWrapper implements org.ehcache.spi.serialization.Serializer {
+ public Serializer serializer;
+
+ public KeySerializerWrapper(Serializer serializer) {
+ this.serializer = serializer;
+ }
+
+ // This constructor must be present, but does not have to work as we are not actually persisting the disk
+ // cache after a restart.
+ // See https://www.ehcache.org/documentation/3.0/serializers-copiers.html#persistent-vs-transient-caches
+ public KeySerializerWrapper(ClassLoader classLoader, FileBasedPersistenceContext persistenceContext) {}
+
+ @Override
+ public ByteBuffer serialize(K object) throws SerializerException {
+ return ByteBuffer.wrap(serializer.serialize(object));
+ }
+
+ @Override
+ public K read(ByteBuffer binary) throws ClassNotFoundException, SerializerException {
+ byte[] arr = new byte[binary.remaining()];
+ binary.get(arr);
+ return serializer.deserialize(arr);
+ }
+
+ @Override
+ public boolean equals(K object, ByteBuffer binary) throws ClassNotFoundException, SerializerException {
+ byte[] arr = new byte[binary.remaining()];
+ binary.get(arr);
+ return serializer.equals(object, arr);
+ }
+ }
+
/**
* Builder object to build Ehcache disk tier.
* @param Type of key
@@ -333,15 +455,14 @@ public static class Builder {
private String storagePath;
private String threadPoolAlias;
-
- private Settings settings;
+ private ClusterSettings clusterSettings;
private String diskCacheAlias;
- private String settingPrefix;
-
// Provides capability to make ehCache event listener to run in sync mode. Used for testing too.
private boolean isEventListenerModeSync;
+ private Serializer keySerializer;
+ private Serializer valueSerializer;
public Builder() {}
@@ -378,24 +499,28 @@ public EhCacheDiskCachingTier.Builder setThreadPoolAlias(String threadPool
return this;
}
- public EhCacheDiskCachingTier.Builder setSettings(Settings settings) {
- this.settings = settings;
+ public EhCacheDiskCachingTier.Builder setDiskCacheAlias(String diskCacheAlias) {
+ this.diskCacheAlias = diskCacheAlias;
return this;
}
- public EhCacheDiskCachingTier.Builder setDiskCacheAlias(String diskCacheAlias) {
- this.diskCacheAlias = diskCacheAlias;
+ public EhCacheDiskCachingTier.Builder setIsEventListenerModeSync(boolean isEventListenerModeSync) {
+ this.isEventListenerModeSync = isEventListenerModeSync;
return this;
}
- public EhCacheDiskCachingTier.Builder setSettingPrefix(String settingPrefix) {
- // TODO: Do some basic validation. So that it doesn't end with "." etc.
- this.settingPrefix = settingPrefix;
+ public EhCacheDiskCachingTier.Builder setKeySerializer(Serializer keySerializer) {
+ this.keySerializer = keySerializer;
return this;
}
- public EhCacheDiskCachingTier.Builder setIsEventListenerModeSync(boolean isEventListenerModeSync) {
- this.isEventListenerModeSync = isEventListenerModeSync;
+ public EhCacheDiskCachingTier.Builder setValueSerializer(Serializer valueSerializer) {
+ this.valueSerializer = valueSerializer;
+ return this;
+ }
+
+ public EhCacheDiskCachingTier.Builder setClusterSettings(ClusterSettings clusterSettings) {
+ this.clusterSettings = clusterSettings;
return this;
}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/OnHeapTierRequestStats.java b/server/src/main/java/org/opensearch/common/cache/tier/OnHeapTierRequestStats.java
new file mode 100644
index 0000000000000..3527dae885fde
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/OnHeapTierRequestStats.java
@@ -0,0 +1,19 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.tier;
+
+/**
+ * A class created by on-heap tier implementations containing on-heap-specific stats for a single request.
+ */
+public class OnHeapTierRequestStats implements TierRequestStats {
+ @Override
+ public TierType getTierType() {
+ return TierType.ON_HEAP;
+ }
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/OpenSearchOnHeapCache.java b/server/src/main/java/org/opensearch/common/cache/tier/OpenSearchOnHeapCache.java
index 22d2f769507cf..c7cd065674de7 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/OpenSearchOnHeapCache.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/OpenSearchOnHeapCache.java
@@ -66,8 +66,8 @@ public TierType getTierType() {
}
@Override
- public V get(K key) {
- return cache.get(key);
+ public CacheValue get(K key) {
+ return new CacheValue(cache.get(key), TierType.ON_HEAP, new OnHeapTierRequestStats());
}
@Override
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/Serializer.java b/server/src/main/java/org/opensearch/common/cache/tier/Serializer.java
new file mode 100644
index 0000000000000..04413fce4b8b0
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/Serializer.java
@@ -0,0 +1,37 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.tier;
+
+/**
+ * An interface for serializers, to be used in disk caching tier and elsewhere.
+ * T is the class of the original object, and U is the serialized class.
+ */
+public interface Serializer {
+ /**
+ * Serializes an object.
+ * @param object A non-serialized object.
+ * @return The serialized representation of the object.
+ */
+ U serialize(T object);
+
+ /**
+ * Deserializes bytes into an object.
+ * @param bytes The serialized representation.
+ * @return The original object.
+ */
+ T deserialize(U bytes);
+
+ /**
+ * Compares an object to a serialized representation of an object.
+ * @param object A non-serialized objet
+ * @param bytes Serialized representation of an object
+ * @return true if representing the same object, false if not
+ */
+ boolean equals(T object, U bytes);
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TierRequestStats.java b/server/src/main/java/org/opensearch/common/cache/tier/TierRequestStats.java
new file mode 100644
index 0000000000000..d156be7b3f028
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/TierRequestStats.java
@@ -0,0 +1,17 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.tier;
+
+/**
+ * An interface for single-request tier-specific stats, which are produced on each request from a cache tier
+ * and then added to the correct shard's overall StatsHolder for the tier.
+ */
+public interface TierRequestStats {
+ TierType getTierType();
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TierType.java b/server/src/main/java/org/opensearch/common/cache/tier/TierType.java
index ca61b636c1dda..3ef4338030899 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/TierType.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/TierType.java
@@ -13,6 +13,17 @@
*/
public enum TierType {
- ON_HEAP,
- DISK;
+ ON_HEAP("on_heap"),
+ DISK("disk");
+
+ private final String stringValue;
+
+ TierType(String stringValue) {
+ // Associate each TierType with a string representation, for use in API responses and elsewhere
+ this.stringValue = stringValue;
+ }
+
+ public String getStringValue() {
+ return this.stringValue;
+ }
}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheEventListener.java b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheEventListener.java
index 05b59bf16b282..d1c333e21818e 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheEventListener.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheEventListener.java
@@ -17,11 +17,12 @@
*/
public interface TieredCacheEventListener {
- void onMiss(K key, TierType tierType);
+ void onMiss(K key, CacheValue cacheValue);
void onRemoval(RemovalNotification notification);
- void onHit(K key, V value, TierType tierType);
+ void onHit(K key, CacheValue cacheValue);
void onCached(K key, V value, TierType tierType);
+ // Since only get() produces a CacheValue with stats, no need to modify onCached or onRemoval to have the CacheValue.
}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
index 153dbf9b330f5..a403f4d729497 100644
--- a/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
+++ b/server/src/main/java/org/opensearch/common/cache/tier/TieredCacheSpilloverStrategyService.java
@@ -11,7 +11,11 @@
import org.opensearch.common.cache.RemovalListener;
import org.opensearch.common.cache.RemovalNotification;
import org.opensearch.common.cache.RemovalReason;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.indices.IndicesRequestCache;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
@@ -31,20 +35,30 @@ public class TieredCacheSpilloverStrategyService implements TieredCacheSer
/**
* Optional in case tiered caching is turned off.
*/
- private final Optional> diskCachingTier;
+ private Optional> diskCachingTier;
private final TieredCacheEventListener tieredCacheEventListener;
/**
* Maintains caching tiers in order of get calls.
*/
- private final List> cachingTierList;
+ private List> cachingTierList;
+ private final List> policies;
+ private boolean diskTierInUse; // True when we are using the disk tier, false when it is deactivated but not deleted
+ private DiskTierProvider diskTierProvider; // Used to obtain new instances of DiskTier when the setting is dynamically turned on
private TieredCacheSpilloverStrategyService(Builder builder) {
this.onHeapCachingTier = Objects.requireNonNull(builder.onHeapCachingTier);
this.diskCachingTier = Optional.ofNullable(builder.diskCachingTier);
+ diskTierInUse = diskCachingTier.isPresent();
this.tieredCacheEventListener = Objects.requireNonNull(builder.tieredCacheEventListener);
this.cachingTierList = this.diskCachingTier.map(diskTier -> Arrays.asList(onHeapCachingTier, diskTier))
.orElse(List.of(onHeapCachingTier));
+ this.policies = Objects.requireNonNull(builder.policies);
+ ClusterSettings clusterSettings = Objects.requireNonNull(builder.clusterSettings);
+ if (FeatureFlags.isEnabled(FeatureFlags.TIERED_CACHING)) {
+ clusterSettings.addSettingsUpdateConsumer(IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED, this::toggleDiskTierEnabled);
+ this.diskTierProvider = Objects.requireNonNull(builder.diskTierProvider);
+ }
setRemovalListeners();
}
@@ -105,6 +119,10 @@ public void invalidateAll() {
for (CachingTier cachingTier : cachingTierList) {
cachingTier.invalidateAll();
}
+ if (!diskTierInUse && diskCachingTier.isPresent()) {
+ // Invalidate all keys in an inactive disk tier as well
+ diskCachingTier.get().invalidateAll();
+ }
}
/**
@@ -130,10 +148,12 @@ public void onRemoval(RemovalNotification notification) {
if (RemovalReason.EVICTED.equals(notification.getRemovalReason())) {
switch (notification.getTierType()) {
case ON_HEAP:
- diskCachingTier.ifPresent(diskTier -> {
- diskTier.put(notification.getKey(), notification.getValue());
- tieredCacheEventListener.onCached(notification.getKey(), notification.getValue(), TierType.DISK);
- });
+ if (checkPolicies(notification.getValue()) && diskTierInUse) {
+ diskCachingTier.ifPresent(diskTier -> {
+ diskTier.put(notification.getKey(), notification.getValue());
+ tieredCacheEventListener.onCached(notification.getKey(), notification.getValue(), TierType.DISK);
+ });
+ }
break;
default:
break;
@@ -152,6 +172,15 @@ public Optional> getDiskCachingTier() {
return this.diskCachingTier;
}
+ boolean checkPolicies(V value) {
+ for (CacheTierPolicy policy : policies) {
+ if (!policy.checkData(value)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
/**
* Register this service as a listener to removal events from different caching tiers.
*/
@@ -164,15 +193,15 @@ private void setRemovalListeners() {
private Function> getValueFromTierCache(boolean trackStats) {
return key -> {
for (CachingTier cachingTier : cachingTierList) {
- V value = cachingTier.get(key);
- if (value != null) {
+ CacheValue cacheValue = cachingTier.get(key);
+ if (cacheValue.value != null) {
if (trackStats) {
- tieredCacheEventListener.onHit(key, value, cachingTier.getTierType());
+ tieredCacheEventListener.onHit(key, cacheValue);
}
- return new CacheValue<>(value, cachingTier.getTierType());
+ return cacheValue; // new CacheValue<>(value, cachingTier.getTierType());
}
if (trackStats) {
- tieredCacheEventListener.onMiss(key, cachingTier.getTierType());
+ tieredCacheEventListener.onMiss(key, cacheValue);
}
}
return null;
@@ -180,16 +209,57 @@ private Function> getValueFromTierCache(boolean trackStats) {
}
/**
- * Represents a cache value along with its associated tier type where it is stored.
- * @param Type of value.
+ * Dynamically add a new disk tier.
+ */
+ private void addNewDiskTier(DiskCachingTier newTier) {
+ assert cachingTierList.size() == 1 && diskCachingTier.isEmpty() && !diskTierInUse;
+ // list only contains heap tier, and there is no inactive disk tier
+ diskCachingTier = Optional.of(newTier);
+ cachingTierList = List.of(onHeapCachingTier, diskCachingTier.get());
+ diskTierInUse = true;
+ }
+
+ /**
+ * Dynamically reenable an existing disk tier that has been disabled but not deleted.
+ */
+ private void enableInactiveDiskTier() {
+ assert cachingTierList.size() == 1 && diskCachingTier.isPresent() && !diskTierInUse;
+ // list only contains heap tier, and there is an inactive disk tier we can reactivate
+ cachingTierList = List.of(onHeapCachingTier, diskCachingTier.get());
+ diskTierInUse = true;
+ }
+
+ /**
+ * Dynamically disable an existing disk tier.
*/
- public static class CacheValue {
- V value;
- TierType source;
+ private void disableDiskTier() {
+ // We want to stop searching in or adding to the disk tier, but we don't delete its contents until cache clear API is run.
+ // This is consistent with the heap tier's behavior.
+ // To do this, we remove the disk tier from cachingTierList but not from diskCachingTier.
+ assert cachingTierList.size() == 2 && diskTierInUse; // Contains both heap tier and disk tier
+ cachingTierList = List.of(onHeapCachingTier);
+ diskTierInUse = false; // disk tier is now inactive
+ }
- CacheValue(V value, TierType source) {
- this.value = value;
- this.source = source;
+ /**
+ * A function to run when disk tier enabled setting (IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED)
+ * is enabled/disabled dynamically.
+ * @param enableDiskTier the new value of the enable disk tier setting
+ */
+ private void toggleDiskTierEnabled(boolean enableDiskTier) {
+ if (enableDiskTier && !diskTierInUse) {
+ // Not currently using a disk tier.
+ if (diskCachingTier.isPresent()) {
+ // There is a deactivated disk tier already instantiated. Reactivate this one rather than creating a new one
+ enableInactiveDiskTier();
+ } else {
+ // There is no disk tier to reactivate, create a new one now.
+ DiskCachingTier newTier = diskTierProvider.createNewDiskTier();
+ addNewDiskTier(newTier);
+ }
+ } else if (!enableDiskTier && diskTierInUse) {
+ // If a disk tier existed before, stop using it. Do not delete it or its contents until cache clear API is run
+ disableDiskTier();
}
}
@@ -202,6 +272,9 @@ public static class Builder {
private OnHeapCachingTier onHeapCachingTier;
private DiskCachingTier diskCachingTier;
private TieredCacheEventListener tieredCacheEventListener;
+ private ArrayList> policies = new ArrayList<>();
+ private ClusterSettings clusterSettings;
+ private DiskTierProvider diskTierProvider;
public Builder() {}
@@ -220,6 +293,27 @@ public Builder setTieredCacheEventListener(TieredCacheEventListener
return this;
}
+ public Builder withPolicy(CacheTierPolicy policy) {
+ this.policies.add(policy);
+ return this;
+ }
+
+ // Add multiple policies at once
+ public Builder withPolicies(List> policiesList) {
+ this.policies.addAll(policiesList);
+ return this;
+ }
+
+ public Builder setClusterSettings(ClusterSettings clusterSettings) {
+ this.clusterSettings = clusterSettings;
+ return this;
+ }
+
+ public Builder setDiskTierProvider(DiskTierProvider provider) {
+ this.diskTierProvider = provider;
+ return this;
+ }
+
public TieredCacheSpilloverStrategyService build() {
return new TieredCacheSpilloverStrategyService(this);
}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java
new file mode 100644
index 0000000000000..dc2b7a4ba1234
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java
@@ -0,0 +1,132 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.common.cache.tier.keystore;
+
+/**
+ * An interface for objects that hold an in-memory record of hashes of keys in the disk cache.
+ * These objects have some internal data structure which stores some transformation of added
+ * int values. The internal representations may have collisions. Example transformations include a modulo
+ * or -abs(value), or some combination.
+ */
+public interface KeyLookupStore {
+
+ /**
+ * Transforms the input value into the internal representation for this keystore
+ * and adds it to the internal data structure.
+ * @param value The value to add.
+ * @return true if the value was added, false if it wasn't added because of a
+ * collision or if it was already present.
+ */
+ boolean add(T value);
+
+ /**
+ * Checks if the transformation of the value is in the keystore.
+ * @param value The value to check.
+ * @return true if the value was found, false otherwise. Due to collisions, false positives are
+ * possible, but there should be no false negatives unless forceRemove() is called.
+ */
+ boolean contains(T value);
+
+ /**
+ * Returns the transformed version of the input value, that would be used to stored it in the keystore.
+ * This transformation should be always be the same for a given instance.
+ * @param value The value to transform.
+ * @return The transformed value.
+ */
+ T getInternalRepresentation(T value);
+
+ /**
+ * Attempts to safely remove a value from the internal structure, maintaining the property that contains(value)
+ * will never return a false negative. If removing would lead to a false negative, the value won't be removed.
+ * Classes may not implement safe removal.
+ * @param value The value to attempt to remove.
+ * @return true if the value was removed, false if it wasn't.
+ */
+ boolean remove(T value);
+
+ /**
+ * Returns the number of distinct values stored in the internal data structure.
+ * Does not count values which weren't successfully added due to collisions.
+ * @return The number of values
+ */
+ int getSize();
+
+ /**
+ * Returns the number of times add() has been run, including unsuccessful attempts.
+ * @return The number of adding attempts.
+ */
+ int getAddAttempts();
+
+ /**
+ * Returns the number of times add() has returned false due to a collision.
+ * @return The number of collisions.
+ */
+ int getCollisions();
+
+ /**
+ * Checks if two values would collide after being transformed by this store's transformation.
+ * @param value1 The first value to compare.
+ * @param value2 The second value to compare.
+ * @return true if the transformations are equal, false otherwise.
+ */
+ boolean isCollision(T value1, T value2);
+
+ /**
+ * Returns an estimate of the store's memory usage.
+ * @return The memory usage
+ */
+ long getMemorySizeInBytes();
+
+ /**
+ * Returns the cap for the store's memory usage.
+ * @return The cap, in bytes
+ */
+ long getMemorySizeCapInBytes();
+
+ /**
+ * Returns whether the store is at memory capacity and can't accept more entries
+ */
+ boolean isFull();
+
+ /**
+ * Deletes the internal data structure and regenerates it from the values passed in.
+ * Also resets all stats related to adding.
+ * @param newValues The keys that should be in the reset structure.
+ */
+ void regenerateStore(T[] newValues);
+
+ /**
+ * Deletes all keys and resets all stats related to adding.
+ */
+ void clear();
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java
new file mode 100644
index 0000000000000..ab3055a81d4c9
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyStoreStats.java
@@ -0,0 +1,38 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.tier.keystore;
+
+import org.opensearch.common.metrics.CounterMetric;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * A stats holder for use in KeyLookupStore implementations.
+ * Getters should be exposed by the KeyLookupStore which uses it.
+ */
+public class KeyStoreStats {
+ protected CounterMetric size;
+ protected long memSizeCapInBytes;
+ protected CounterMetric numAddAttempts;
+ protected CounterMetric numCollisions;
+ protected boolean guaranteesNoFalseNegatives;
+ protected AtomicBoolean atCapacity;
+ protected CounterMetric numRemovalAttempts;
+ protected CounterMetric numSuccessfulRemovals;
+
+ protected KeyStoreStats(long memSizeCapInBytes) {
+ this.size = new CounterMetric();
+ this.numAddAttempts = new CounterMetric();
+ this.numCollisions = new CounterMetric();
+ this.memSizeCapInBytes = memSizeCapInBytes;
+ this.atCapacity = new AtomicBoolean(false);
+ this.numRemovalAttempts = new CounterMetric();
+ this.numSuccessfulRemovals = new CounterMetric();
+ }
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
new file mode 100644
index 0000000000000..a9734343fa762
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java
@@ -0,0 +1,407 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.common.cache.tier.keystore;
+
+import org.opensearch.common.metrics.CounterMetric;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.core.common.unit.ByteSizeValue;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.roaringbitmap.RoaringBitmap;
+
+/**
+ * This class implements KeyLookupStore using a roaring bitmap with a modulo applied to values.
+ * The modulo increases the density of values, which makes RBMs more memory-efficient. The recommended modulo is ~2^28.
+ * It also maintains a hash set of values which have had collisions. Values which haven't had collisions can be
+ * safely removed from the store. The fraction of collided values should be low,
+ * about 0.5% for a store with 10^7 values and a modulo of 2^28.
+ * The store estimates its memory footprint and will stop adding more values once it reaches its memory cap.
+ */
+public class RBMIntKeyLookupStore implements KeyLookupStore {
+ public static final Setting INDICES_CACHE_KEYSTORE_SIZE = Setting.memorySizeSetting(
+ "indices.requests.cache.tiered.disk.keystore.size",
+ "0.05%", // 5% of INDICES_CACHE_QUERY_SIZE
+ Setting.Property.Dynamic,
+ Setting.Property.NodeScope
+ );
+
+ /**
+ * An enum representing modulo values for use in the keystore
+ */
+ public enum KeystoreModuloValue {
+ NONE(0), // No modulo applied
+ TWO_TO_THIRTY_ONE((int) Math.pow(2, 31)),
+ TWO_TO_TWENTY_NINE((int) Math.pow(2, 29)), // recommended value
+ TWO_TO_TWENTY_EIGHT((int) Math.pow(2, 28)),
+ TWO_TO_TWENTY_SIX((int) Math.pow(2, 26));
+
+ private final int value;
+
+ private KeystoreModuloValue(int value) {
+ this.value = value;
+ }
+
+ public int getValue() {
+ return this.value;
+ }
+ }
+
+ protected final int modulo;
+ protected final int modulo_bitmask;
+ // Since our modulo is always a power of two we can optimize it by ANDing with a particular bitmask
+ KeyStoreStats stats;
+ protected RoaringBitmap rbm;
+ private HashMap collidedIntCounters;
+ private HashMap> removalSets;
+ protected final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ protected final Lock readLock = lock.readLock();
+ protected final Lock writeLock = lock.writeLock();
+ private long mostRecentByteEstimate;
+ static final int REFRESH_SIZE_EST_INTERVAL = 10_000;
+ // Refresh size estimate every X new elements. Refreshes use the RBM's internal size estimator, which takes ~0.01 ms,
+ // so we don't want to do it on every get(), and it doesn't matter much if there are +- 10000 keys in this store
+ // in terms of storage impact
+
+ // Use this constructor to set the memory cap to the value in ClusterSettings
+ public RBMIntKeyLookupStore(ClusterSettings clusterSettings) {
+ this(clusterSettings.get(INDICES_CACHE_KEYSTORE_SIZE).getBytes(), clusterSettings);
+ }
+
+ // Use this constructor to specify some other memory cap
+ public RBMIntKeyLookupStore(long memSizeCapInBytes, ClusterSettings clusterSettings) {
+ this(KeystoreModuloValue.TWO_TO_TWENTY_EIGHT, memSizeCapInBytes, clusterSettings);
+ }
+
+ // Use this constructor to specify memory cap and modulo
+ public RBMIntKeyLookupStore(KeystoreModuloValue moduloValue, long memSizeCapInBytes, ClusterSettings clusterSettings) {
+ this.modulo = moduloValue.getValue();
+ if (modulo > 0) {
+ this.modulo_bitmask = modulo - 1; // keep last log_2(modulo) bits
+ } else {
+ this.modulo_bitmask = -1; // -1 in twos complement is all ones -> includes all bits -> same as no modulo
+ }
+ this.stats = new KeyStoreStats(memSizeCapInBytes);
+ this.rbm = new RoaringBitmap();
+ this.collidedIntCounters = new HashMap<>();
+ this.removalSets = new HashMap<>();
+ this.mostRecentByteEstimate = 0L;
+ clusterSettings.addSettingsUpdateConsumer(INDICES_CACHE_KEYSTORE_SIZE, this::setMemSizeCap);
+ }
+
+ private int transform(int value) {
+ return value & modulo_bitmask;
+ }
+
+ private void handleCollisions(int transformedValue) {
+ stats.numCollisions.inc();
+ CounterMetric numCollisions = collidedIntCounters.get(transformedValue);
+ if (numCollisions == null) { // First time the transformedValue has had a collision
+ numCollisions = new CounterMetric();
+ numCollisions.inc(2);
+ collidedIntCounters.put(transformedValue, numCollisions); // Initialize the number of colliding keys to 2
+ } else {
+ numCollisions.inc();
+ }
+ }
+
+ @Override
+ public boolean add(Integer value) {
+ if (value == null) {
+ return false;
+ }
+ stats.numAddAttempts.inc();
+
+ if (getSize() % REFRESH_SIZE_EST_INTERVAL == 0) {
+ mostRecentByteEstimate = computeMemorySizeInBytes();
+ }
+ if (getMemorySizeCapInBytes() > 0 && mostRecentByteEstimate > getMemorySizeCapInBytes()) {
+ stats.atCapacity.set(true);
+ return false;
+ }
+ int transformedValue = transform(value);
+
+ writeLock.lock();
+ try {
+ boolean alreadyContained;
+ // saves calling transform() an additional time
+ alreadyContained = rbm.contains(transformedValue);
+ if (!alreadyContained) {
+ rbm.add(transformedValue);
+ stats.size.inc();
+ return true;
+ }
+ // If the value is already pending removal, take it out of the removalList
+ HashSet removalSet = removalSets.get(transformedValue);
+ if (removalSet != null) {
+ removalSet.remove(value);
+ // Don't increment the counter - this is handled by handleCollisions() later
+ if (removalSet.isEmpty()) {
+ removalSets.remove(transformedValue);
+ }
+ }
+
+ handleCollisions(transformedValue);
+ return false;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public boolean contains(Integer value) {
+ if (value == null) {
+ return false;
+ }
+ int transformedValue = transform(value);
+ readLock.lock();
+ try {
+ return rbm.contains(transformedValue);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public Integer getInternalRepresentation(Integer value) {
+ if (value == null) {
+ return 0;
+ }
+ return Integer.valueOf(transform(value));
+ }
+
+ /**
+ * Attempts to remove a value from the keystore. WARNING: Removing keys which have not been added to the keystore
+ * may cause undefined behavior, including future false negatives!!
+ * @param value The value to attempt to remove.
+ * @return true if the value was removed, false otherwise
+ */
+ @Override
+ public boolean remove(Integer value) {
+ if (value == null) {
+ return false;
+ }
+ int transformedValue = transform(value);
+ readLock.lock();
+ try {
+ if (!rbm.contains(transformedValue)) { // saves additional transform() call
+ return false;
+ }
+ // move below into write lock
+ stats.numRemovalAttempts.inc();
+ } finally {
+ readLock.unlock();
+ }
+ writeLock.lock();
+ try {
+ CounterMetric numCollisions = collidedIntCounters.get(transformedValue);
+ if (numCollisions != null) {
+ // This transformed value has had a collision before
+ HashSet removalSet = removalSets.get(transformedValue);
+ if (removalSet == null) {
+ // First time a removal has been attempted for this transformed value
+ HashSet newRemovalSet = new HashSet<>();
+ newRemovalSet.add(value); // Add the key value, not the transformed value, to the list of attempted removals for this
+ // transformedValue
+ removalSets.put(transformedValue, newRemovalSet);
+ numCollisions.dec();
+ } else {
+ if (removalSet.contains(value)) {
+ return false; // We have already attempted to remove this value. Do nothing
+ }
+ removalSet.add(value);
+ numCollisions.dec();
+ // If numCollisions has reached zero, we can safely remove all values in removalList
+ if (numCollisions.count() == 0) {
+ removeFromRBM(transformedValue);
+ collidedIntCounters.remove(transformedValue);
+ removalSets.remove(transformedValue);
+ return true;
+ }
+ }
+ return false;
+ }
+ // Otherwise, there's not been a collision for this transformedValue, so we can safely remove
+ removeFromRBM(transformedValue);
+ return true;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ // Helper fn for remove()
+ private void removeFromRBM(int transformedValue) {
+ rbm.remove(transformedValue);
+ stats.size.dec();
+ stats.numSuccessfulRemovals.inc();
+ }
+
+ @Override
+ public int getSize() {
+ readLock.lock();
+ try {
+ return (int) stats.size.count();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public int getAddAttempts() {
+ return (int) stats.numAddAttempts.count();
+ }
+
+ @Override
+ public int getCollisions() {
+ return (int) stats.numCollisions.count();
+ }
+
+ @Override
+ public boolean isCollision(Integer value1, Integer value2) {
+ if (value1 == null || value2 == null) {
+ return false;
+ }
+ return transform(value1) == transform(value2);
+ }
+
+ static double getRBMSizeMultiplier(int numEntries, int modulo) {
+ double effectiveModulo = (double) modulo / 2;
+ /* This model was created when we used % operator to calculate modulo. This has range (-modulo, modulo).
+ Now we have optimized to use a bitmask, which has range [0, modulo). So the number of possible values stored
+ is halved. */
+ if (modulo == 0) {
+ effectiveModulo = Math.pow(2, 32);
+ }
+ double x = Math.log10((double) numEntries / effectiveModulo);
+ if (x < -5) {
+ return 7.0;
+ }
+ if (x < -2.75) {
+ return -2.5 * x - 5.5;
+ }
+ if (x <= 0) {
+ return -3.0 / 22.0 * x + 1;
+ }
+ return 1;
+ }
+
+ /**
+ * Return the most recent memory size estimate, without updating it.
+ * @return the size estimate (bytes)
+ */
+ @Override
+ public long getMemorySizeInBytes() {
+ return mostRecentByteEstimate;
+ }
+
+ /**
+ * Calculate a new memory size estimate. This is somewhat expensive so we don't call this every time we run get().
+ * @return a new size estimate (bytes)
+ */
+ private long computeMemorySizeInBytes() {
+ double multiplier = getRBMSizeMultiplier((int) stats.size.count(), modulo);
+ return (long) (rbm.getSizeInBytes() * multiplier);
+ }
+
+ @Override
+ public long getMemorySizeCapInBytes() {
+ return stats.memSizeCapInBytes;
+ }
+
+ @Override
+ public boolean isFull() {
+ return stats.atCapacity.get();
+ }
+
+ @Override
+ public void regenerateStore(Integer[] newValues) {
+ rbm.clear();
+ collidedIntCounters = new HashMap<>();
+ removalSets = new HashMap<>();
+ stats.size = new CounterMetric();
+ stats.numAddAttempts = new CounterMetric();
+ stats.numCollisions = new CounterMetric();
+ stats.guaranteesNoFalseNegatives = true;
+ stats.numRemovalAttempts = new CounterMetric();
+ stats.numSuccessfulRemovals = new CounterMetric();
+ for (int i = 0; i < newValues.length; i++) {
+ if (newValues[i] != null) {
+ add(newValues[i]);
+ }
+ }
+ }
+
+ @Override
+ public void clear() {
+ regenerateStore(new Integer[] {});
+ }
+
+ public int getNumRemovalAttempts() {
+ return (int) stats.numRemovalAttempts.count();
+ }
+
+ public int getNumSuccessfulRemovals() {
+ return (int) stats.numSuccessfulRemovals.count();
+ }
+
+ public boolean valueHasHadCollision(Integer value) {
+ if (value == null) {
+ return false;
+ }
+ return collidedIntCounters.containsKey(transform(value));
+ }
+
+ CounterMetric getNumCollisionsForValue(int value) { // package private for testing
+ return collidedIntCounters.get(transform(value));
+ }
+
+ HashSet getRemovalSetForValue(int value) {
+ return removalSets.get(transform(value));
+ }
+
+ /**
+ * Run when the setting for keystore memory cap is updated.
+ * @param newMemSizeCap The new cap size.
+ */
+ protected void setMemSizeCap(ByteSizeValue newMemSizeCap) {
+ stats.memSizeCapInBytes = newMemSizeCap.getBytes();
+ mostRecentByteEstimate = getMemorySizeInBytes();
+ if (mostRecentByteEstimate > getMemorySizeCapInBytes()) {
+ stats.atCapacity.set(true);
+ }
+ }
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/tier/keystore/package-info.java b/server/src/main/java/org/opensearch/common/cache/tier/keystore/package-info.java
new file mode 100644
index 0000000000000..f3be16dfa3d32
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/tier/keystore/package-info.java
@@ -0,0 +1,9 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+/** Package for disk tier keystore. */
+package org.opensearch.common.cache.tier.keystore;
diff --git a/server/src/main/java/org/opensearch/common/metrics/CounterMetric.java b/server/src/main/java/org/opensearch/common/metrics/CounterMetric.java
index cb181840406a5..5c48c1f772ff0 100644
--- a/server/src/main/java/org/opensearch/common/metrics/CounterMetric.java
+++ b/server/src/main/java/org/opensearch/common/metrics/CounterMetric.java
@@ -62,5 +62,4 @@ public void dec(long n) {
public long count() {
return counter.sum();
}
-
}
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index c2c6effc3336f..c572c732aafb6 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -79,6 +79,9 @@
import org.opensearch.cluster.service.ClusterManagerTaskThrottler;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.annotation.PublicApi;
+import org.opensearch.common.cache.tier.DiskTierTookTimePolicy;
+import org.opensearch.common.cache.tier.EhCacheDiskCachingTier;
+import org.opensearch.common.cache.tier.keystore.RBMIntKeyLookupStore;
import org.opensearch.common.logging.Loggers;
import org.opensearch.common.network.NetworkModule;
import org.opensearch.common.network.NetworkService;
@@ -687,11 +690,11 @@ public void apply(Settings value, Settings current, Settings previous) {
RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING,
IndicesService.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING,
IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING,
+ IndicesService.CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING,
AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE,
CPUBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER_TRANSPORT_LAYER_MODE,
CPUBasedAdmissionControllerSettings.INDEXING_CPU_USAGE_LIMIT,
- CPUBasedAdmissionControllerSettings.SEARCH_CPU_USAGE_LIMIT,
- IndicesService.CLUSTER_RESTRICT_INDEX_REPLICATION_TYPE_SETTING
+ CPUBasedAdmissionControllerSettings.SEARCH_CPU_USAGE_LIMIT
)
)
);
@@ -716,6 +719,19 @@ public void apply(Settings value, Settings current, Settings previous) {
TelemetrySettings.METRICS_PUBLISH_INTERVAL_SETTING,
TelemetrySettings.TRACER_FEATURE_ENABLED_SETTING,
TelemetrySettings.METRICS_FEATURE_ENABLED_SETTING
+ ),
+ List.of(FeatureFlags.TIERED_CACHING),
+ List.of(
+ IndicesRequestCache.INDICES_CACHE_DISK_TIER_ENABLED,
+ IndicesRequestCache.INDICES_CACHE_DISK_TIER_SIZE,
+ RBMIntKeyLookupStore.INDICES_CACHE_KEYSTORE_SIZE,
+ EhCacheDiskCachingTier.REQUEST_CACHE_DISK_MIN_THREADS,
+ EhCacheDiskCachingTier.REQUEST_CACHE_DISK_MAX_THREADS,
+ EhCacheDiskCachingTier.REQUEST_CACHE_DISK_WRITE_CONCURRENCY,
+ EhCacheDiskCachingTier.REQUEST_CACHE_DISK_SEGMENTS,
+ DiskTierTookTimePolicy.DISK_TOOKTIME_THRESHOLD_SETTING,
+ IndicesService.INDICES_REQUEST_CACHE_DISK_CLEAN_THRESHOLD_SETTING,
+ IndicesService.INDICES_REQUEST_CACHE_DISK_CLEAN_INTERVAL_SETTING
)
);
}
diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java
index 387b0c9753574..04872ab53ea02 100644
--- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java
@@ -40,7 +40,8 @@ protected FeatureFlagSettings(
FeatureFlags.IDENTITY_SETTING,
FeatureFlags.CONCURRENT_SEGMENT_SEARCH_SETTING,
FeatureFlags.TELEMETRY_SETTING,
- FeatureFlags.DATETIME_FORMATTER_CACHING_SETTING
+ FeatureFlags.DATETIME_FORMATTER_CACHING_SETTING,
+ FeatureFlags.TIERED_CACHING_SETTING
)
)
);
diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java
index 4e9b417e3433b..1516cc3719576 100644
--- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java
+++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java
@@ -60,6 +60,11 @@ public class FeatureFlags {
*/
public static final String DATETIME_FORMATTER_CACHING = "opensearch.experimental.optimization.datetime_formatter_caching.enabled";
+ /**
+ * Gates the functionality of using a disk tier in the IndicesRequestCache.
+ */
+ public static final String TIERED_CACHING = "opensearch.experimental.feature.tiered_caching.enabled";
+
/**
* Should store the settings from opensearch.yml.
*/
@@ -122,4 +127,6 @@ public static boolean isEnabled(Setting featureFlag) {
true,
Property.NodeScope
);
+
+ public static final Setting TIERED_CACHING_SETTING = Setting.boolSetting(TIERED_CACHING, false, Property.NodeScope);
}
diff --git a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
index 24f68899c2ac7..a5ab4e8354aaa 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/RequestCacheStats.java
@@ -32,6 +32,10 @@
package org.opensearch.index.cache.request;
+import org.opensearch.OpenSearchException;
+import org.opensearch.Version;
+import org.opensearch.common.cache.tier.TierType;
+import org.opensearch.common.util.FeatureFlags;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
@@ -40,6 +44,8 @@
import org.opensearch.core.xcontent.XContentBuilder;
import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
/**
* Request for the query cache statistics
@@ -48,73 +54,171 @@
*/
public class RequestCacheStats implements Writeable, ToXContentFragment {
- private long memorySize;
- private long evictions;
- private long hitCount;
- private long missCount;
+ private Map defaultStatsMap = new HashMap<>() {
+ {
+ for (TierType tierType : TierType.values()) {
+ put(tierType.getStringValue(), new StatsHolder());
+ // Every possible tier type must have counters, even if they are disabled. Then the counters report 0
+ }
+ }
+ };
+
+ private Map tierSpecificStatsMap = new HashMap<>() {
+ {
+ put(TierType.ON_HEAP.getStringValue(), new ShardRequestCache.OnHeapStatsAccumulator());
+ put(TierType.DISK.getStringValue(), new ShardRequestCache.DiskStatsAccumulator());
+ }
+ };
public RequestCacheStats() {}
public RequestCacheStats(StreamInput in) throws IOException {
- memorySize = in.readVLong();
- evictions = in.readVLong();
- hitCount = in.readVLong();
- missCount = in.readVLong();
+ if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+ this.defaultStatsMap = in.readMap(StreamInput::readString, StatsHolder::new);
+ // Manually fill the tier-specific stats map
+ this.tierSpecificStatsMap = new HashMap<>();
+ tierSpecificStatsMap.put(TierType.ON_HEAP.getStringValue(), new ShardRequestCache.OnHeapStatsAccumulator(in));
+ tierSpecificStatsMap.put(TierType.DISK.getStringValue(), new ShardRequestCache.DiskStatsAccumulator(in));
+ } else {
+ // objects from earlier versions only contain on-heap info, and do not have entries info
+ long memorySize = in.readVLong();
+ long evictions = in.readVLong();
+ long hitCount = in.readVLong();
+ long missCount = in.readVLong();
+ this.defaultStatsMap.put(TierType.ON_HEAP.getStringValue(), new StatsHolder(memorySize, evictions, hitCount, missCount, 0));
+ }
+ checkTierSpecificMap();
}
- public RequestCacheStats(long memorySize, long evictions, long hitCount, long missCount) {
- this.memorySize = memorySize;
- this.evictions = evictions;
- this.hitCount = hitCount;
- this.missCount = missCount;
+ public RequestCacheStats(Map defaultMap, Map tierSpecificMap) {
+ // Create a RequestCacheStats with multiple tiers' statistics
+ // The input maps don't have to have all tiers statistics available
+ for (TierType tierType : defaultMap.keySet()) {
+ defaultStatsMap.put(tierType.getStringValue(), defaultMap.get(tierType));
+ }
+ for (TierType tierType : tierSpecificMap.keySet()) {
+ tierSpecificStatsMap.put(tierType.getStringValue(), tierSpecificMap.get(tierType));
+ }
+ checkTierSpecificMap();
}
public void add(RequestCacheStats stats) {
- this.memorySize += stats.memorySize;
- this.evictions += stats.evictions;
- this.hitCount += stats.hitCount;
- this.missCount += stats.missCount;
+ for (TierType tierType : TierType.values()) {
+ defaultStatsMap.get(tierType.getStringValue()).add(stats.defaultStatsMap.get(tierType.getStringValue()));
+ tierSpecificStatsMap.get(tierType.getStringValue()).add(stats.tierSpecificStatsMap.get(tierType.getStringValue()));
+ }
}
+ private StatsHolder getTierStats(TierType tierType) {
+ return defaultStatsMap.get(tierType.getStringValue());
+ }
+
+ ShardRequestCache.TierStatsAccumulator getTierSpecificStats(TierType tierType) { // pkg-private for testing
+ return tierSpecificStatsMap.get(tierType.getStringValue());
+ }
+
+ public ShardRequestCache.DiskStatsAccumulator getDiskSpecificStats() {
+ return (ShardRequestCache.DiskStatsAccumulator) tierSpecificStatsMap.get(TierType.DISK.getStringValue());
+ }
+
+ public long getMemorySizeInBytes(TierType tierType) {
+ return getTierStats(tierType).totalMetric.count();
+ }
+
+ public ByteSizeValue getMemorySize(TierType tierType) {
+ return new ByteSizeValue(getMemorySizeInBytes(tierType));
+ }
+
+ public long getEvictions(TierType tierType) {
+ return getTierStats(tierType).evictionsMetric.count();
+ }
+
+ public long getHitCount(TierType tierType) {
+ return getTierStats(tierType).hitCount.count();
+ }
+
+ public long getMissCount(TierType tierType) {
+ return getTierStats(tierType).missCount.count();
+ }
+
+ public long getEntries(TierType tierType) {
+ return getTierStats(tierType).entries.count();
+ }
+
+ // By default, return on-heap stats if no tier is specified
+
public long getMemorySizeInBytes() {
- return this.memorySize;
+ return getMemorySizeInBytes(TierType.ON_HEAP);
}
public ByteSizeValue getMemorySize() {
- return new ByteSizeValue(memorySize);
+ return getMemorySize(TierType.ON_HEAP);
}
public long getEvictions() {
- return this.evictions;
+ return getEvictions(TierType.ON_HEAP);
}
public long getHitCount() {
- return this.hitCount;
+ return getHitCount(TierType.ON_HEAP);
}
public long getMissCount() {
- return this.missCount;
+ return getMissCount(TierType.ON_HEAP);
+ }
+
+ public long getEntries() {
+ return getEntries(TierType.ON_HEAP);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
- out.writeVLong(memorySize);
- out.writeVLong(evictions);
- out.writeVLong(hitCount);
- out.writeVLong(missCount);
+ if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+ out.writeMap(this.defaultStatsMap, StreamOutput::writeString, (o, v) -> v.writeTo(o)); // ?
+ // Manually write tier-specific stats map
+ tierSpecificStatsMap.get(TierType.ON_HEAP.getStringValue()).writeTo(out);
+ tierSpecificStatsMap.get(TierType.DISK.getStringValue()).writeTo(out);
+ } else {
+ // Write only on-heap values, and don't write entries metric
+ StatsHolder heapStats = defaultStatsMap.get(TierType.ON_HEAP.getStringValue());
+ out.writeVLong(heapStats.getMemorySize());
+ out.writeVLong(heapStats.getEvictions());
+ out.writeVLong(heapStats.getHitCount());
+ out.writeVLong(heapStats.getMissCount());
+ }
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(Fields.REQUEST_CACHE_STATS);
- builder.humanReadableField(Fields.MEMORY_SIZE_IN_BYTES, Fields.MEMORY_SIZE, getMemorySize());
- builder.field(Fields.EVICTIONS, getEvictions());
- builder.field(Fields.HIT_COUNT, getHitCount());
- builder.field(Fields.MISS_COUNT, getMissCount());
+ // write on-heap stats outside of tiers object
+ getTierStats(TierType.ON_HEAP).toXContent(builder, params);
+ getTierSpecificStats(TierType.ON_HEAP).toXContent(builder, params);
+ if (FeatureFlags.isEnabled(FeatureFlags.TIERED_CACHING)) {
+ builder.startObject(Fields.TIERS);
+ for (TierType tierType : TierType.values()) { // fixed order
+ if (tierType != TierType.ON_HEAP) {
+ String tier = tierType.getStringValue();
+ builder.startObject(tier);
+ defaultStatsMap.get(tier).toXContent(builder, params);
+ tierSpecificStatsMap.get(tier).toXContent(builder, params);
+ builder.endObject();
+ }
+ }
+ builder.endObject();
+ }
builder.endObject();
return builder;
}
+ private void checkTierSpecificMap() {
+ for (TierType tierType : TierType.values()) {
+ if (tierSpecificStatsMap.get(tierType.getStringValue()) == null) {
+ throw new OpenSearchException("Missing TierStatsAccumulator for TierType " + tierType.getStringValue());
+ }
+ }
+ }
+
/**
* Fields used for parsing and toXContent
*
@@ -122,10 +226,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
*/
static final class Fields {
static final String REQUEST_CACHE_STATS = "request_cache";
+ static final String TIERS = "tiers";
static final String MEMORY_SIZE = "memory_size";
static final String MEMORY_SIZE_IN_BYTES = "memory_size_in_bytes";
static final String EVICTIONS = "evictions";
static final String HIT_COUNT = "hit_count";
static final String MISS_COUNT = "miss_count";
+ static final String ENTRIES = "entries";
}
}
diff --git a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
index efad437804bef..67f920850e949 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
@@ -33,10 +33,21 @@
package org.opensearch.index.cache.request;
import org.apache.lucene.util.Accountable;
+import org.opensearch.OpenSearchException;
+import org.opensearch.common.cache.tier.CacheValue;
+import org.opensearch.common.cache.tier.DiskTierRequestStats;
+import org.opensearch.common.cache.tier.OnHeapTierRequestStats;
+import org.opensearch.common.cache.tier.TierRequestStats;
import org.opensearch.common.cache.tier.TierType;
import org.opensearch.common.metrics.CounterMetric;
import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.ToXContentFragment;
+import org.opensearch.core.xcontent.XContentBuilder;
+import java.io.IOException;
import java.util.EnumMap;
/**
@@ -46,39 +57,47 @@
*/
public final class ShardRequestCache {
- private EnumMap statsHolder = new EnumMap<>(TierType.class);
+ // Holds stats common to all tiers
+ private final EnumMap defaultStatsHolder = new EnumMap<>(TierType.class);
+
+ // Holds tier-specific stats
+ private final EnumMap tierSpecificStatsHolder = new EnumMap<>(TierType.class);
public ShardRequestCache() {
+ tierSpecificStatsHolder.put(TierType.ON_HEAP, new OnHeapStatsAccumulator());
+ tierSpecificStatsHolder.put(TierType.DISK, new DiskStatsAccumulator());
for (TierType tierType : TierType.values()) {
- statsHolder.put(tierType, new StatsHolder());
+ defaultStatsHolder.put(tierType, new StatsHolder());
+ if (tierSpecificStatsHolder.get(tierType) == null) {
+ throw new OpenSearchException("Missing TierStatsAccumulator for TierType " + tierType.getStringValue());
+ }
}
}
public RequestCacheStats stats() {
- // TODO: Change RequestCacheStats to support disk tier stats.
- return new RequestCacheStats(
- statsHolder.get(TierType.ON_HEAP).totalMetric.count(),
- statsHolder.get(TierType.ON_HEAP).evictionsMetric.count(),
- statsHolder.get(TierType.ON_HEAP).hitCount.count(),
- statsHolder.get(TierType.ON_HEAP).missCount.count()
- );
+ return new RequestCacheStats(defaultStatsHolder, tierSpecificStatsHolder);
}
- public void onHit(TierType tierType) {
- statsHolder.get(tierType).hitCount.inc();
+ public void onHit(CacheValue cacheValue) {
+ TierType source = cacheValue.getSource();
+ defaultStatsHolder.get(source).hitCount.inc();
+ tierSpecificStatsHolder.get(source).addRequestStats(cacheValue.getStats());
}
- public void onMiss(TierType tierType) {
- statsHolder.get(tierType).missCount.inc();
+ public void onMiss(CacheValue cacheValue) {
+ TierType source = cacheValue.getSource();
+ defaultStatsHolder.get(source).missCount.inc();
+ tierSpecificStatsHolder.get(source).addRequestStats(cacheValue.getStats());
}
public void onCached(Accountable key, BytesReference value, TierType tierType) {
- statsHolder.get(tierType).totalMetric.inc(key.ramBytesUsed() + value.ramBytesUsed());
+ defaultStatsHolder.get(tierType).totalMetric.inc(key.ramBytesUsed() + value.ramBytesUsed());
+ defaultStatsHolder.get(tierType).entries.inc();
}
public void onRemoval(Accountable key, BytesReference value, boolean evicted, TierType tierType) {
if (evicted) {
- statsHolder.get(tierType).evictionsMetric.inc();
+ defaultStatsHolder.get(tierType).evictionsMetric.inc();
}
long dec = 0;
if (key != null) {
@@ -87,14 +106,118 @@ public void onRemoval(Accountable key, BytesReference value, boolean evicted, Ti
if (value != null) {
dec += value.ramBytesUsed();
}
- statsHolder.get(tierType).totalMetric.dec(dec);
+ defaultStatsHolder.get(tierType).totalMetric.dec(dec);
+ defaultStatsHolder.get(tierType).entries.dec();
+ }
+
+ /**
+ * An abstract whose extending classes accumulate tier-specific stats.
+ * All extending classes should provide a constructor like TierStatsAccumulator(StreamInput in)
+ * as well as a no-argument constructor
+ * @param The tier-specific implementation of TierRequestStats to use
+ */
+ static abstract class TierStatsAccumulator implements Writeable, ToXContentFragment {
+ /**
+ * Add new stats from a single request to this holder.
+ * @param stats The stats from a single request to add
+ */
+ public abstract void addRequestStats(S stats);
+
+ /**
+ * Add the stats from another TierStatsHolder to this TierStatsHolder.
+ * Used when combining stats across multiple shards.
+ * @param other The other TierStatsHolder.
+ */
+ public abstract void add(TierStatsAccumulator other);
+ }
+
+ /**
+ * This class accumulates on-heap-tier-specific stats for a single shard.
+ * For now, on-heap tier has no unique stats, but future stats would be added here.
+ */
+ public static class OnHeapStatsAccumulator extends TierStatsAccumulator {
+ OnHeapStatsAccumulator() {}
+
+ OnHeapStatsAccumulator(StreamInput in) {}
+
+ @Override
+ public void addRequestStats(OnHeapTierRequestStats stats) {}
+
+ @Override
+ public void add(TierStatsAccumulator other) {}
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {}
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ return builder;
+ }
}
- static class StatsHolder {
+ /**
+ * This class accumulates disk-tier-specific stats for a single shard.
+ */
+ public static class DiskStatsAccumulator extends TierStatsAccumulator {
+ final CounterMetric totalGetTime;
+ final CounterMetric totalDiskReaches; // Number of times a get() has actually reached the disk
+
+ public DiskStatsAccumulator() {
+ this.totalGetTime = new CounterMetric();
+ this.totalDiskReaches = new CounterMetric();
+ }
+
+ public DiskStatsAccumulator(long totalGetTime, long totalDiskReaches) {
+ this.totalGetTime = new CounterMetric();
+ this.totalGetTime.inc(totalGetTime);
+ this.totalDiskReaches = new CounterMetric();
+ this.totalDiskReaches.inc(totalDiskReaches);
+ }
- final CounterMetric evictionsMetric = new CounterMetric();
- final CounterMetric totalMetric = new CounterMetric();
- final CounterMetric hitCount = new CounterMetric();
- final CounterMetric missCount = new CounterMetric();
+ public DiskStatsAccumulator(StreamInput in) throws IOException {
+ this(in.readVLong(), in.readVLong());
+ }
+
+ public long getTotalGetTime() {
+ return totalGetTime.count();
+ }
+
+ public long getTotalDiskReaches() {
+ return totalDiskReaches.count();
+ }
+
+ @Override
+ public void addRequestStats(DiskTierRequestStats stats) {
+ if (stats.getRequestReachedDisk()) {
+ this.totalDiskReaches.inc();
+ this.totalGetTime.inc(stats.getRequestGetTimeNanos());
+ }
+ }
+
+ @Override
+ public void add(TierStatsAccumulator other) {
+ assert other.getClass() == DiskStatsAccumulator.class;
+ DiskStatsAccumulator castOther = (DiskStatsAccumulator) other;
+ this.totalDiskReaches.inc(castOther.totalDiskReaches.count());
+ this.totalGetTime.inc(castOther.totalGetTime.count());
+ }
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {
+ out.writeVLong(totalGetTime.count());
+ out.writeVLong(totalDiskReaches.count());
+ }
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.field(Fields.TOTAL_GET_TIME, getTotalGetTime());
+ builder.field(Fields.TOTAL_DISK_REACHES, getTotalDiskReaches());
+ return builder;
+ }
+
+ static final class Fields { // Used for field names in API response
+ static final String TOTAL_GET_TIME = "total_get_time_nanos";
+ static final String TOTAL_DISK_REACHES = "total_disk_reaches";
+ }
}
}
diff --git a/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java b/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java
new file mode 100644
index 0000000000000..74e610cf6d9cc
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/cache/request/StatsHolder.java
@@ -0,0 +1,114 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.cache.request;
+
+import org.opensearch.common.metrics.CounterMetric;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.core.xcontent.ToXContentFragment;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.io.Serializable;
+
+/**
+ * The basic StatsHolder class, which accumulates shard-level stats that are common across all tiers.
+ * Used in ShardRequestCache. Extending classes also handle tier-specific stats for each tier.
+ */
+public class StatsHolder implements Serializable, Writeable, ToXContentFragment {
+ final CounterMetric totalMetric;
+ final CounterMetric evictionsMetric;
+ final CounterMetric hitCount;
+ final CounterMetric missCount;
+ final CounterMetric entries;
+
+ public StatsHolder() {
+ this.totalMetric = new CounterMetric();
+ this.evictionsMetric = new CounterMetric();
+ this.hitCount = new CounterMetric();
+ this.missCount = new CounterMetric();
+ this.entries = new CounterMetric();
+ }
+
+ public StatsHolder(long memorySize, long evictions, long hitCount, long missCount, long entries) {
+ // Switched argument order to match RequestCacheStats
+ this.totalMetric = new CounterMetric();
+ this.totalMetric.inc(memorySize);
+ this.evictionsMetric = new CounterMetric();
+ this.evictionsMetric.inc(evictions);
+ this.hitCount = new CounterMetric();
+ this.hitCount.inc(hitCount);
+ this.missCount = new CounterMetric();
+ this.missCount.inc(missCount);
+ this.entries = new CounterMetric();
+ this.entries.inc(entries);
+ }
+
+ public StatsHolder(StreamInput in) throws IOException {
+ // Read and write the values of the counter metrics. They should always be positive
+ // This object is new, so we shouldn't need version checks for different behavior
+ this(in.readVLong(), in.readVLong(), in.readVLong(), in.readVLong(), in.readVLong());
+ // java forces us to do this in one line
+ // guaranteed to be evaluated in correct order (https://docs.oracle.com/javase/specs/jls/se7/html/jls-15.html#jls-15.7.4)
+ }
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {
+ out.writeVLong(totalMetric.count());
+ out.writeVLong(evictionsMetric.count());
+ out.writeVLong(hitCount.count());
+ out.writeVLong(missCount.count());
+ out.writeVLong(entries.count());
+ }
+
+ public void add(StatsHolder otherStats) {
+ // Add the argument's metrics to this object's metrics.
+ totalMetric.inc(otherStats.totalMetric.count());
+ evictionsMetric.inc(otherStats.evictionsMetric.count());
+ hitCount.inc(otherStats.hitCount.count());
+ missCount.inc(otherStats.missCount.count());
+ entries.inc(otherStats.entries.count());
+ }
+
+ public long getEvictions() {
+ return evictionsMetric.count();
+ }
+
+ public long getMemorySize() {
+ return totalMetric.count();
+ }
+
+ public long getHitCount() {
+ return hitCount.count();
+ }
+
+ public long getMissCount() {
+ return missCount.count();
+ }
+
+ public long getEntries() {
+ return entries.count();
+ }
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.humanReadableField(
+ RequestCacheStats.Fields.MEMORY_SIZE_IN_BYTES,
+ RequestCacheStats.Fields.MEMORY_SIZE,
+ new ByteSizeValue(getMemorySize())
+ );
+ builder.field(RequestCacheStats.Fields.EVICTIONS, getEvictions());
+ builder.field(RequestCacheStats.Fields.HIT_COUNT, getHitCount());
+ builder.field(RequestCacheStats.Fields.MISS_COUNT, getMissCount());
+ builder.field(RequestCacheStats.Fields.ENTRIES, getEntries());
+ return builder;
+ }
+}
diff --git a/server/src/main/java/org/opensearch/indices/AbstractIndexShardCacheEntity.java b/server/src/main/java/org/opensearch/indices/AbstractIndexShardCacheEntity.java
index d9c256b4b4a94..ac12e6bdf389a 100644
--- a/server/src/main/java/org/opensearch/indices/AbstractIndexShardCacheEntity.java
+++ b/server/src/main/java/org/opensearch/indices/AbstractIndexShardCacheEntity.java
@@ -34,6 +34,7 @@
import org.opensearch.common.cache.RemovalNotification;
import org.opensearch.common.cache.RemovalReason;
+import org.opensearch.common.cache.tier.CacheValue;
import org.opensearch.common.cache.tier.TierType;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.index.cache.request.ShardRequestCache;
@@ -57,13 +58,13 @@ public final void onCached(IndicesRequestCache.Key key, BytesReference value, Ti
}
@Override
- public final void onHit(TierType tierType) {
- stats().onHit(tierType);
+ public final void onHit(CacheValue cacheValue) {
+ stats().onHit(cacheValue);
}
@Override
- public final void onMiss(TierType tierType) {
- stats().onMiss(tierType);
+ public final void onMiss(CacheValue cacheValue) {
+ stats().onMiss(cacheValue);
}
@Override
diff --git a/server/src/main/java/org/opensearch/indices/IRCKeyWriteableSerializer.java b/server/src/main/java/org/opensearch/indices/IRCKeyWriteableSerializer.java
new file mode 100644
index 0000000000000..55d770804431f
--- /dev/null
+++ b/server/src/main/java/org/opensearch/indices/IRCKeyWriteableSerializer.java
@@ -0,0 +1,65 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.indices;
+
+import org.opensearch.OpenSearchException;
+import org.opensearch.common.cache.tier.Serializer;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.common.io.stream.BytesStreamInput;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * This class serializes the IndicesRequestCache.Key using its writeTo method.
+ */
+public class IRCKeyWriteableSerializer implements Serializer {
+
+ IndicesRequestCache irc;
+
+ public IRCKeyWriteableSerializer(IndicesRequestCache irc) {
+ this.irc = irc;
+ }
+
+ @Override
+ public byte[] serialize(IndicesRequestCache.Key object) {
+ try {
+ BytesStreamOutput os = new BytesStreamOutput();
+ object.writeTo(os);
+ return BytesReference.toBytes(os.bytes());
+ } catch (IOException e) {
+ throw new OpenSearchException(e);
+ }
+ }
+
+ @Override
+ public IndicesRequestCache.Key deserialize(byte[] bytes) {
+ if (bytes == null) {
+ return null;
+ }
+ try {
+ BytesStreamInput is = new BytesStreamInput(bytes, 0, bytes.length);
+ return irc.new Key(is);
+ } catch (IOException e) {
+ throw new OpenSearchException(e);
+ }
+ }
+
+ @Override
+ public boolean equals(IndicesRequestCache.Key object, byte[] bytes) {
+ // Deserialization is much slower than serialization for keys of order 1 KB,
+ // while time to serialize is fairly constant (per byte)
+ if (bytes.length < 5000) {
+ return Arrays.equals(serialize(object), bytes);
+ } else {
+ return object.equals(deserialize(bytes));
+ }
+ }
+}
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 2f06394ab7120..371664dd8a64e 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -40,6 +40,13 @@
import org.apache.lucene.util.RamUsageEstimator;
import org.opensearch.common.CheckedSupplier;
import org.opensearch.common.cache.RemovalNotification;
+import org.opensearch.common.cache.tier.BytesReferenceSerializer;
+import org.opensearch.common.cache.tier.CachePolicyInfoWrapper;
+import org.opensearch.common.cache.tier.CacheValue;
+import org.opensearch.common.cache.tier.DiskTierProvider;
+import org.opensearch.common.cache.tier.CachingTier;
+import org.opensearch.common.cache.tier.DiskTierTookTimePolicy;
+import org.opensearch.common.cache.tier.EhCacheDiskCachingTier;
import org.opensearch.common.cache.tier.OnHeapCachingTier;
import org.opensearch.common.cache.tier.OpenSearchOnHeapCache;
import org.opensearch.common.cache.tier.TierType;
@@ -48,26 +55,34 @@
import org.opensearch.common.cache.tier.TieredCacheService;
import org.opensearch.common.cache.tier.TieredCacheSpilloverStrategyService;
import org.opensearch.common.lucene.index.OpenSearchDirectoryReader;
+import org.opensearch.common.settings.ClusterSettings;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.util.FeatureFlags;
import org.opensearch.common.util.concurrent.ConcurrentCollections;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.common.io.stream.StreamInput;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.common.unit.ByteSizeUnit;
import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.index.shard.IndexShard;
+
import java.io.Closeable;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
-import java.util.Iterator;
+import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
/**
* The indices request cache allows to cache a shard level request stage responses, helping with improving
@@ -84,13 +99,14 @@
*
* @opensearch.internal
*/
-public final class IndicesRequestCache implements TieredCacheEventListener, Closeable {
+public final class IndicesRequestCache implements TieredCacheEventListener, Closeable, DiskTierProvider {
private static final Logger logger = LogManager.getLogger(IndicesRequestCache.class);
/**
* A setting to enable or disable request caching on an index level. Its dynamic by default
* since we are checking on the cluster state IndexMetadata always.
+ *
*/
public static final Setting INDEX_CACHE_REQUEST_ENABLED_SETTING = Setting.boolSetting(
"index.requests.cache.enable",
@@ -109,44 +125,123 @@ public final class IndicesRequestCache implements TieredCacheEventListener INDICES_CACHE_DISK_TIER_ENABLED = Setting.boolSetting(
+ "indices.requests.cache.tiered.disk.enable",
+ false,
+ Property.Dynamic,
+ Property.NodeScope
+ );
+
+ public static final Setting INDICES_CACHE_DISK_TIER_SIZE = Setting.memorySizeSetting(
+ "indices.requests.cache.tiered.disk.size",
+ new ByteSizeValue(100, ByteSizeUnit.MB),
+ Property.NodeScope
+ ); // TODO: This is 1% of the minimum EBS size for most EC2 instances. In future set to 1% of actual disk size if possible
+
+ public static final Setting INDICES_CACHE_KEYSTORE_SIZE = Setting.memorySizeSetting(
+ "indices.requests.cache.tiered.disk.keystore.size",
+ "0.05%", // 5% of INDICES_CACHE_QUERY_SIZE
+ Property.Dynamic,
+ Property.NodeScope
+ );
+
private final ConcurrentMap registeredClosedListeners = ConcurrentCollections.newConcurrentMap();
- private final Set keysToClean = ConcurrentCollections.newConcurrentSet();
+ private final Map keysToClean = ConcurrentCollections.newConcurrentMap();
+ // A map to keep track of the number of keys to be cleaned for a given ShardId and readerCacheKeyId
+ private final ConcurrentMap> diskCleanupKeyToCountMap = ConcurrentCollections.newConcurrentMap();
+ private final AtomicInteger staleKeysInDiskCount = new AtomicInteger(0);
private final ByteSizeValue size;
private final TimeValue expire;
private final TieredCacheService tieredCacheService;
-
private final IndicesService indicesService;
+ private final Settings settings;
+ private final ClusterSettings clusterSettings;
- IndicesRequestCache(Settings settings, IndicesService indicesService) {
+ IndicesRequestCache(Settings settings, IndicesService indicesService, ClusterSettings clusterSettings) {
this.size = INDICES_CACHE_QUERY_SIZE.get(settings);
this.expire = INDICES_CACHE_QUERY_EXPIRE.exists(settings) ? INDICES_CACHE_QUERY_EXPIRE.get(settings) : null;
long sizeInBytes = size.getBytes();
+ this.indicesService = indicesService;
+ this.settings = settings;
+ this.clusterSettings = clusterSettings;
// Initialize onHeap cache tier first.
OnHeapCachingTier openSearchOnHeapCache = new OpenSearchOnHeapCache.Builder().setWeigher(
(k, v) -> k.ramBytesUsed() + v.ramBytesUsed()
).setMaximumWeight(sizeInBytes).setExpireAfterAccess(expire).build();
- // Initialize tiered cache service. TODO: Enable Disk tier when tiered support is turned on.
- tieredCacheService = new TieredCacheSpilloverStrategyService.Builder().setOnHeapCachingTier(
- openSearchOnHeapCache
- ).setTieredCacheEventListener(this).build();
+ Function transformationFunction = (data) -> {
+ try {
+ return getPolicyInfo(data);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ };
+
+ // Initialize tiered cache service.
+ TieredCacheSpilloverStrategyService.Builder tieredCacheServiceBuilder =
+ new TieredCacheSpilloverStrategyService.Builder()
+ .setOnHeapCachingTier(openSearchOnHeapCache)
+ .setTieredCacheEventListener(this)
+ .setClusterSettings(clusterSettings);
+
+ if (FeatureFlags.isEnabled(FeatureFlags.TIERED_CACHING)) {
+ if (clusterSettings.get(INDICES_CACHE_DISK_TIER_ENABLED)) {
+ EhCacheDiskCachingTier ehcacheDiskTier = createNewDiskTier();
+ tieredCacheServiceBuilder.setOnDiskCachingTier(ehcacheDiskTier);
+ }
+ // if the disk tier dynamic setting is disabled, still provide the necessary things to turn it on
+ tieredCacheServiceBuilder.setDiskTierProvider(this);
+ tieredCacheServiceBuilder.withPolicy(new DiskTierTookTimePolicy(settings, clusterSettings, transformationFunction));
+ }
+ tieredCacheService = tieredCacheServiceBuilder.build();
+ }
+
+ // added for testing
+ IndicesRequestCache(
+ Settings settings,
+ ClusterSettings clusterSettings,
+ IndicesService indicesService,
+ TieredCacheService tieredCacheService
+ ) {
+ this.size = INDICES_CACHE_QUERY_SIZE.get(settings);
+ this.expire = INDICES_CACHE_QUERY_EXPIRE.exists(settings) ? INDICES_CACHE_QUERY_EXPIRE.get(settings) : null;
+ long sizeInBytes = size.getBytes();
this.indicesService = indicesService;
+ this.tieredCacheService = tieredCacheService;
+ this.settings = settings;
+ this.clusterSettings = clusterSettings;
}
@Override
public void close() {
tieredCacheService.invalidateAll();
+ if (tieredCacheService.getDiskCachingTier().isPresent()) {
+ tieredCacheService.getDiskCachingTier().get().close();
+ }
}
void clear(CacheEntity entity) {
- keysToClean.add(new CleanupKey(entity, null));
+ CleanupKey cleanupKey = new CleanupKey(entity, null);
+ keysToClean.put(cleanupKey, new CleanupStatus());
+ updateStaleKeysInDiskCount(cleanupKey);
cleanCache();
+ /*
+ this would be triggered by the cache clear API call
+ we need to make sure we clean the disk cache as well
+ hence passing threshold as 0
+ */
+ cleanDiskCache(0);
}
@Override
- public void onMiss(Key key, TierType tierType) {
- key.entity.onMiss(tierType);
+ public void onMiss(Key key, CacheValue cacheValue) {
+ key.entity.onMiss(cacheValue);
}
@Override
@@ -155,13 +250,84 @@ public void onRemoval(RemovalNotification notification) {
}
@Override
- public void onHit(Key key, BytesReference value, TierType tierType) {
- key.entity.onHit(tierType);
+ public void onHit(Key key, CacheValue cacheValue) {
+ key.entity.onHit(cacheValue);
}
@Override
public void onCached(Key key, BytesReference value, TierType tierType) {
key.entity.onCached(key, value, tierType);
+ updateDiskCleanupKeyToCountMap(new CleanupKey(key.entity, key.readerCacheKeyId), tierType);
+ }
+
+ /**
+ * Updates the diskCleanupKeyToCountMap with the given CleanupKey and TierType.
+ * If the TierType is not DISK, the method returns without making any changes.
+ * If the ShardId associated with the CleanupKey does not exist in the map, a new entry is created.
+ * The method increments the count of the CleanupKey in the map.
+ *
+ * Why use ShardID as the key ?
+ * CacheEntity mainly contains IndexShard, both of these classes do not override equals() and hashCode() methods.
+ * ShardID class properly overrides equals() and hashCode() methods.
+ * Therefore, to avoid modifying CacheEntity and IndexShard classes to override these methods, we use ShardID as the key.
+ *
+ * @param cleanupKey the CleanupKey to be updated in the map
+ * @param tierType the TierType of the CleanupKey
+ */
+ private void updateDiskCleanupKeyToCountMap(CleanupKey cleanupKey, TierType tierType) {
+ if(!tierType.equals(TierType.DISK)) {
+ return;
+ }
+ IndexShard indexShard = (IndexShard)cleanupKey.entity.getCacheIdentity();
+ if(indexShard == null) {
+ logger.warn("IndexShard is null for CleanupKey: {} while cleaning tier : {}",
+ cleanupKey.readerCacheKeyId, tierType.getStringValue());
+ return;
+ }
+ ShardId shardId = indexShard.shardId();
+
+ diskCleanupKeyToCountMap
+ .computeIfAbsent(shardId, k -> ConcurrentCollections.newConcurrentMap())
+ .merge(cleanupKey.readerCacheKeyId, 1, Integer::sum);
+ }
+
+ /**
+ * Updates the count of stale keys in the disk cache.
+ * This method is called when a CleanupKey is added to the keysToClean set.
+ * It increments the staleKeysInDiskCount by the count of the CleanupKey in the diskCleanupKeyToCountMap.
+ * If the CleanupKey's readerCacheKeyId is null or the CleanupKey's entity is not open, it increments the staleKeysInDiskCount
+ * by the total count of keys associated with the CleanupKey's ShardId in the diskCleanupKeyToCountMap and removes the ShardId from the map.
+ *
+ * @param cleanupKey the CleanupKey that has been marked for cleanup
+ */
+ private void updateStaleKeysInDiskCount(CleanupKey cleanupKey) {
+ IndexShard indexShard = (IndexShard) cleanupKey.entity.getCacheIdentity();
+ if(indexShard == null) {
+ logger.warn("IndexShard is null for CleanupKey: {}", cleanupKey.readerCacheKeyId);
+ return;
+ }
+ ShardId shardId = indexShard.shardId();
+
+ ConcurrentMap countMap = diskCleanupKeyToCountMap.get(shardId);
+ if (countMap == null) {
+ return;
+ }
+
+ if (cleanupKey.readerCacheKeyId == null) {
+ int totalSum = countMap.values().stream().mapToInt(Integer::intValue).sum();
+ staleKeysInDiskCount.addAndGet(totalSum);
+ diskCleanupKeyToCountMap.remove(shardId);
+ return;
+ }
+ Integer count = countMap.get(cleanupKey.readerCacheKeyId);
+ if (count == null) {
+ return;
+ }
+ staleKeysInDiskCount.addAndGet(count);
+ countMap.remove(cleanupKey.readerCacheKeyId);
+ if (countMap.isEmpty()) {
+ diskCleanupKeyToCountMap.remove(shardId);
+ }
}
BytesReference getOrCompute(
@@ -181,7 +347,7 @@ BytesReference getOrCompute(
Loader cacheLoader = new Loader(cacheEntity, loader);
BytesReference value = tieredCacheService.computeIfAbsent(key, cacheLoader);
if (cacheLoader.isLoaded()) {
- // see if its the first time we see this reader, and make sure to register a cleanup key
+ // see if it's the first time we see this reader, and make sure to register a cleanup key
CleanupKey cleanupKey = new CleanupKey(cacheEntity, readerCacheKeyId);
if (!registeredClosedListeners.containsKey(cleanupKey)) {
Boolean previous = registeredClosedListeners.putIfAbsent(cleanupKey, Boolean.TRUE);
@@ -195,9 +361,10 @@ BytesReference getOrCompute(
/**
* Invalidates the given the cache entry for the given key and it's context
+ *
* @param cacheEntity the cache entity to invalidate for
- * @param reader the reader to invalidate the cache entry for
- * @param cacheKey the cache key to invalidate
+ * @param reader the reader to invalidate the cache entry for
+ * @param cacheKey the cache key to invalidate
*/
void invalidate(CacheEntity cacheEntity, DirectoryReader reader, BytesReference cacheKey) {
assert reader.getReaderCacheHelper() != null;
@@ -209,13 +376,18 @@ void invalidate(CacheEntity cacheEntity, DirectoryReader reader, BytesReference
tieredCacheService.invalidate(new Key(cacheEntity, cacheKey, readerCacheKeyId));
}
+ public static CachePolicyInfoWrapper getPolicyInfo(BytesReference data) throws IOException {
+ // Reads the policy info corresponding to this QSR, written in IndicesService$loadIntoContext,
+ // without having to create a potentially large short-lived QSR object just for this purpose
+ return new CachePolicyInfoWrapper(data.streamInput());
+ }
+
/**
* Loader for the request cache
*
* @opensearch.internal
*/
private static class Loader implements TieredCacheLoader {
-
private final CacheEntity entity;
private final CheckedSupplier loader;
private boolean loaded;
@@ -237,6 +409,11 @@ public BytesReference load(Key key) throws Exception {
}
}
+ public class CleanupStatus {
+ public boolean cleanedInHeap;
+ public boolean cleanedOnDisk;
+ }
+
/**
* Basic interface to make this cache testable.
*/
@@ -262,12 +439,12 @@ interface CacheEntity extends Accountable, Writeable {
/**
* Called each time this entity has a cache hit.
*/
- void onHit(TierType tierType);
+ void onHit(CacheValue cacheValue);
/**
* Called each time this entity has a cache miss.
*/
- void onMiss(TierType tierType);
+ void onMiss(CacheValue cacheValue);
/**
* Called when this entity instance is removed
@@ -351,7 +528,8 @@ private CleanupKey(CacheEntity entity, String readerCacheKeyId) {
public void onClose(IndexReader.CacheKey cacheKey) {
Boolean remove = registeredClosedListeners.remove(this);
if (remove != null) {
- keysToClean.add(this);
+ keysToClean.put(this, new CleanupStatus());
+ updateStaleKeysInDiskCount(this);
}
}
@@ -379,33 +557,130 @@ public int hashCode() {
* Logic to clean up in-memory cache.
*/
synchronized void cleanCache() {
+ cleanCache(TierType.ON_HEAP);
+ tieredCacheService.getOnHeapCachingTier().refresh();
+ }
+
+ /**
+ * Logic to clean up disk based cache.
+ *
+ * TODO: cleanDiskCache is very specific to disk caching tier. We can refactor this to be more generic.
+ */
+ synchronized void cleanDiskCache(double diskCachesCleanThresholdPercent) {
+ if (!canSkipDiskCacheCleanup(diskCachesCleanThresholdPercent)) {
+ cleanCache(TierType.DISK);
+ }
+ }
+
+ private synchronized void cleanCache(TierType cacheType) {
final Set currentKeysToClean = new HashSet<>();
final Set