diff --git a/developers/weaviate/concepts/replication-architecture/consistency.md b/developers/weaviate/concepts/replication-architecture/consistency.md index 8ac241ad8..a107d2839 100644 --- a/developers/weaviate/concepts/replication-architecture/consistency.md +++ b/developers/weaviate/concepts/replication-architecture/consistency.md @@ -51,6 +51,21 @@ A clean (without fails) execution has two phases: +### Collection definition requests in queries + +:::info Added in `v1.27.10`, `v1.28.4` +::: + +Some queries require the collection definition. Prior to the introduction of this feature, every such query led to the local (requesting) node to fetch the collection definition from the leader node. This meant that the definition was strongly consistent, but it could lead to additional traffic and load. + +Where available, the `COLLECTION_RETRIEVAL_STRATEGY` [environment variable](../../config-refs/env-vars.md#multi-node-instances) can be set to `LeaderOnly`, `LocalOnly`, or `LeaderOnMismatch`. + +- `LeaderOnly` (default): Always requests the definition from the leader node. This is the most consistent behavior but can lead to higher intra-cluster traffic. +- `LocalOnly`: Always use the local definition; leading to eventually consistent behavior while reducing intra-cluster traffic. +- `LeaderOnMismatch`: Checks if the local definition is outdated, and requests the definition if necessary. Balances consistency and intra-cluster traffic. + +The default behavior is `LeaderOnly` to achieve strong consistency. However, `LocalOnly` and `LeaderOnMismatch` can be used to reduce intra-cluster traffic according to the desired consistency level. + ## Data objects Weaviate uses two-phase commits for objects, adjusted for the consistency level. For example for a `QUORUM` write (see below), if there are 5 nodes, 3 requests will be sent out, each of them using a 2-phase commit under the hood. diff --git a/developers/weaviate/config-refs/env-vars.md b/developers/weaviate/config-refs/env-vars.md index d9f3918fe..c37f4589d 100644 --- a/developers/weaviate/config-refs/env-vars.md +++ b/developers/weaviate/config-refs/env-vars.md @@ -113,6 +113,7 @@ default hostname has changed and a single node cluster believes there are suppos | `CLUSTER_HOSTNAME` | Hostname of a node. Always set this value if the default OS hostname might change over time. | `string` | `node1` | | `CLUSTER_JOIN` | The service name of the "founding" member node in a cluster setup | `string` | `weaviate-node-1:7100` | | `HNSW_STARTUP_WAIT_FOR_VECTOR_CACHE` | If `true`, vector cache prefill is synchronous when a node starts. The node reports ready to serve when the cache is hot. Defaults to `false`. Added in 1.24.20 and 1.25.5. | `boolean` | `false` | +| `COLLECTION_RETRIEVAL_STRATEGY`| Set collection definition retrieval behavior for a data request.

([Read more](../concepts/replication-architecture/consistency.md#collection-definition-requests-in-queries)) (Added in `v1.27.10`, `v1.28.4`) | `string` | `LeaderOnly` | | `RAFT_ENABLE_FQDN_RESOLVER` | If `true`, use DNS lookup instead of memberlist lookup for Raft. Added in `v1.25.15`. ([Read more](../concepts/cluster.md#fqdn-for-node-discovery)) | `boolean` | `true` | | `RAFT_FQDN_RESOLVER_TLD` | The top-level domain to use for DNS lookup, in `[node-id].[tld]` format. Added in `v1.25.15`. ([Read more](../concepts/cluster.md#fqdn-for-node-discovery)) | `string` | `example.com` | | `RAFT_BOOTSTRAP_EXPECT` | The number of voter notes at bootstrapping time | `string - number` | `1` | diff --git a/yarn.lock b/yarn.lock index b36cd1b2f..f3ae0a82b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4438,6 +4438,13 @@ resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.4.tgz#0b92dcc0cc1c81f6f306a381f28e31b1a56536e9" integrity sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA== +"@types/adm-zip@^0.5.7": + version "0.5.7" + resolved "https://registry.yarnpkg.com/@types/adm-zip/-/adm-zip-0.5.7.tgz#eec10b6f717d3948beb64aca0abebc4b344ac7e9" + integrity sha512-DNEs/QvmyRLurdQPChqq0Md4zGvPwHerAJYWk9l2jCbD1VPpnzRJorOdiq4zsw09NFbYnhfsoEhWtxIzXpn2yw== + dependencies: + "@types/node" "*" + "@types/body-parser@*": version "1.19.5" resolved "https://registry.yarnpkg.com/@types/body-parser/-/body-parser-1.19.5.tgz#04ce9a3b677dc8bd681a17da1ab9835dc9d3ede4" @@ -5354,7 +5361,7 @@ abbrev@1: resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== -abort-controller-x@^0.4.0: +abort-controller-x@^0.4.0, abort-controller-x@^0.4.3: version "0.4.3" resolved "https://registry.yarnpkg.com/abort-controller-x/-/abort-controller-x-0.4.3.tgz#ff269788386fabd58a7b6eeaafcb6cf55c2958e0" integrity sha512-VtUwTNU8fpMwvWGn4xE93ywbogTYsuT+AUxAXOeelbXuQVIwNmC5YLeho9sH4vZ4ITW8414TTAOG1nW6uIVHCA== @@ -5414,6 +5421,11 @@ adm-zip@^0.5.10: resolved "https://registry.yarnpkg.com/adm-zip/-/adm-zip-0.5.13.tgz#e2fbd87d0bb75603ccfdcb03a57a57c4831d9758" integrity sha512-4U51tTl9J8UVEcuKGr6zRzY95tWoAa9l+ureGBNmsfleszjZblm5NyEEL/ZQxkhi86co5mZhSvL2T7gkZ6feYQ== +adm-zip@^0.5.16: + version "0.5.16" + resolved "https://registry.yarnpkg.com/adm-zip/-/adm-zip-0.5.16.tgz#0b5e4c779f07dedea5805cdccb1147071d94a909" + integrity sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ== + ag-charts-types@10.3.3: version "10.3.3" resolved "https://registry.yarnpkg.com/ag-charts-types/-/ag-charts-types-10.3.3.tgz#042c747f1247af827c29ccfdd0890d7b9adcaae9" @@ -18399,16 +18411,18 @@ wcwidth@^1.0.1: dependencies: defaults "^1.0.3" -weaviate-client@^3.2.3: - version "3.2.4" - resolved "https://registry.yarnpkg.com/weaviate-client/-/weaviate-client-3.2.4.tgz#9ba1b4016e9eec45a4f8c677e352f4e33423abc4" - integrity sha512-Y6KnrPQqUDK1hf15aH4/ib3FJwCXbUE8yGGtK8Pq8REbjgHCGXs+2cDr0XbihTbe75yXmBZ5UIgbIusgabgieQ== +weaviate-client@^3.3.4: + version "3.3.4" + resolved "https://registry.yarnpkg.com/weaviate-client/-/weaviate-client-3.3.4.tgz#f3648ee0a3a51f1bf3906ad5e84a6b32f2fe4716" + integrity sha512-aHZwgJR+HHV/xmLwFJ57Ny1OuhE7EC1sheHZueJoOKfn+iov031XwuZ5oH6WIYQY0tGqvs6hTmXwgdy51ZuXkg== dependencies: + abort-controller-x "^0.4.3" graphql "^16.9.0" graphql-request "^6.1.0" long "^5.2.3" nice-grpc "^2.1.10" nice-grpc-client-middleware-retry "^3.1.9" + nice-grpc-common "^2.0.2" uuid "^9.0.1" weaviate-ts-client@^1.3.3: