diff --git a/basho_builds.yml b/basho_builds.yml new file mode 100644 index 000000000..f6e528f6b --- /dev/null +++ b/basho_builds.yml @@ -0,0 +1,19 @@ +docker_rt_riak_test_config: + - version: current + product_version: current + product: "riak{% if ee == 'true' %}_ee{% endif %}" + - version: previous + product_version: 2.2.0 + product: "riak{% if ee == 'true' %}_ee{% endif %}" + - version: legacy + product_version: 2.0.9 + product: "riak{% if ee == 'true' %}_ee{% endif %}" + - version: 2.0.2 + product_version: 2.0.2 + product: "riak{% if ee == 'true' %}_ee{% endif %}" + - version: 2.0.4 + product_version: 2.0.4 + product: "riak{% if ee == 'true' %}_ee{% endif %}" + - version: 2.0.5 + product_version: 2.0.5 + product: "riak{% if ee == 'true' %}_ee{% endif %}" diff --git a/bin/rtdev-install.sh b/bin/rtdev-install.sh index b367636da..9df8e1329 100755 --- a/bin/rtdev-install.sh +++ b/bin/rtdev-install.sh @@ -46,15 +46,17 @@ cd $RT_DEST_DIR echo " - Reinitializing git state" -## Some versions of git and/or OS require these fields -HAS_LOCAL=$(git config --local 2>&1 | grep unknown) -if [ -z "$HAS_LOCAL" ]; then - git config --local user.name "Riak Test" - git config --local user.email "dev@basho.com" - git config --local core.autocrlf input - git config --local core.safecrlf false - git config --local core.filemode true -fi +# Set the path to the root of the working tree. This prevents git to +# ascend the directory tree (and possibly mess with other +# repositories). That is, changes we are going to make with `git +# config` will only affect the rt repo we have just created. +GIT_WORK_TREE="$RT_DEST_DIR" +git config user.name "Riak Test" +git config user.email "dev@basho.com" +git config core.autocrlf input +git config core.safecrlf false +git config core.filemode true + git add --all --force . git commit -a -m "riak_test init" --amend > /dev/null diff --git a/bin/rtdev-setup-releases.sh b/bin/rtdev-setup-releases.sh index 4b958b417..ff3654213 100755 --- a/bin/rtdev-setup-releases.sh +++ b/bin/rtdev-setup-releases.sh @@ -35,18 +35,19 @@ else echo "No devdirs found. Not copying any releases." fi -cd $RT_DEST_DIR -git init - -## Some versions of git and/or OS require these fields -HAS_LOCAL=$(git config --local 2>&1 | grep unknown) -if [ -z "$HAS_LOCAL" ]; then - git config --local user.name "Riak Test" - git config --local user.email "dev@basho.com" - git config --local core.autocrlf input - git config --local core.safecrlf false - git config --local core.filemode true -fi +cd "$RT_DEST_DIR" +git init + +# Set the path to the root of the working tree. This prevents git to +# ascend the directory tree (and possibly mess with other +# repositories). That is, changes we are going to make with `git +# config` will only affect the rt repo we have just created. +GIT_WORK_TREE="$RT_DEST_DIR" +git config user.name "Riak Test" +git config user.email "dev@basho.com" +git config core.autocrlf input +git config core.safecrlf false +git config core.filemode true ## this prevents priv/*.so files from being deleted by git clean -fd ## (the latter is executed in rtdev-current.sh): @@ -54,5 +55,4 @@ echo "priv/" >.gitignore git add --all --force . git commit -a -m "riak_test init" > /dev/null - echo " - Successfully completed initial git commit of $RT_DEST_DIR" diff --git a/intercepts/riak_kv_bitcask_backend_intercepts.erl b/intercepts/riak_kv_bitcask_backend_intercepts.erl index dc42cbeb3..a5cc31b96 100644 --- a/intercepts/riak_kv_bitcask_backend_intercepts.erl +++ b/intercepts/riak_kv_bitcask_backend_intercepts.erl @@ -48,3 +48,11 @@ corrupting_get(Bucket, Key, ModState) -> corrupt_binary(O) -> crypto:rand_bytes(byte_size(O)). + +always_corrupt_get(Bucket, Key, ModState) -> + case ?M:get_orig(Bucket, Key, ModState) of + {ok, BinVal0, UpdModState} -> + BinVal = corrupt_binary(BinVal0), + {ok, BinVal, UpdModState}; + Else -> Else + end. diff --git a/intercepts/riak_kv_qry_buffers_intercepts.erl b/intercepts/riak_kv_qry_buffers_intercepts.erl index f54c2e994..7bd01d75b 100644 --- a/intercepts/riak_kv_qry_buffers_intercepts.erl +++ b/intercepts/riak_kv_qry_buffers_intercepts.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2016 Basho Technologies, Inc. +%% Copyright (c) 2016, 2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file diff --git a/intercepts/yz_noop_extractor_intercepts.erl b/intercepts/yz_noop_extractor_intercepts.erl deleted file mode 100644 index ac7146289..000000000 --- a/intercepts/yz_noop_extractor_intercepts.erl +++ /dev/null @@ -1,44 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2015 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%%------------------------------------------------------------------- - -%% Example from: -%% http://docs.basho.com/riak/latest/dev/search/custom-extractors/#An-Example-Custom-Extractor - --module(yz_noop_extractor_intercepts). --compile(export_all). --include("intercept.hrl"). - -extract_httpheader(Value) -> - extract_httpheader(Value, []). - -extract_httpheader(Value, _Opts) -> - {ok, - {http_request, - Method, - {absoluteURI, http, Host, undefined, Uri}, - _Version}, - _Rest} = erlang:decode_packet(http, Value, []), - [{method, Method}, {host, list_to_binary(Host)}, {uri, list_to_binary(Uri)}]. - -extract_non_unicode_data(Value) -> - extract_non_unicode_data(Value, []). - -extract_non_unicode_data(_Value, _Opts) -> - [{blob, <<9147374713>>}]. diff --git a/intercepts/yz_solrq_helper_intercepts.erl b/intercepts/yz_solrq_helper_intercepts.erl deleted file mode 100644 index 926929d0f..000000000 --- a/intercepts/yz_solrq_helper_intercepts.erl +++ /dev/null @@ -1,48 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2015 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%%------------------------------------------------------------------- --module(yz_solrq_helper_intercepts). --compile(export_all). - --include("intercept.hrl"). - --define(M, yz_solrq_helper_orig). - -handle_get_ops_for_no_sibling_deletes(LI, P, Obj) -> - Lookup = ets:lookup(intercepts_tab, del_put), - case Lookup of - [] -> original_get_ops_for_no_sibling_deletes(LI, P, Obj); - _ -> - case proplists:get_value(del_put, Lookup) of - 0 -> - error_logger:info_msg( - "Delete operation intercepted for BKey ~p", - [{riak_object:bucket(Obj), riak_object:key(Obj)}]), - ets:update_counter(intercepts_tab, del_put, 1), - []; - _ -> - original_get_ops_for_no_sibling_deletes(LI, P, Obj) - end - end. - -original_get_ops_for_no_sibling_deletes(LI, P, Obj) -> - error_logger:info_msg( - "Delete operation original for BKey ~p", - [{riak_object:bucket(Obj), riak_object:key(Obj)}]), - ?M:get_ops_for_no_sibling_deletes_orig(LI, P, Obj). diff --git a/rebar.config.lock b/rebar.config.lock index 8916829ad..19e6a9ad0 100644 --- a/rebar.config.lock +++ b/rebar.config.lock @@ -25,13 +25,13 @@ "51b79c5b05eb04640e13131f06ef96561b96ba8e"}}, {hamcrest,".*", {git,"https://github.com/basho/hamcrest-erlang.git", - "98bc7aa19ea081478c816824aa05fc5a48acae66"}}, + "ad3dbab419762fc2d5821abb88b989da006b85c6"}}, {riak_pb,".*", {git,"https://github.com/basho/riak_pb", - "6c67dc58251e1a1c29b2e0c444b1e9c9aa2d4a24"}}, + "08771aba2ce4935b715d32d1b92555efdc3da994"}}, {riakc,".*", {git,"https://github.com/basho/riak-erlang-client", - "87958bdf1926d752fb1319b6788330176f6dd591"}}, + "9167ad627348d4995ef60c93c63ce39a16eb9735"}}, {ibrowse,".*", {git,"https://github.com/basho/ibrowse.git", "b28542d1e326ba44bcfaf7fd6d3c7f8761d20f08"}}, diff --git a/src/rt.erl b/src/rt.erl index 507fa50f6..1cc104f24 100644 --- a/src/rt.erl +++ b/src/rt.erl @@ -561,6 +561,20 @@ heal({_NewCookie, OldCookie, P1, P2}) -> {_GN, []} = rpc:sbcast(Cluster, riak_core_node_watcher, broadcast), ok. +%% @doc heal the partition created by call to partition/2, but if some +%% node in P1 is down, just skip it, rather than failing. Returns {ok, +%% list(node())} where the list is those nodes down and therefore not +%% healed/reconnected. +heal_upnodes({_NewCookie, OldCookie, P1, P2}) -> + %% set OldCookie on UP P1 Nodes + Res = [{N, rpc:call(N, erlang, set_cookie, [N, OldCookie])} || N <- P1], + UpForReconnect = [N || {N, true} <- Res], + DownForReconnect = [N || {N, RPC} <- Res, RPC /= true], + Cluster = UpForReconnect ++ P2, + wait_until_connected(Cluster), + {_GN, []} = rpc:sbcast(Cluster, riak_core_node_watcher, broadcast), + {ok, DownForReconnect}. + %% @doc Spawn `Cmd' on the machine running the test harness spawn_cmd(Cmd) -> ?HARNESS:spawn_cmd(Cmd). @@ -778,6 +792,17 @@ wait_until_transfers_complete([Node0|_]) -> ?assertEqual(ok, wait_until(Node0, F)), ok. +%% @doc Waits until hinted handoffs from `Node0' are complete +wait_until_node_handoffs_complete(Node0) -> + lager:info("Wait until Node's transfers complete ~p", [Node0]), + F = fun(Node) -> + Handoffs = rpc:call(Node, riak_core_handoff_manager, status, [{direction, outbound}]), + lager:info("Handoffs: ~p", [Handoffs]), + Handoffs =:= [] + end, + ?assertEqual(ok, wait_until(Node0, F)), + ok. + wait_for_service(Node, Services) when is_list(Services) -> F = fun(N) -> case rpc:call(N, riak_core_node_watcher, services, [N]) of diff --git a/src/yokozuna_rt.erl b/src/yokozuna_rt.erl deleted file mode 100644 index 21a492317..000000000 --- a/src/yokozuna_rt.erl +++ /dev/null @@ -1,563 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2015 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%%------------------------------------------------------------------- --module(yokozuna_rt). - --include_lib("eunit/include/eunit.hrl"). --include("yokozuna_rt.hrl"). - --export([brutal_kill_remove_index_dirs/3, - check_exists/2, - clear_trees/1, - commit/2, - drain_solrqs/1, - expire_trees/1, - gen_keys/1, - host_entries/1, - create_indexed_bucket_type/3, - create_indexed_bucket_type/4, - override_schema/5, - remove_index_dirs/3, - rolling_upgrade/2, - search/4, - search/5, - search_expect/5, - search_expect/6, - search_expect/7, - assert_search/6, - verify_num_found_query/3, - wait_for_aae/1, - wait_for_full_exchange_round/2, - wait_for_index/2, - wait_for_schema/2, - wait_for_schema/3, - wait_until/2, - write_data/5, - write_data/6, - http/4, - http/5, - http/6]). - --type host() :: string(). --type portnum() :: integer(). --type count() :: non_neg_integer(). --type json_string() :: atom | string() | binary(). --type search_type() :: solr | yokozuna. --type method() :: get | post | head | options | put | delete | trace | - mkcol | propfind | proppatch | lock | unlock | move | copy. --type response() :: {ok, string(), [{string(), string()}], string()|binary()} | - {error, term()}. --type cluster() :: [node()]. - - --define(FMT(S, Args), lists:flatten(io_lib:format(S, Args))). --define(SOFTCOMMIT, 1000). - --spec host_entries(rt:conn_info()) -> [{host(), portnum()}]. -host_entries(ClusterConnInfo) -> - [riak_http(I) || {_,I} <- ClusterConnInfo]. - -%% @doc Generate `SeqMax' keys. Yokozuna supports only UTF-8 compatible keys. --spec gen_keys(pos_integer()) -> list(). -gen_keys(SeqMax) -> - [<> || N <- lists:seq(1, SeqMax), - not lists:any( - fun(E) -> E > 127 end, - binary_to_list(<>))]. - -%% @doc Write `Keys' via the PB inteface to a `Bucket' and have them -%% searchable in an `Index'. --spec write_data([node()], pid(), index_name(), bucket(), [binary()]) -> ok. -write_data(Cluster, Pid, Index, Bucket, Keys) -> - riakc_pb_socket:set_options(Pid, [queue_if_disconnected]), - - create_and_set_index(Cluster, Pid, Bucket, Index), - yokozuna_rt:commit(Cluster, Index), - - %% Write keys - lager:info("Writing ~p keys", [length(Keys)]), - [ok = rt:pbc_write(Pid, Bucket, Key, Key, "text/plain") || Key <- Keys], - ok. - --spec write_data([node()], pid(), index_name(), {schema_name(), raw_schema()}, - bucket(), [binary()]) -> ok. -write_data(Cluster, Pid, Index, {SchemaName, SchemaData}, - Bucket, Keys) -> - riakc_pb_socket:set_options(Pid, [queue_if_disconnected]), - - riakc_pb_socket:create_search_schema(Pid, SchemaName, SchemaData), - - create_and_set_index(Cluster, Pid, Bucket, Index, SchemaName), - yokozuna_rt:commit(Cluster, Index), - - %% Write keys - lager:info("Writing ~p keys", [length(Keys)]), - [ok = rt:pbc_write(Pid, Bucket, Key, Key, "text/plain") || Key <- Keys], - ok. - -%% @doc Peform a rolling upgrade of the `Cluster' to a different `Version' based -%% on current | previous | legacy. --spec rolling_upgrade([node()], current | previous | legacy) -> ok. -rolling_upgrade(Cluster, Version) -> - rolling_upgrade(Cluster, Version, same, [riak_kv, yokozuna]). --spec rolling_upgrade([node()], current | previous | legacy, [term()] | same, [atom()]) -> ok. -rolling_upgrade(Cluster, Version, UpgradeConfig, WaitForServices) when is_list(Cluster) -> - lager:info("Perform rolling upgrade on cluster ~p", [Cluster]), - [rolling_upgrade(Node, Version, UpgradeConfig, WaitForServices) || Node <- Cluster], - ok; -rolling_upgrade(Node, Version, UpgradeConfig, WaitForServices) -> - rt:upgrade(Node, Version, UpgradeConfig), - [rt:wait_for_service(Node, Service) || Service <- WaitForServices], - ok. - -%% @doc Use AAE status to verify that exchange has occurred for all -%% partitions since the time this function was invoked. --spec wait_for_aae([node()]) -> ok. -wait_for_aae(Cluster) -> - lager:info("Wait for AAE to migrate/repair indexes"), - wait_for_all_trees(Cluster), - wait_for_full_exchange_round(Cluster, erlang:now()), - ok. - -%% @doc Wait for all AAE trees to be built. --spec wait_for_all_trees([node()]) -> ok. -wait_for_all_trees(Cluster) -> - F = fun(Node) -> - lager:info("Check if all trees built for node ~p", [Node]), - Info = rpc:call(Node, yz_kv, compute_tree_info, []), - NotBuilt = [X || {_,undefined}=X <- Info], - NotBuilt == [] - end, - wait_until(Cluster, F), - ok. - -%% @doc Wait for a full exchange round since `Timestamp'. This means -%% that all `{Idx,N}' for all partitions must have exchanged after -%% `Timestamp'. --spec wait_for_full_exchange_round([node()], os:now()) -> ok. -wait_for_full_exchange_round(Cluster, Timestamp) -> - lager:info("wait for full AAE exchange round on cluster ~p", [Cluster]), - MoreRecent = - fun({_Idx, _, undefined, _RepairStats}) -> - false; - ({_Idx, _, AllExchangedTime, _RepairStats}) -> - AllExchangedTime > Timestamp - end, - AllExchanged = - fun(Node) -> - Exchanges = rpc:call(Node, yz_kv, compute_exchange_info, []), - {_Recent, WaitingFor1} = lists:partition(MoreRecent, Exchanges), - WaitingFor2 = [element(1,X) || X <- WaitingFor1], - lager:info("Still waiting for AAE of ~p ~p", [Node, WaitingFor2]), - [] == WaitingFor2 - end, - wait_until(Cluster, AllExchanged), - ok. - -%% @doc Wait for index creation. This is to handle *legacy* versions of yokozuna -%% in upgrade tests --spec wait_for_index(list(), index_name()) -> ok. -wait_for_index(Cluster, Index) -> - IsIndexUp = - fun(Node) -> - lager:info("Waiting for index ~s to be avaiable on node ~p", - [Index, Node]), - rpc:call(Node, yz_index, exists, [Index]) - end, - wait_until(Cluster, IsIndexUp), - ok. - -%% @see wait_for_schema/3 -wait_for_schema(Cluster, Name) -> - wait_for_schema(Cluster, Name, ignore). - -%% @doc Wait for the schema `Name' to be read by all nodes in -%% `Cluster' before returning. If `Content' is binary data when -%% verify the schema bytes exactly match `Content'. --spec wait_for_schema([node()], schema_name(), ignore | raw_schema()) -> ok. -wait_for_schema(Cluster, Name, Content) -> - F = fun(Node) -> - lager:info("Attempt to read schema ~s from node ~p", - [Name, Node]), - {Host, Port} = riak_pb(hd(rt:connection_info([Node]))), - {ok, PBConn} = riakc_pb_socket:start_link(Host, Port), - R = riakc_pb_socket:get_search_schema(PBConn, Name), - riakc_pb_socket:stop(PBConn), - case R of - {ok, PL} -> - case Content of - ignore -> - Name == proplists:get_value(name, PL); - _ -> - (Name == proplists:get_value(name, PL)) and - (Content == proplists:get_value(content, PL)) - end; - _ -> - false - end - end, - wait_until(Cluster, F), - ok. - -%% @doc Expire YZ trees --spec expire_trees([node()]) -> ok. -expire_trees(Cluster) -> - lager:info("Expire all trees"), - _ = [ok = rpc:call(Node, yz_entropy_mgr, expire_trees, []) - || Node <- Cluster], - - %% The expire is async so just give it a moment - timer:sleep(100), - ok. - -%% @doc Expire YZ trees --spec clear_trees([node()]) -> ok. -clear_trees(Cluster) -> - lager:info("Expire all trees"), - _ = [ok = rpc:call(Node, yz_entropy_mgr, clear_trees, []) - || Node <- Cluster], - ok. - -brutal_kill_remove_index_dirs(Nodes, IndexName, Services) -> - IndexDirs = get_index_dirs(IndexName, Nodes), - rt:brutal_kill(hd(Nodes)), - [rt:stop(ANode) || ANode <- tl(Nodes)], - remove_index_dirs2(Nodes, IndexDirs, Services), - ok. - -%% @doc Remove index directories, removing the index. --spec remove_index_dirs([node()], index_name(), [atom()]) -> ok. -remove_index_dirs(Nodes, IndexName, Services) -> - IndexDirs = get_index_dirs(IndexName, Nodes), - [rt:stop(ANode) || ANode <- Nodes], - remove_index_dirs2(Nodes, IndexDirs, Services), - ok. - -remove_index_dirs2(Nodes, IndexDirs, Services) -> - lager:info("Remove index dirs: ~p, on nodes: ~p~n", - [IndexDirs, Nodes]), - [rt:del_dir(binary_to_list(IndexDir)) || IndexDir <- IndexDirs], - [start_and_wait(ANode, Services) || ANode <- Nodes]. - -get_index_dirs(IndexName, Nodes) -> - IndexDirs = [rpc:call(Node, yz_index, index_dir, [IndexName]) || - Node <- Nodes], - IndexDirs. - -start_and_wait(Node, WaitForServices) -> - rt:start(Node), - [rt:wait_for_service(Node, Service) || Service <- WaitForServices]. - -%% @doc Check if index/core exists in metadata, disk via yz_index:exists. --spec check_exists([node()], index_name()) -> ok. -check_exists(Nodes, IndexName) -> - wait_until(Nodes, - fun(N) -> - rpc:call(N, yz_index, exists, [IndexName]) - end). - --spec verify_num_found_query([node()], index_name(), count()) -> ok. -verify_num_found_query(Cluster, Index, ExpectedCount) -> - F = fun(Node) -> - Pid = rt:pbc(Node), - {ok, {_, _, _, NumFound}} = riakc_pb_socket:search(Pid, Index, <<"*:*">>), - lager:info("Check Count, Expected: ~p | Actual: ~p~n", - [ExpectedCount, NumFound]), - ExpectedCount =:= NumFound - end, - wait_until(Cluster, F), - ok. - -%% @doc Brought over from yz_rt in the yokozuna repo - FORNOW. --spec search_expect(node()|[node()], index_name(), string(), string(), - non_neg_integer()) -> ok. -search_expect(NodeOrNodes, Index, Name, Term, Expect) -> - search_expect(NodeOrNodes, yokozuna, Index, Name, Term, Expect). - --spec search_expect(node()|[node()], search_type(), index_name(), - string(), string(), [string()], non_neg_integer()) -> ok. -search_expect(Nodes, solr, Index, Name0, Term0, Shards, Expect) - when is_list(Shards), length(Shards) > 0, is_list(Nodes) -> - Name = quote_unicode(Name0), - Term = quote_unicode(Term0), - Node = rt:select_random(Nodes), - {Host, Port} = solr_hp(Node, Nodes), - URL = internal_solr_url(Host, Port, Index, Name, Term, Shards), - lager:info("Run solr search ~s", [URL]), - Opts = [{response_format, binary}], - F = fun(_) -> - {ok, "200", _, R} = ibrowse:send_req(URL, [], get, [], Opts, - ?IBROWSE_TIMEOUT), - verify_count_http(Expect, R) - end, - wait_until(Nodes, F); -search_expect(Node, solr=Type, Index, Name, Term, Shards, Expect) - when is_list(Shards), length(Shards) > 0 -> - search_expect([Node], Type, Index, Name, Term, Shards, Expect). - --spec search_expect(node()|[node()], search_type(), index_name(), - string(), string(), non_neg_integer()) -> ok. -search_expect(Nodes, solr=Type, Index, Name, Term, Expect) when is_list(Nodes) -> - Node = rt:select_random(Nodes), - HP = solr_hp(Node, Nodes), - - %% F could actually be returned in a shared fun, but w/ so much arity, - %% just using it twice makes sense. - F = fun(_) -> - {ok, "200", _, R} = search(Type, HP, Index, Name, Term), - verify_count_http(Expect, R) - end, - - wait_until(Nodes, F); -search_expect(Nodes, yokozuna=Type, Index, Name, Term, Expect) - when is_list(Nodes) -> - HP = hd(host_entries(rt:connection_info(Nodes))), - - F = fun(_) -> - {ok, "200", _, R} = search(Type, HP, Index, Name, Term), - verify_count_http(Expect, R) - end, - - wait_until(Nodes, F); -search_expect(Node, Type, Index, Name, Term, Expect) -> - search_expect([Node], Type, Index, Name, Term, Expect). - -assert_search(Pid, Cluster, Index, Search, SearchExpect, Params) -> - F = fun(_) -> - lager:info("Searching ~p and asserting it exists", - [SearchExpect]), - case riakc_pb_socket:search(Pid, Index, Search, Params) of - {ok,{search_results,[{_Index,Fields}], _Score, Found}} -> - ?assert(lists:member(SearchExpect, Fields)), - case Found of - 1 -> true; - 0 -> false - end; - {ok, {search_results, [], _Score, 0}} -> - lager:info("Search has not yet yielded data"), - false - end - end, - wait_until(Cluster, F). - -search(HP, Index, Name, Term) -> - search(yokozuna, HP, Index, Name, Term). - -search(Type, {Host, Port}, Index, Name, Term) when is_integer(Port) -> - search(Type, {Host, integer_to_list(Port)}, Index, Name, Term); - -search(Type, {Host, Port}, Index, Name0, Term0) -> - Name = quote_unicode(Name0), - Term = quote_unicode(Term0), - FmtStr = case Type of - solr -> - "http://~s:~s/internal_solr/~s/select?q=~s:~s&wt=json"; - yokozuna -> - "http://~s:~s/search/query/~s?q=~s:~s&wt=json" - end, - URL = ?FMT(FmtStr, [Host, Port, Index, Name, Term]), - lager:info("Run search ~s", [URL]), - Opts = [{response_format, binary}], - ibrowse:send_req(URL, [], get, [], Opts, ?IBROWSE_TIMEOUT). - -%%%=================================================================== -%%% Private -%%%=================================================================== - --spec verify_count_http(count(), json_string()) -> boolean(). -verify_count_http(Expected, Resp) -> - Count = get_count_http(Resp), - lager:info("Expected: ~p, Actual: ~p", [Expected, Count]), - Expected =:= Count. - --spec get_count_http(json_string()) -> count(). -get_count_http(Resp) -> - Struct = mochijson2:decode(Resp), - kvc:path([<<"response">>, <<"numFound">>], Struct). - --spec riak_http({node(), rt:interfaces()} | rt:interfaces()) -> - {host(), portnum()}. -riak_http({_Node, ConnInfo}) -> - riak_http(ConnInfo); -riak_http(ConnInfo) -> - proplists:get_value(http, ConnInfo). - --spec riak_pb({node(), rt:interfaces()} | rt:interfaces()) -> - {host(), portnum()}. -riak_pb({_Node, ConnInfo}) -> - riak_pb(ConnInfo); -riak_pb(ConnInfo) -> - proplists:get_value(pb, ConnInfo). - --spec create_and_set_index([node()], pid(), bucket(), index_name()) -> ok. -create_and_set_index(Cluster, Pid, Bucket, Index) -> - %% Create a search index and associate with a bucket - lager:info("Create a search index ~s and associate it with bucket ~s", - [Index, Bucket]), - _ = riakc_pb_socket:create_search_index(Pid, Index), - %% For possible legacy upgrade reasons or general check around the cluster, - %% wrap create index in a wait - wait_for_index(Cluster, Index), - set_index(Pid, hd(Cluster), Bucket, Index). --spec create_and_set_index([node()], pid(), bucket(), index_name(), - schema_name()) -> ok. -create_and_set_index(Cluster, Pid, Bucket, Index, Schema) -> - %% Create a search index and associate with a bucket - lager:info("Create a search index ~s with a custom schema named ~s and " ++ - "associate it with bucket ~p", [Index, Schema, Bucket]), - _ = riakc_pb_socket:create_search_index(Pid, Index, Schema, []), - %% For possible legacy upgrade reasons or general check around the cluster, - %% wrap create index in a wait - wait_for_index(Cluster, Index), - set_index(Pid, hd(Cluster), Bucket, Index). - --spec set_index(pid(), node(), bucket(), index_name()) -> ok. -set_index(_Pid, Node, {BucketType, _Bucket}, Index) -> - lager:info("Create and activate map-based bucket type ~s and tie it to search_index ~s", - [BucketType, Index]), - rt:create_and_activate_bucket_type(Node, BucketType, [{search_index, Index}]); -set_index(Pid, _Node, Bucket, Index) -> - ok = riakc_pb_socket:set_search_index(Pid, Bucket, Index). - -internal_solr_url(Host, Port, Index) -> - ?FMT("http://~s:~B/internal_solr/~s", [Host, Port, Index]). -internal_solr_url(Host, Port, Index, Name, Term, Shards) -> - Ss = [internal_solr_url(Host, ShardPort, Index) - || {_, ShardPort} <- Shards], - ?FMT("http://~s:~B/internal_solr/~s/select?wt=json&q=~s:~s&shards=~s", - [Host, Port, Index, Name, Term, string:join(Ss, ",")]). - -quote_unicode(Value) -> - mochiweb_util:quote_plus(binary_to_list( - unicode:characters_to_binary(Value))). - --spec commit([node()], index_name()) -> ok. -commit(Nodes, Index) -> - %% Wait for yokozuna index to trigger, then force a commit - timer:sleep(?SOFTCOMMIT), - lager:info("Commit search writes to ~s at softcommit (default) ~p", - [Index, ?SOFTCOMMIT]), - rpc:multicall(Nodes, yz_solr, commit, [Index]), - ok. - --spec drain_solrqs(node() | cluster()) -> ok. -drain_solrqs(Cluster) when is_list(Cluster) -> - [drain_solrqs(Node) || Node <- Cluster]; -drain_solrqs(Node) -> - rpc:call(Node, yz_solrq_drain_mgr, drain, []), - ok. - --spec override_schema(pid(), [node()], index_name(), schema_name(), string()) -> - {ok, [node()]}. -override_schema(Pid, Cluster, Index, Schema, RawUpdate) -> - lager:info("Overwrite schema with updated schema"), - ok = riakc_pb_socket:create_search_schema(Pid, Schema, RawUpdate), - yokozuna_rt:wait_for_schema(Cluster, Schema, RawUpdate), - [Node|_] = Cluster, - {ok, _} = rpc:call(Node, yz_index, reload, [Index]). - -%% @doc Wrapper around `rt:wait_until' to verify `F' against multiple -%% nodes. The function `F' is passed one of the `Nodes' as -%% argument and must return a `boolean()' delcaring whether the -%% success condition has been met or not. --spec wait_until([node()], fun((node()) -> boolean())) -> ok. -wait_until(Nodes, F) -> - [?assertEqual(ok, rt:wait_until(Node, F)) || Node <- Nodes], - ok. - --spec solr_hp(node(), [node()]) -> {host(), portnum()}. -solr_hp(Node, Cluster) -> - CI = connection_info(Cluster), - solr_http(proplists:get_value(Node, CI)). - --spec connection_info(list()) -> orddict:orddict(). -connection_info(Cluster) -> - CI = orddict:from_list(rt:connection_info(Cluster)), - SolrInfo = orddict:from_list([{Node, [{solr_http, get_yz_conn_info(Node)}]} - || Node <- Cluster]), - orddict:merge(fun(_,V1,V2) -> V1 ++ V2 end, CI, SolrInfo). - --spec solr_http({node(), orddict:orddict()}) -> {host(), portnum()}. -solr_http({_Node, ConnInfo}) -> - solr_http(ConnInfo); -solr_http(ConnInfo) -> - proplists:get_value(solr_http, ConnInfo). - --spec get_yz_conn_info(node()) -> {string(), string()}. -get_yz_conn_info(Node) -> - {ok, SolrPort} = rpc:call(Node, application, get_env, [yokozuna, solr_port]), - %% Currently Yokozuna hardcodes listener to all interfaces - {"127.0.0.1", SolrPort}. - --spec http(method(), string(), list(), binary()|[]) -> response(). -http(Method, URL, Headers, Body) -> - Opts = [], - ibrowse:send_req(URL, Headers, Method, Body, Opts, ?IBROWSE_TIMEOUT). - --spec http(method(), string(), list(), binary()|[], list()|timeout()) - -> response(). -http(Method, URL, Headers, Body, Opts) when is_list(Opts) -> - ibrowse:send_req(URL, Headers, Method, Body, Opts, ?IBROWSE_TIMEOUT); -http(Method, URL, Headers, Body, Timeout) when is_integer(Timeout) -> - Opts = [], - ibrowse:send_req(URL, Headers, Method, Body, Opts, Timeout). - --spec http(method(), string(), list(), binary()|[], list(), timeout()) - -> response(). -http(Method, URL, Headers, Body, Opts, Timeout) when - is_list(Opts) andalso is_integer(Timeout) -> - ibrowse:send_req(URL, Headers, Method, Body, Opts, Timeout). - --spec create_indexed_bucket_type(cluster(), binary(), index_name()) -> ok. -create_indexed_bucket_type(Cluster, BucketType, IndexName) -> - ok = create_index(Cluster, IndexName), - ok = create_bucket_type(Cluster, BucketType, [{search_index, IndexName}]). - --spec create_indexed_bucket_type(cluster(), binary(), index_name(), - schema_name()) -> ok. -create_indexed_bucket_type(Cluster, BucketType, IndexName, SchemaName) -> - ok = create_index(Cluster, IndexName, SchemaName), - ok = create_bucket_type(Cluster, BucketType, [{search_index, IndexName}]). - --spec create_index(cluster(), index_name()) -> ok. -create_index(Cluster, Index) -> - Node = select_random(Cluster), - lager:info("Creating index ~s [~p]", [Index, Node]), - rpc:call(Node, yz_index, create, [Index]), - ok = wait_for_index(Cluster, Index). - --spec create_index(cluster(), index_name(), schema_name()) -> ok. -create_index(Cluster, Index, SchemaName) -> - Node = select_random(Cluster), - lager:info("Creating index ~s with schema ~s [~p]", - [Index, SchemaName, Node]), - rpc:call(Node, yz_index, create, [Index, SchemaName]), - ok = wait_for_index(Cluster, Index). - -select_random(List) -> - Length = length(List), - Idx = random:uniform(Length), - lists:nth(Idx, List). - --spec create_bucket_type(cluster(), binary(), [term()]) -> ok. -create_bucket_type(Cluster, BucketType, Props) -> - Node = select_random(Cluster), - rt:create_and_activate_bucket_type(Node, BucketType, Props), - rt:wait_until_bucket_type_status(BucketType, active, Node), - rt:wait_until_bucket_type_visible(Cluster, BucketType). diff --git a/tests/bucket_types.erl b/tests/bucket_types.erl index 7e426be5a..3ea0e2052 100644 --- a/tests/bucket_types.erl +++ b/tests/bucket_types.erl @@ -8,6 +8,10 @@ confirm() -> application:start(inets), lager:info("Deploy some nodes"), + + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + Nodes = rt:build_cluster(4, [], [ {riak_core, [{default_bucket_props, [ diff --git a/tests/http_security.erl b/tests/http_security.erl index fac70e329..413fb4c82 100644 --- a/tests/http_security.erl +++ b/tests/http_security.erl @@ -11,6 +11,9 @@ -define(assertDenied(Op), ?assertMatch({error, {forbidden, _}}, Op)). confirm() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + application:start(crypto), application:start(asn1), application:start(public_key), diff --git a/tests/kv679_dataloss_fb.erl b/tests/kv679_dataloss_fb.erl new file mode 100644 index 000000000..c709e85f6 --- /dev/null +++ b/tests/kv679_dataloss_fb.erl @@ -0,0 +1,185 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2017 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%%% @copyright (C) 2017, Basho Technologies +%%% @doc +%%% riak_test for kv679 lost clock/fallback/handoff flavour. +%%% +%%% issue kv679 is a possible dataloss issue, it's basically caused by +%%% the fact that per key logical clocks can go backwards in time in +%%% certain situations. The situation under test here is as follows: +%%% +%% A coords a write to K [{a, 1}] and replicates to fallbacks D, E +%% A coords a write to K [{a, 2}] and replicates to primaries B, C +%% A coords a write K [{a, 3}] and replicates to primaries B, C +%% A loses it's clock for K (so far this is like the lost clock case above) +%% Read of A, D, E read repairs A with K=[{a, 1}] +%% A coords a write, issues [{a, 2}] again +%% Acked write is lost +%%% +%%% +%%% @end + +-module(kv679_dataloss_fb). +-behavior(riak_test). +-compile([export_all]). +-export([confirm/0]). + +-include_lib("eunit/include/eunit.hrl"). + +-define(BUCKET, <<"kv679">>). +-define(KEY, <<"test">>). + +confirm() -> + Conf = [ + {riak_kv, [{anti_entropy, {off, []}}]}, + {riak_core, [{default_bucket_props, [{allow_mult, true}, + {dvv_enabled, true}, + {ring_creation_size, 8}, + {vnode_management_timer, 1000}, + {handoff_concurrency, 100}, + {vnode_inactivity_timeout, 1000}]}]}, + {bitcask, [{sync_strategy, o_sync}, {io_mode, nif}]}], + + %% Such nodes 'cos I want a perfect preflist when 2 primaries are + %% down. i.e. I want to kill the fallbacks before they can + %% handoff without effecting the primaries + Nodes = rt:build_cluster(6, Conf), + + Clients = kv679_tombstone:create_pb_clients(Nodes), + + %% Get preflist for key + PL = kv679_tombstone:get_preflist(hd(Nodes)), + + ?assert(kv679_tombstone2:perfect_preflist(PL)), + + lager:info("Got preflist"), + + {CoordNode, _}=CoordClient = kv679_tombstone:coordinating_client(Clients, PL), + + OtherPrimaries = [Node || {{_Idx, Node}, Type} <- PL, + Type == primary, + Node /= CoordNode], + + [rt:stop_and_wait(N) || N <- OtherPrimaries], + + lager:info("Killed 2 primaries"), + + rt:wait_until(fun() -> + NewPL = kv679_tombstone:get_preflist(CoordNode), + primary_and_fallback_counts(NewPL) == {1, 2} + end), + + FBPL = kv679_tombstone:get_preflist(CoordNode), + + lager:info("Got a preflist with coord and 2 fbs ~p~n", [FBPL]), + + %% Write key twice at remaining, coordinating primary + kv679_tombstone:write_key(CoordClient, [<<"bob">>, <<"jim">>]), + kv679_tombstone2:dump_clock(CoordClient), + lager:info("Clock at 2 fallbacks"), + + %% Kill the fallbacks before they can handoff + Fallbacks = [Node || {{_Idx, Node}, Type} <- FBPL, + Type == fallback], + + [rt:brutal_kill(FB) || FB <- Fallbacks], + + %% Bring back the primaries and do some more writes + [rt:start_and_wait(P) || P <- OtherPrimaries], + lager:info("started primaries back up"), + rt:wait_until(fun() -> + NewPL = kv679_tombstone:get_preflist(CoordNode), + NewPL == PL + end), + kv679_tombstone:write_key(CoordClient, [<<"jon">>, <<"joe">>]), + kv679_tombstone2:dump_clock(CoordClient), + + %% Kill those primaries with their frontier clocks + [rt:brutal_kill(P) || P <- OtherPrimaries], + lager:info("killed primaries again"), + + %% delete the local data at the coordinator Key + kv679_dataloss:delete_datadir(hd(PL)), + + %% Start up those fallbacks + [rt:start_and_wait(F) || F <- Fallbacks], + lager:info("restart fallbacks"), + %% Wait for the fallback prefist + rt:wait_until(fun() -> + NewPL = kv679_tombstone:get_preflist(CoordNode), + NewPL == FBPL + end), + + %% Read the key, read repair will mean that the data deleted vnode + %% will have an old clock (gone back in time!) + await_read_repair(CoordClient), + kv679_tombstone2:dump_clock(CoordClient), + + %% write a new value, this _should_ be a sibling of what is on + %% crashed primaries + kv679_tombstone:write_key(CoordClient, <<"anne">>), + kv679_tombstone2:dump_clock(CoordClient), + + %% Time to start up those primaries, let handoff happen, and see + %% what happened to that last write + [rt:start_and_wait(P) || P <- OtherPrimaries], + lager:info("restart primaries _again_"), + rt:wait_until(fun() -> + NewPL = kv679_tombstone:get_preflist(CoordNode), + NewPL == PL + end), + + lager:info("wait for handoffs"), + [begin + rpc:call(FB, riak_core_vnode_manager, force_handoffs, []), + rt:wait_until_transfers_complete([FB]) + end || FB <- Fallbacks], + + lager:info("final get"), + + Res = kv679_tombstone:read_key(CoordClient), + ?assertMatch({ok, _}, Res), + {ok, O} = Res, + + %% A nice riak would have somehow managed to make a sibling of the + %% last write + ?assertEqual([<<"anne">>, <<"joe">>], riakc_obj:get_values(O)), + lager:info("Final Object ~p~n", [O]), + pass. + +primary_and_fallback_counts(PL) -> + lists:foldl(fun({{_, _}, primary}, {P, F}) -> + {P+1, F}; + ({{_, _}, fallback}, {P, F}) -> + {P, F+1} + end, + {0, 0}, + PL). + +%% Wait until a read repair has occured, or at least, wait until there +%% is a value on disk at the coordinating/primary vnode (and assume it +%% must have got there via read repair) +await_read_repair(Client) -> + rt:wait_until(fun() -> + {ok, _O} = kv679_tombstone:read_key(Client), + {T, V} = kv679_tombstone:read_key(Client, [{pr,1},{r,1}, {sloppy_quorum, false}]), + lager:info("pr=1 fetch res ~p ~p", [T, V]), + T /= error + end). diff --git a/tests/kv679_dataloss_fb2.erl b/tests/kv679_dataloss_fb2.erl new file mode 100644 index 000000000..245fdf7db --- /dev/null +++ b/tests/kv679_dataloss_fb2.erl @@ -0,0 +1,249 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2017 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%%% @copyright (C) 2017, Basho Technologies +%%% @doc +%%% riak_test for kv679 lost clock/fallback/handoff flavour. +%%% +%%% issue kv679 is a possible dataloss issue, it's basically caused by +%%% the fact that per key logical clocks can go backwards in time in +%%% certain situations. The situation under test here is as follows: +%%% +%% write at P1 replicates to F1 clock is at [{p1, 1}] at p1, f1 +%% write twice more at P1, replciates to F2, clock is at [{p1, 3}] at p1, f2 +%% write at p2 replicates to f3, clock is at [{p2, 1}] at p2, f3 +%% handoff from f3->p1, during which the local read at p1 fails (disk error, whatever) +%% clock at p1,p2 is [{p2, 1}] f3 deletes after handoff (no more f3) +%% handoff from f1->p2 clock at p2 [{p1, 1}, {p2, 1}] +%% read repair from p2 -> p1, clock at p1 [{p1, 1}, {p2, 1}] +%% write on p1 replicate to p2, clock at [{p1, 2}, {p2, 1}] at p1, p2 +%% handoff f2->p2, merge causes last write to be lost since entry {p1, 3} > {p1, 2} +%% read repair p2->p1 and the acked write is silently lost forever. +%%% +%%% +%% NOTE this failure occurs even in riak2.1 with actor-epochs, since +%% it is caused by a local-not-found on a non-coordinating write. EQC +%% found this issue, the counter example is below. +%%% +%%% +%% [{set,{var,1},{call,kv679_eqc,put,[p1,p2,<<"A">>,959494,undefined,false]}}, +%% {set,{var,3},{call,kv679_eqc,put,[p3,p2,<<"A">>,960608,{var,2},false]}}, +%% {set,{var,5},{call,kv679_eqc,put,[p1,f3,<<"A">>,960760,{var,4},false]}}, +%% {set,{var,6},{call,kv679_eqc,put,[p1,f1,<<"A">>,960851,{var,4},false]}}, +%% {set,{var,7},{call,kv679_eqc,forget,[p1,<<"A">>]}}, +%% {set,{var,8},{call,kv679_eqc,replicate,[<<"A">>,p3,p1]}}, +%% {set,{var,9},{call,kv679_eqc,replicate,[<<"A">>,f3,p1]}}, +%% {set,{var,11},{call,kv679_eqc,put,[p1,p2,<<"A">>,962942,{var,10},false]}}] +%% +%%% +%%% +%%% @end + +-module(kv679_dataloss_fb2). +-behavior(riak_test). +-compile([export_all]). +-export([confirm/0]). +-import(kv679_dataloss_fb, [primary_and_fallback_counts/1]). + +-include_lib("eunit/include/eunit.hrl"). + +-define(BUCKET, <<"kv679">>). +-define(KEY, <<"test">>). +-define(NVAL, 2). + +confirm() -> + Conf = [ + {riak_kv, [{anti_entropy, {off, []}}]}, + {riak_core, [{default_bucket_props, [ + %% note reduced n_val + %% is to reduce + %% number of nodes + %% required for test + %% case and to + %% simplify case. + {n_val, ?NVAL}, + {allow_mult, true}, + {dvv_enabled, true}, + {ring_creation_size, 8}, + {vnode_management_timer, 1000}, + {handoff_concurrency, 100}, + {vnode_inactivity_timeout, 1000}]}]}, + {bitcask, [{sync_strategy, o_sync}, {io_mode, nif}]}], + + Nodes = rt:build_cluster(5, Conf), + Clients = kv679_tombstone:create_pb_clients(Nodes), + {_, TempPBC} = hd(Clients), + rt:pbc_set_bucket_prop(TempPBC, ?BUCKET, [{n_val, ?NVAL}]), + rt:wait_until_bucket_props(Nodes, ?BUCKET, [{n_val, ?NVAL}]), + + %% Get preflist for key + PL = kv679_tombstone:get_preflist(hd(Nodes), ?NVAL), + ?assert(kv679_tombstone2:perfect_preflist(PL, ?NVAL)), + + lager:info("Got preflist ~p", [PL]), + + {CoordNode, _}=CoordClient = kv679_tombstone:coordinating_client(Clients, PL), + OtherPrimaries = [Node || {{_Idx, Node}, Type} <- PL, + Type == primary, + Node /= CoordNode], + Fallbacks = [FB || FB <- Nodes, FB /= CoordNode andalso not lists:member(FB, OtherPrimaries)], + + ?assert(length(Fallbacks) == 3), + + lager:info("fallbacks ~p, primaries ~p~n", [Fallbacks, [CoordNode] ++ OtherPrimaries]), + + %% Partition the other primary and one non-preflist node + {_, _, _P1, P2} = PartInfo = rt:partition([CoordNode] ++ tl(Fallbacks), OtherPrimaries ++ [hd(Fallbacks)]), + lager:info("Partitioned ~p~n", [PartInfo]), + + rt:wait_until(fun() -> + NewPL = kv679_tombstone:get_preflist(CoordNode, ?NVAL), + lager:info("new PL ~p~n", [NewPL]), + primary_and_fallback_counts(NewPL) == {1, 1} + end), + + FBPL = kv679_tombstone:get_preflist(CoordNode, ?NVAL), + lager:info("Got a preflist with coord and 1 fb ~p~n", [FBPL]), + + %% Write key once at coordinating primary + kv679_tombstone:write_key(CoordClient, [<<"alice">>]), + kv679_tombstone2:dump_clock(CoordClient), + lager:info("Clock at fallback"), + + %% Kill the fallback before it can handoff + [FB1] = [Node || {{_Idx, Node}, Type} <- FBPL, + Type == fallback], + rt:brutal_kill(FB1), + + %% get a new preflist with a different fallback + rt:wait_until(fun() -> + NewPL = kv679_tombstone:get_preflist(CoordNode, ?NVAL), + primary_and_fallback_counts(NewPL) == {1, 1} andalso NewPL /= FBPL + end), + FBPL2 = kv679_tombstone:get_preflist(CoordNode, ?NVAL), + lager:info("Got a preflist with coord and 1 fb ~p~n", [FBPL2]), + ?assert(FBPL2 /= FBPL), + + %% do two more writes so that there exists out there a clock of + %% {Primary1, 1} on the first fallback node and {Primary1, 3} on + %% the second + kv679_tombstone:write_key(CoordClient, [<<"bob">>, <<"charlie">>]), + kv679_tombstone2:dump_clock(CoordClient), + lager:info("Clock at fallback2"), + + %% Kill the fallback before it can handoff + [FB2] = [Node || {{_Idx, Node}, Type} <- FBPL2, + Type == fallback], + rt:brutal_kill(FB2), + + %% meanwhile, in the other partition, let's write some data + P2PL = kv679_tombstone:get_preflist(hd(P2), ?NVAL), + lager:info("partition 2 PL ~p", [P2PL]), + [P2FB] = [Node || {{_Idx, Node}, Type} <- P2PL, + Type == fallback], + [P2P] = [Node || {{_Idx, Node}, Type} <- P2PL, + Type == primary], + P2Client = kv679_tombstone:coordinating_client(Clients, P2PL), + kv679_tombstone:write_key(P2Client, [<<"dave">>]), + kv679_tombstone2:dump_clock(P2Client), + lager:info("Clock in Partition 2"), + + %% set up a local read error on Primary 1 (this is any disk error, + %% delete the datadir, an intercept for a local crc failure or + %% deserialisation error) + rt_intercept:add(CoordNode, {riak_kv_bitcask_backend, [{{get, 3}, always_corrupt_get}]}), + + %% heal the partition, remember the Partition 1 fallbacks aren't + %% handing off yet (we killed them, but it could be any delay in + %% handoff, not a failure, killing them is just a way to control + %% timing (maybe should've used intercepts??)) + {ok, DownNodes} = rt:heal_upnodes(PartInfo), + lager:info("These nodes still need healing ~p", [DownNodes]), + + %% wait for the partition 2 fallback to hand off + rpc:call(P2FB, riak_core_vnode_manager, force_handoffs, []), + rt:wait_until_node_handoffs_complete(P2FB), + + %% what's the clock on primary 1 now? NOTE: this extra read was + %% the only way I could ensure that the handoff had occured before + %% cleaning out the intercept. A sleep also worked. + kv679_tombstone2:dump_clock(CoordClient), + rt_intercept:clean(CoordNode, riak_kv_bitcask_backend), + kv679_tombstone2:dump_clock(CoordClient), + + %% restart fallback one and wait for it to handoff + rt:start_and_wait(FB1), + lager:info("started fallback 1 back up"), + rpc:call(FB1, riak_core_vnode_manager, force_handoffs, []), + rt:wait_until_node_handoffs_complete(FB1), + + %% kill dev5 again (there is something very weird here, unless I + %% do this dev3 _never_ hands off (and since n=2 it is never used + %% in a preflist for read repair)) So the only way to get it's + %% data onto CoordNode is to ensure a preflist of FB1 and + %% CoordNode for a read! To be fair this rather ruins the test + %% case. @TODO investigate this handoff problem + rt:stop_and_wait(P2P), + rt:stop_and_wait(P2FB), + rt:wait_until(fun() -> + NewPL = kv679_tombstone:get_preflist(CoordNode, ?NVAL), + lager:info("new PL ~p~n", [NewPL]), + primary_and_fallback_counts(NewPL) == {1, 1} andalso + different_nodes(NewPL) + end), + + %% %% what's the clock on primary 1 now? + kv679_tombstone2:dump_clock(CoordClient), + + %% restart P2P and P2FB + rt:start_and_wait(P2P), + rt:start_and_wait(P2FB), + + %% get a primary PL + rt:wait_until(fun() -> + NewPL = kv679_tombstone:get_preflist(CoordNode, ?NVAL), + lager:info("new PL ~p~n", [NewPL]), + primary_and_fallback_counts(NewPL) == {2, 0} andalso + different_nodes(NewPL) + end), + + %% write a new value + kv679_tombstone:write_key(CoordClient, [<<"emma">>]), + kv679_tombstone2:dump_clock(CoordClient), + + %% finally start the last offline fallback, await all transfers, + %% and do a read, the last write, `emma' should be present + rt:start_and_wait(FB2), + rpc:call(FB2, riak_core_vnode_manager, force_handoffs, []), + rt:wait_until_transfers_complete(Nodes), + + lager:info("final get"), + Res = kv679_tombstone:read_key(CoordClient), + ?assertMatch({ok, _}, Res), + {ok, O} = Res, + + %% A nice riak would have somehow managed to make a sibling of the + %% last acked write, even with all the craziness + ?assertEqual([<<"charlie">>, <<"emma">>], lists:sort(riakc_obj:get_values(O))), + lager:info("Final Object ~p~n", [O]), + pass. + +different_nodes(PL) -> + Nodes = [Node || {{_, Node}, _} <- PL], + length(lists:usort(Nodes)) == length(PL). diff --git a/tests/kv679_tombstone.erl b/tests/kv679_tombstone.erl index 690e4570e..e96956b51 100644 --- a/tests/kv679_tombstone.erl +++ b/tests/kv679_tombstone.erl @@ -137,15 +137,19 @@ write_key({_, Client}, Val, Opts) when is_binary(Val) -> Object = case riakc_pb_socket:get(Client, ?BUCKET, ?KEY, []) of {ok, O1} -> lager:info("writing existing!"), - riakc_obj:update_value(O1, Val); + O2 = riakc_obj:update_metadata(O1, dict:new()), + riakc_obj:update_value(O2, Val); _ -> lager:info("writing new!"), riakc_obj:new(?BUCKET, ?KEY, Val) end, riakc_pb_socket:put(Client, Object, Opts). -read_key({_, Client}) -> - riakc_pb_socket:get(Client, ?BUCKET, ?KEY, []). +read_key({_, Client}, Opts) -> + riakc_pb_socket:get(Client, ?BUCKET, ?KEY, Opts). + +read_key(C) -> + read_key(C, []). delete_key({_, Client}) -> riakc_pb_socket:delete(Client, ?BUCKET, ?KEY), @@ -179,9 +183,12 @@ start_node(Node, Preflist) -> wait_for_new_pl(Preflist, Node). get_preflist(Node) -> + get_preflist(Node, 3). + +get_preflist(Node, NVal) -> Chash = rpc:call(Node, riak_core_util, chash_key, [{?BUCKET, ?KEY}]), UpNodes = rpc:call(Node, riak_core_node_watcher, nodes, [riak_kv]), - PL = rpc:call(Node, riak_core_apl, get_apl_ann, [Chash, 3, UpNodes]), + PL = rpc:call(Node, riak_core_apl, get_apl_ann, [Chash, NVal, UpNodes]), PL. kill_primary(Preflist) -> diff --git a/tests/kv679_tombstone2.erl b/tests/kv679_tombstone2.erl index 3135717e1..22d43205d 100644 --- a/tests/kv679_tombstone2.erl +++ b/tests/kv679_tombstone2.erl @@ -128,9 +128,12 @@ confirm() -> perfect_preflist(PL) -> - %% N=3 primaries, each on a unique node + perfect_preflist(PL, 3). + +perfect_preflist(PL, NVal) -> + %% N=NVal primaries, each on a unique node length(lists:usort([Node || {{_Idx, Node}, Type} <- PL, - Type == primary])) == 3. + Type == primary])) == NVal. get_coord_client_and_patsy(Clients, PL) -> {CoordNode, _}=CoordClient=kv679_tombstone:coordinating_client(Clients, PL), diff --git a/tests/overload.erl b/tests/overload.erl index 8f5aab142..9be681b3c 100644 --- a/tests/overload.erl +++ b/tests/overload.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2013 Basho Technologies, Inc. +%% Copyright (c) 2013-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -81,6 +81,9 @@ default_config(#config{ {riak_api, [{pb_backlog, 1024}]}]. confirm() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + [Node1 | _] = Nodes = setup(), ok = create_bucket_type(Nodes, ?NORMAL_TYPE, [{n_val, 3}]), diff --git a/tests/partition_repair.erl b/tests/partition_repair.erl index b5c54da61..4c9849438 100644 --- a/tests/partition_repair.erl +++ b/tests/partition_repair.erl @@ -169,7 +169,7 @@ kill_repair_verify({Partition, Node}, DataSuffix, Service) -> lager:info("Verify ~p on ~p is fully repaired", [Partition, Node]), Data2 = get_data(Service, {Partition, Node}), - {Verified, NotFound} = dict:fold(verify(Service, Data2), {0, []}, Stash), + {Verified, NotFound} = dict:fold(verify(Node, Service, Data2), {0, []}, Stash), case NotFound of [] -> ok; _ -> @@ -188,7 +188,7 @@ kill_repair_verify({Partition, Node}, DataSuffix, Service) -> {ok, [StashB]} = file:consult(StashPathB), ExpectToVerifyB = dict:size(StashB), BeforeData = get_data(Service, B), - {VerifiedB, NotFoundB} = dict:fold(verify(Service, BeforeData), {0, []}, StashB), + {VerifiedB, NotFoundB} = dict:fold(verify(Node, Service, BeforeData), {0, []}, StashB), case NotFoundB of [] -> ok; _ -> @@ -203,7 +203,7 @@ kill_repair_verify({Partition, Node}, DataSuffix, Service) -> {ok, [StashA]} = file:consult(StashPathA), ExpectToVerifyA = dict:size(StashA), AfterData = get_data(Service, A), - {VerifiedA, NotFoundA} = dict:fold(verify(Service, AfterData), {0, []}, StashA), + {VerifiedA, NotFoundA} = dict:fold(verify(Node, Service, AfterData), {0, []}, StashA), case NotFoundA of [] -> ok; _ -> @@ -214,19 +214,23 @@ kill_repair_verify({Partition, Node}, DataSuffix, Service) -> ?assertEqual(ExpectToVerifyA, VerifiedA). -verify(riak_kv, DataAfterRepair) -> +verify(Node, riak_kv, DataAfterRepair) -> fun(BKey, StashedValue, {Verified, NotFound}) -> StashedData={BKey, StashedValue}, case dict:find(BKey, DataAfterRepair) of error -> {Verified, [StashedData|NotFound]}; {ok, Value} -> - if Value == StashedValue -> {Verified+1, NotFound}; - true -> {Verified, [StashedData|NotFound]} + %% NOTE: since kv679 fixes, the binary values may + %% not be equal where a new epoch-actor-entry has + %% been added to the repaired value in the vnode + case gte(Node, Value, StashedValue, BKey) of + true -> {Verified+1, NotFound}; + false -> {Verified, [StashedData|NotFound]} end end end; -verify(riak_search, PostingsAfterRepair) -> +verify(_Node, riak_search, PostingsAfterRepair) -> fun(IFT, StashedPostings, {Verified, NotFound}) -> StashedPosting={IFT, StashedPostings}, case dict:find(IFT, PostingsAfterRepair) of @@ -241,6 +245,22 @@ verify(riak_search, PostingsAfterRepair) -> end end. +%% @private gte checks that `Value' is _at least_ `StashedValue'. With +%% the changes for kv679 when a vnode receives a write of a key that +%% contains the vnode's id as an entry in the version vector, it adds +%% a new actor-epoch-entry to the version vector to guard against data +%% loss from repeated events (remember a VV history is supposed to be +%% unique!) This function then must deserialise the stashed and vnode +%% data and check that they are equal, or if not, the only difference +%% is an extra epoch actor in the vnode value's vclock. +gte(Node, Value, StashedData, {B, K}) -> + VnodeObject = riak_object:from_binary(B, K, Value), + StashedObject = riak_object:from_binary(B, K, StashedData), + %% NOTE: we need a ring and all that jazz for bucket props, needed + %% by merge, so use an RPC to merge on a riak node. + Merged = rpc:call(Node, riak_object, syntactic_merge, [VnodeObject, StashedObject]), + riak_object:equal(VnodeObject, Merged). + is_true(X) -> X == true. diff --git a/tests/pb_security.erl b/tests/pb_security.erl index e43a5ac1d..531ffe239 100644 --- a/tests/pb_security.erl +++ b/tests/pb_security.erl @@ -11,6 +11,9 @@ -define(assertDenied(Op), ?assertMatch({error, <<"Permission",_/binary>>}, Op)). confirm() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + application:start(crypto), application:start(asn1), application:start(public_key), diff --git a/tests/repl_aae_fullsync.erl b/tests/repl_aae_fullsync.erl index a14651f86..cdcb044f7 100644 --- a/tests/repl_aae_fullsync.erl +++ b/tests/repl_aae_fullsync.erl @@ -29,6 +29,7 @@ [ %% Specify fast building of AAE trees {sweep_tick, 10000}, + {force_hashtree_upgrade, true}, {anti_entropy, {on, [debug]}}, {anti_entropy_build_limit, {100, 1000}}, {anti_entropy_concurrency, 100} diff --git a/tests/riaknostic_rt.erl b/tests/riaknostic_rt.erl index 85ed822a5..4519e9ebc 100644 --- a/tests/riaknostic_rt.erl +++ b/tests/riaknostic_rt.erl @@ -22,19 +22,11 @@ -export([confirm/0]). -include_lib("eunit/include/eunit.hrl"). -%% Change when a new release comes out. --define(RIAKNOSTIC_URL, "https://github.com/basho/riaknostic/downloads/riaknostic-1.0.2.tar.gz"). - -%% REQUIRES (sh, curl, tar) - confirm() -> %% Build a small cluster [Node1, _Node2] = rt:build_cluster(2, []), ?assertEqual(ok, rt:wait_until_nodes_ready([Node1])), - %% Install riaknostic for Riak versions below 1.3.0 - riaknostic_bootstrap(Node1), - %% Run through all tests on Node1 check_riaknostic_execute(Node1), check_riaknostic_usage(Node1), @@ -45,26 +37,6 @@ confirm() -> lager:info("Test riaknostic: PASS"), pass. -riaknostic_bootstrap(Node) -> - lager:info("Check if riaknostic is installed"), - {ok, RiaknosticOut1} = rt:admin(Node, ["diag"]), - riaknostic_install((rt:str(RiaknosticOut1, "is not present!")), Node). - -%% riaknostic is already installed, move along -riaknostic_install(false, _Node) -> - ok; - -%% install riaknostic -riaknostic_install(true, Node) -> - %% Install - lager:info("Installing Riaknostic"), - {ok, LibDir} = rpc:call(Node, application, get_env, [riak_core, platform_lib_dir]), - Cmd = io_lib:format("sh -c \"cd ~s && curl -O -L ~s && tar xzf ~s\"", - [LibDir, ?RIAKNOSTIC_URL, filename:basename(?RIAKNOSTIC_URL)]), - lager:info("Running command: ~s", [Cmd]), - lager:debug("~p~n", [rpc:call(Node, os, cmd, [Cmd])]), - ok. - %% Check that riaknostic executes check_riaknostic_execute(Node) -> %% Execute diff --git a/tests/rolling_capabilities.erl b/tests/rolling_capabilities.erl index 5b05788b4..716a4dddd 100644 --- a/tests/rolling_capabilities.erl +++ b/tests/rolling_capabilities.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2012 Basho Technologies, Inc. +%% Copyright (c) 2012-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -23,6 +23,9 @@ -include_lib("eunit/include/eunit.hrl"). confirm() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + TestMetaData = riak_test_runner:metadata(), Count = 4, OldVsn = proplists:get_value(upgrade_version, TestMetaData, previous), diff --git a/tests/ts_cluster_capabilities_SUITE.erl b/tests/ts_cluster_capabilities_SUITE.erl index b055eb042..30e84dfd9 100644 --- a/tests/ts_cluster_capabilities_SUITE.erl +++ b/tests/ts_cluster_capabilities_SUITE.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2016 Basho Technologies, Inc. +%% Copyright (c) 2016-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -30,19 +30,13 @@ %%-------------------------------------------------------------------- -define(TS_VERSION_CURRENT, "current"). - -%% git checkout riak_ts-1.3.1 && make locked-deps --define(TS_VERSION_1_3, riak_ts_1_3_1). - -define(SQL_SELECT_CAP, {riak_kv, sql_select_version}). suite() -> [{timetrap,{minutes,10}}]. init_per_suite(Config) -> - Versions = rt:find_version_by_name(["riak_ts-1.3.1", "riak_ts_ee-1.3.1"]), - Vsn131 = maybe_missing_1_3_1(Versions), - [{?TS_VERSION_1_3, Vsn131} | Config]. + Config. end_per_suite(_Config) -> ok. @@ -143,21 +137,19 @@ capability_not_specified_on_one_node_test(_Ctx) -> %% Riak TS Capability Tests %%-------------------------------------------------------------------- -sql_select_upgrade_a_node_from_1_3_test(Config) -> - Vsn131 = ?config(?TS_VERSION_1_3, Config), +sql_select_upgrade_a_node_from_legacy_test(_Ctx) -> [Node_A, Node_B, Node_C] = - rt:deploy_nodes([Vsn131, Vsn131, Vsn131]), + rt:deploy_nodes([legacy, legacy, legacy]), ok = rt:join_cluster([Node_A,Node_B,Node_C]), ok = rt:wait_until_ring_converged([Node_A,Node_B,Node_C]), ok = rt:upgrade(Node_A, ?TS_VERSION_CURRENT), ok = rt:wait_until_ring_converged([Node_A,Node_B,Node_C]), - ok = rt:wait_until_capability(Node_A, ?SQL_SELECT_CAP, v1), + ok = rt:wait_until_capability(Node_A, ?SQL_SELECT_CAP, v2), ok. -sql_select_join_with_all_nodes_upgraded_test(Config) -> - Vsn131 = ?config(?TS_VERSION_1_3, Config), +sql_select_join_with_all_nodes_upgraded_test(_Ctx) -> [Node_A, Node_B, Node_C] = - rt:deploy_nodes([Vsn131, Vsn131, Vsn131]), + rt:deploy_nodes([legacy, legacy, legacy]), ok = rt:join_cluster([Node_A,Node_B,Node_C]), rt:wait_until_ring_converged([Node_A,Node_B,Node_C]), rt:upgrade(Node_A, ?TS_VERSION_CURRENT), @@ -168,18 +160,14 @@ sql_select_join_with_all_nodes_upgraded_test(Config) -> rt:wait_until_capability(Node_C, ?SQL_SELECT_CAP, v3), ok. -%% This test passed for 1.4.0, expecting v1 where commented, but fails for 1.5.0 due to code change. We set it to expect -%% v3 until future changes are made. After 1.5.0 if this test will fail if new code is introduced to handle capability -%% communication. https://bashoeng.atlassian.net/browse/RTS-1415 -sql_select_downgrade_a_node_test(Config) -> - Vsn131 = ?config(?TS_VERSION_1_3, Config), +sql_select_downgrade_a_node_test(_Ctx) -> [Node_A, Node_B, Node_C] = - rt:deploy_nodes([Vsn131, ?TS_VERSION_CURRENT, ?TS_VERSION_CURRENT]), + rt:deploy_nodes([legacy, ?TS_VERSION_CURRENT, ?TS_VERSION_CURRENT]), ok = rt:join_cluster([Node_A,Node_B,Node_C]), rt:wait_until_ring_converged([Node_A,Node_B,Node_C]), % rt:wait_until_capability(Node_A, ?SQL_SELECT_CAP, v3), - rt:wait_until_capability(Node_B, ?SQL_SELECT_CAP, v1), - rt:wait_until_capability(Node_C, ?SQL_SELECT_CAP, v1), + rt:wait_until_capability(Node_B, ?SQL_SELECT_CAP, v2), + rt:wait_until_capability(Node_C, ?SQL_SELECT_CAP, v2), rt:upgrade(Node_A, ?TS_VERSION_CURRENT), rt:wait_until_ring_converged([Node_A,Node_B,Node_C]), rt:wait_until_capability(Node_A, ?SQL_SELECT_CAP, v3), @@ -189,8 +177,8 @@ sql_select_downgrade_a_node_test(Config) -> %% pending changes to how capabilities are communicated around the ring, this %% section will expect v2. Once capabilities are changed in a future version of riak %% (or if testing 1.4.0 as current) - %% the expected version should go back to v1. - rt:upgrade(Node_A, Vsn131), + %% the expected version should go back to v2. + rt:upgrade(Node_A, legacy), rt:wait_until_ring_converged([Node_A,Node_B,Node_C]), rt:wait_until_capability(Node_B, ?SQL_SELECT_CAP, v3), rt:wait_until_capability(Node_C, ?SQL_SELECT_CAP, v3), @@ -200,10 +188,9 @@ sql_select_downgrade_a_node_test(Config) -> %% Perform queries in mixed version cluster %%-------------------------------------------------------------------- -query_in_mixed_version_cluster_test(Config) -> - Vsn131 = ?config(?TS_VERSION_1_3, Config), +query_in_mixed_version_cluster_test(_Ctx) -> [Node_A, Node_B, Node_C] = - rt:deploy_nodes([?TS_VERSION_CURRENT, Vsn131, ?TS_VERSION_CURRENT]), + rt:deploy_nodes([?TS_VERSION_CURRENT, previous, ?TS_VERSION_CURRENT]), ok = rt:join_cluster([Node_A,Node_B,Node_C]), rt:wait_until_ring_converged([Node_A,Node_B,Node_C]), Table = "grouptab1", @@ -220,7 +207,7 @@ query_in_mixed_version_cluster_test(Config) -> [{1,1,B*C} || B <- lists:seq(1,10), C <- lists:seq(1000,5000,1000)]), ExpectedResultSet = [{N} || N <- lists:seq(1000,5000,1000)], %% - %% Test that the current version can query version 1.3 + %% Test that the current version can query older version %% Query = "SELECT c FROM grouptab1 " @@ -232,7 +219,7 @@ query_in_mixed_version_cluster_test(Config) -> {ok,{Cols, Rows}} ), %% - %% Test that the 1.3 can query the current version + %% Test that the previous can query the current version %% {ok, {Cols, Rows}} = run_query(rt:pbc(Node_B), Query), ts_data:assert_row_sets( @@ -240,9 +227,3 @@ query_in_mixed_version_cluster_test(Config) -> {ok,{Cols, Rows}} ). -%% If 1.3.1 is not found, use a meaningful atom to prompt the error message -%% rtdev:check_version/1 -maybe_missing_1_3_1([]) -> - missing_version_ts_1_3_1; -maybe_missing_1_3_1([Vsn]) -> - Vsn. diff --git a/tests/ts_cluster_comprehensive.erl b/tests/ts_cluster_comprehensive.erl index d48019ac0..5e06299a1 100644 --- a/tests/ts_cluster_comprehensive.erl +++ b/tests/ts_cluster_comprehensive.erl @@ -1,6 +1,6 @@ % ------------------------------------------------------------------- %% -%% Copyright (c) 2015 Basho Technologies, Inc. +%% Copyright (c) 2015-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -39,6 +39,9 @@ confirm() -> run_tests(?PVAL_P1, ?PVAL_P2). run_tests(PvalP1, PvalP2) -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + Data = make_data(PvalP1, PvalP2), io:format("Data to be written:\n~p\n...\n~p\n", [hd(Data), lists:last(Data)]), diff --git a/tests/ts_cluster_group_by_SUITE.erl b/tests/ts_cluster_group_by_SUITE.erl index 1477e6b33..f879ddfb9 100644 --- a/tests/ts_cluster_group_by_SUITE.erl +++ b/tests/ts_cluster_group_by_SUITE.erl @@ -55,7 +55,7 @@ end_per_testcase(_TestCase, _Config) -> groups() -> []. -all() -> +all() -> rt:grep_test_functions(?MODULE). client_pid(Ctx) -> @@ -135,8 +135,3 @@ group_by_time_test(Ctx) -> {rt_ignore_columns, [{N*1000,500} || N <- lists:seq(0,9)]}, {ok,{Cols, lists:sort(Rows)}} ). - - - - - diff --git a/tests/ts_cluster_list_irreg_keys_SUITE.erl b/tests/ts_cluster_list_irreg_keys_SUITE.erl index b0ee90f8a..b885f69b6 100644 --- a/tests/ts_cluster_list_irreg_keys_SUITE.erl +++ b/tests/ts_cluster_list_irreg_keys_SUITE.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2016 Basho Technologies, Inc. +%% Copyright (c) 2016-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -39,6 +39,9 @@ %%-------------------------------------------------------------------- suite() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + [{timetrap,{minutes,10}}]. init_per_suite(Config) -> diff --git a/tests/ts_cluster_stream_list_keys_SUITE.erl b/tests/ts_cluster_stream_list_keys_SUITE.erl index 9553d66ef..8c48d9b9f 100644 --- a/tests/ts_cluster_stream_list_keys_SUITE.erl +++ b/tests/ts_cluster_stream_list_keys_SUITE.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2016 Basho Technologies, Inc. +%% Copyright (c) 2016-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -31,6 +31,9 @@ %%-------------------------------------------------------------------- suite() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + [{timetrap,{minutes,10}}]. init_per_suite(Config) -> diff --git a/tests/ts_cluster_updowngrade_group_by_SUITE.erl b/tests/ts_cluster_updowngrade_group_by_SUITE.erl index 2bfdcfa1f..d20b914c5 100644 --- a/tests/ts_cluster_updowngrade_group_by_SUITE.erl +++ b/tests/ts_cluster_updowngrade_group_by_SUITE.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2016 Basho Technologies, Inc. +%% Copyright (c) 2016, 2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -40,8 +40,8 @@ make_scenarios() -> need_query_node_transition = NeedQueryNodeTransition, need_pre_cluster_mixed = NeedPreClusterMixed, need_post_cluster_mixed = NeedPostClusterMixed, - ensure_full_caps = [{{riak_kv, sql_select_version}, v3}, {{riak_kv, riak_ql_ddl_rec_version}, v2}, {{riak_kv, decode_query_results_at_vnode}, true}], - ensure_degraded_caps = [{{riak_kv, sql_select_version}, v2}, {{riak_kv, riak_ql_ddl_rec_version}, v1}, {{riak_kv, decode_query_results_at_vnode}, false}], + ensure_full_caps = ts_updown_util:caps_to_ensure(full), + ensure_degraded_caps = ts_updown_util:caps_to_ensure(degraded), convert_config_to_previous = fun ts_updown_util:convert_riak_conf_to_previous/1} || TableNodeVsn <- [previous, current], QueryNodeVsn <- [previous, current], @@ -51,6 +51,7 @@ make_scenarios() -> NeedPostClusterMixed <- [true, false]], [add_tests(X) || X <- BaseScenarios]. + %% This test will not use config invariants %% see ts_cluster_updowngrade_select_aggregation_SUITE.erl for an example %% of how to use them diff --git a/tests/ts_cluster_updowngrade_order_by_SUITE.erl b/tests/ts_cluster_updowngrade_order_by_SUITE.erl index cbda22f41..d1a79397d 100644 --- a/tests/ts_cluster_updowngrade_order_by_SUITE.erl +++ b/tests/ts_cluster_updowngrade_order_by_SUITE.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2016 Basho Technologies, Inc. +%% Copyright (c) 2016, 2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -60,8 +60,8 @@ make_scenarios() -> need_query_node_transition = NeedQueryNodeTransition, need_pre_cluster_mixed = NeedPreClusterMixed, need_post_cluster_mixed = NeedPostClusterMixed, - ensure_full_caps = [{{riak_kv, sql_select_version}, v3}, {{riak_kv, riak_ql_ddl_rec_version}, v2}, {{riak_kv, decode_query_results_at_vnode}, true}], - ensure_degraded_caps = [{{riak_kv, sql_select_version}, v2}, {{riak_kv, riak_ql_ddl_rec_version}, v1}, {{riak_kv, decode_query_results_at_vnode}, false}], + ensure_full_caps = ts_updown_util:caps_to_ensure(full), + ensure_degraded_caps = ts_updown_util:caps_to_ensure(degraded), convert_config_to_previous = fun ts_updown_util:convert_riak_conf_to_previous/1} || TableNodeVsn <- [previous, current], QueryNodeVsn <- [current], diff --git a/tests/ts_cluster_updowngrade_select_aggregation_SUITE.erl b/tests/ts_cluster_updowngrade_select_aggregation_SUITE.erl index fb5b48123..67b3ffb2c 100644 --- a/tests/ts_cluster_updowngrade_select_aggregation_SUITE.erl +++ b/tests/ts_cluster_updowngrade_select_aggregation_SUITE.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2016 Basho Technologies, Inc. +%% Copyright (c) 2016, 2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -35,8 +35,8 @@ make_scenarios() -> need_query_node_transition = NeedQueryNodeTransition, need_pre_cluster_mixed = NeedPreClusterMixed, need_post_cluster_mixed = NeedPostClusterMixed, - ensure_full_caps = [{{riak_kv, sql_select_version}, v3}, {{riak_kv, riak_ql_ddl_rec_version}, v2}, {{riak_kv, decode_query_results_at_vnode}, true}], - ensure_degraded_caps = [{{riak_kv, sql_select_version}, v2}, {{riak_kv, riak_ql_ddl_rec_version}, v1}, {{riak_kv, decode_query_results_at_vnode}, false}], + ensure_full_caps = ts_updown_util:caps_to_ensure(full), + ensure_degraded_caps = ts_updown_util:caps_to_ensure(degraded), convert_config_to_previous = fun ts_updown_util:convert_riak_conf_to_previous/1} || TableNodeVsn <- [previous, current], QueryNodeVsn <- [previous, current], diff --git a/tests/ts_qbuf_util.erl b/tests/ts_qbuf_util.erl index 0b268012f..7b839f55b 100644 --- a/tests/ts_qbuf_util.erl +++ b/tests/ts_qbuf_util.erl @@ -23,7 +23,9 @@ -export([ ack_query_error/3, + ack_query_error/4, base_query/1, + base_query/2, create_table/2, full_query/2, insert_data/3, @@ -80,8 +82,10 @@ insert_data(C, Table, Data) -> %% Form queries base_query(Table) -> - fmt("select * from ~s where a = '~s' and b = '~s' and c >= ~b and c <= ~b", - [Table, + base_query(Table, "*"). +base_query(Table, Select) -> + fmt("select ~s from ~s where a = '~s' and b = '~s' and c >= ~b and c <= ~b", + [Select, Table, binary_to_list(?WHERE_FILTER_A), binary_to_list(?WHERE_FILTER_B), ?TIMEBASE, ?TIMEBASE + ?LIFESPAN_EXTRA * 1000]). @@ -108,18 +112,26 @@ make_orderby_with_qualifiers(F) -> ack_query_error(Cfg, Query, ErrCode) -> + ack_query_error(Cfg, Query, ErrCode, <<".*">>). + +ack_query_error(Cfg, Query, ExpErrCode, ExpErrMsgPat) -> Node = hd(proplists:get_value(cluster, Cfg)), C = rt:pbc(Node), Res = riakc_ts:query(C, Query, [], undefined, []), case Res of - {error, {ErrCode, ErrMsg}} -> - io:format("reported error ~s", [ErrMsg]), - ok; + {error, {ExpErrCode, GotErrMsg}} -> + case re:run(GotErrMsg, ExpErrMsgPat, []) of + {match, _} -> + ok; + nomatch -> + ct:pal("Query: ~s\nWrong reported error: ~s\n", [Query, GotErrMsg]), + fail + end; {error, OtherReason} -> - io:format("error not correctly reported: got ~p instead", [OtherReason]), + ct:pal("Query: ~s\nError not correctly reported: got ~p instead\n", [Query, OtherReason]), fail; NonError -> - io:format("error not detected: got ~p instead", [NonError]), + ct:pal("Query: ~s\nError not detected: got ~p instead\n", [Query, NonError]), fail end. diff --git a/tests/ts_qbuf_util.hrl b/tests/ts_qbuf_util.hrl index 3f17b6e0f..3fd4593e8 100644 --- a/tests/ts_qbuf_util.hrl +++ b/tests/ts_qbuf_util.hrl @@ -27,6 +27,7 @@ -define(ORDBY_COLS, ["a", "b", "c", "d", "e", undefined]). %% error codes as defined in riak_kv_ts_svc.erl +-define(E_SUBMIT, 1001). -define(E_SELECT_RESULT_TOO_BIG, 1022). -define(E_QBUF_CREATE_ERROR, 1023). -define(E_QBUF_LDB_ERROR, 1024). diff --git a/tests/ts_simple_invdist_funs_SUITE.erl b/tests/ts_simple_invdist_funs_SUITE.erl new file mode 100644 index 000000000..5eae61680 --- /dev/null +++ b/tests/ts_simple_invdist_funs_SUITE.erl @@ -0,0 +1,520 @@ +% ------------------------------------------------------------------- +%% +%% Copyright (c) 2017 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%% @doc Inverse distribution functions (PERCENTILE, MEDIAN) + +-module(ts_simple_invdist_funs_SUITE). + +-export([suite/0, init_per_suite/1, groups/0, all/0]). +-export([query_invdist_selftest/1, + query_invdist_percentile/1, + query_invdist_percentile_backends/1, + query_invdist_percentile_multiple/1, + query_invdist_median/1, + query_invdist_mode/1, + query_invdist_errors/1]). + +-export([percentile_disc/3, percentile_cont/3]). %% make them 'used' for erlc + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include("ts_qbuf_util.hrl"). + +-define(TABLE_A, "table_with_regular_keys"). +-define(TABLE_D, "table_with_descending_keys"). + +suite() -> + [{timetrap, {minutes, 5}}]. + +init_per_suite(Cfg) -> + Cluster = ts_setup:start_cluster(1), + C = rt:pbc(hd(Cluster)), + Data = make_data(), + ok = create_table(C, ?TABLE_A), + ok = create_table_desc(C, ?TABLE_D), + ok = insert_data(C, ?TABLE_A, Data), + ok = insert_data(C, ?TABLE_D, Data), + [{cluster, Cluster}, + {data, Data} | Cfg]. + +groups() -> + []. + +all() -> + [ + query_invdist_selftest, + query_invdist_percentile, + query_invdist_percentile_multiple, + query_invdist_percentile_backends, + query_invdist_median, + query_invdist_mode, + query_invdist_errors + ]. + + +%% test cases +%% ------------------------------- + +query_invdist_selftest(_Cfg) -> + Data = + [{X} || X <- lists:sort(pseudo_random_data())], + {GotDisc, GotCont} = + lists:unzip( + [{round(100 * percentile_disc("a", Pc/100, Data)), + round(100 * percentile_cont("a", Pc/100, Data))} || Pc <- lists:seq(1, 97)]), + {NeedDisc, NeedCont} = + lists:unzip( + [{round(X * 100), round(Y * 100)} || {X, Y} <- externally_verified_percentiles()]), + io:format("~p ~p\n", [length(GotDisc), length(NeedDisc)]), + ?assertEqual(GotDisc, NeedDisc), + ?assertEqual(GotCont, NeedCont). + + + +query_invdist_percentile(Cfg) -> + C = rt:pbc(hd(proplists:get_value(cluster, Cfg))), + Data = proplists:get_value(data, Cfg), + lists:foreach( + fun(Col) -> + check_column(C, Col, Data) orelse + ct:fail("") + end, + ["a", "b", "c", "d", "e"]). + + +query_invdist_percentile_backends(Cfg) -> + Node = hd(proplists:get_value(cluster, Cfg)), + C = rt:pbc(Node), + Data = proplists:get_value(data, Cfg), + + WorkF = + fun() -> + lists:foreach( + fun(Col) -> + check_column(C, Col, Data) orelse + ct:fail("") + end, + ["a", "b", "c", "d", "e"]) + end, + + rpc:call(Node, code, add_patha, [filename:join([rt_local:home_dir(), "../../ebin"])]), + rt_intercept:load_code(Node), + + load_intercept(Node, C, {riak_kv_qry_buffers, [{{can_afford_inmem, 1}, can_afford_inmem_yes}]}), + ct:log("----------- all inmem", []), + WorkF(), + load_intercept(Node, C, {riak_kv_qry_buffers, [{{can_afford_inmem, 1}, can_afford_inmem_no}]}), + ct:log("----------- all ldb", []), + WorkF(), + load_intercept(Node, C, {riak_kv_qry_buffers, [{{can_afford_inmem, 1}, can_afford_inmem_random}]}), + ct:log("----------- random", []), + WorkF(), + rt_intercept:clean(Node, riak_kv_qry_buffers), + ok. + + +query_invdist_percentile_multiple(Cfg) -> + C = rt:pbc(hd(proplists:get_value(cluster, Cfg))), + Data = proplists:get_value(data, Cfg), + {Col1, Col2, Pc1, Pc2} = {"b", "b", 0.22, 0.77}, + Query = make_query(fmt("percentile_disc(~s, ~g), percentile_cont(~s, ~g)", [Col1, Pc1, Col2, Pc2])), + {ok, {_Cols, [{Got1, Got2}]}} = + riakc_ts:query(C, Query, [], undefined, []), + {Need1, Need2} = {percentile_disc(Col1, Pc1, order_by(Col1, Data)), + percentile_cont(Col2, Pc2, order_by(Col2, Data))}, + ct:log("Query \"~s\"", [Query]), + case {Got1 == Need1, Got2 == Need2} of + {true, true} -> + ok; + _ -> + ct:fail("PERCENTILE_DISC vs _CONT: Got {~p, ~p}, Need {~p, ~p}\n", [Got1, Got2, Need1, Need2]) + end. + + +query_invdist_median(Cfg) -> + C = rt:pbc(hd(proplists:get_value(cluster, Cfg))), + lists:foreach( + fun(Col) -> + Query = make_query(fmt("percentile_disc(~s, 0.5), median(~s)", [Col, Col])), + {ok, {_Cols, [{Got1, Got2}]}} = + riakc_ts:query(C, Query, [], undefined, []), + ct:log("Query \"~s\"", [Query]), + case Got1 == Got2 of + true -> + ok; + _ -> + ct:fail("MEDIAN: Got {~p, ~p}, Need equal\n", [Got1, Got2]) + end + end, + ["a", "b", "c", "d", "e"]). + + +query_invdist_mode(Cfg) -> + C = rt:pbc(hd(proplists:get_value(cluster, Cfg))), + Data = proplists:get_value(data, Cfg), + lists:foreach( + fun(Col) -> + SortedData = order_by(Col, Data), + Query = make_query(fmt("mode(~s)", [Col])), + {ok, {_Cols, [{Got}]}} = + riakc_ts:query(C, Query, [], undefined, []), + ct:log("Query \"~s\"", [Query]), + Need = mode(Col, SortedData), + case Got == Need of + true -> + ok; + _ -> + ct:fail("MODE: Got ~p, Need ~p\n", [Got, Need]) + end + end, + ["a", "b", "c", "d", "e"]). + + +query_invdist_errors(Cfg) -> + lists:foreach( + fun({Select, Extra, ErrPat}) -> + Qry = make_query(?TABLE_A, Select, Extra), + ?assertEqual( + ok, + ts_qbuf_util:ack_query_error(Cfg, Qry, ?E_SUBMIT, ErrPat)) + end, + [{"percentile_disc(nxcol, 0.2)", [], + "Unknown column \"nxcol\""}, + {"median(a), b", [], + "Inverse distribution functions .* cannot be used with other columns in SELECT clause"}, + {"percentile_disc(1)", [], + "Function PERCENTILE_DISC/1 called with 1 argument"}, + {"percentile_disc(a, 1.2)", [], + "Invalid argument 2 in call to function PERCENTILE_DISC"}, + {"percentile_disc(a, 0.1), percentile_disc(b, 0.2)", [], + "Multiple inverse distribution functions .* must all have the same column argument"}, + {"percentile_disc(a, 1+c)", [], + "Function 'PERCENTILE_DISC' called with arguments of the wrong type"}, + {"percentile_disc(a, 1.1+c)", [], + "Inverse distribution functions .* must have a static const expression for its parameters"}, + {"percentile_disc(a, 1.1/0)", [], + "Invalid expression passed as parameter for inverse distribution function"}, + {"percentile_disc(a, 0.3), avg(b)", [], + "Inverse distribution functions .* cannot be used with GROUP BY clause or other aggregating window functions"}, + {"percentile_disc(a, 0.1)", "group by a", + "Inverse distribution functions .* cannot be used with GROUP BY clause or other aggregating window functions"}, + {"percentile_disc(a, 0.1)", "order by a", + "Inverse distribution functions .* cannot be used with any of ORDER BY, LIMIT or OFFSET clauses"}, + {"percentile_disc(a, 0.1)", "limit 1", + "Inverse distribution functions .* cannot be used with any of ORDER BY, LIMIT or OFFSET clauses"}]). + + +%% tested functions implememnted independently +%% ------------------------------- + +percentile_disc(F, Pc, Data_) -> + Col = remove_nulls(F, Data_), + RN = (1 + (Pc * (length(Col) - 1))), + lists:nth(trunc(RN), Col). + +percentile_cont(F, Pc, Data_) -> + Col = remove_nulls(F, Data_), + RN = (1 + (Pc * (length(Col) - 1))), + {LoRN, HiRN} = {trunc(RN), ceil(RN)}, + case LoRN == HiRN of + true -> + lists:nth(LoRN, Col); + false -> + LoVal = lists:nth(LoRN, Col), + HiVal = lists:nth(HiRN, Col), + (HiRN - RN) * LoVal + (RN - LoRN) * HiVal + end. + +ceil(X) -> + T = trunc(X), + case X - T == 0 of + true -> T; + false -> T + 1 + end. + +mode(F, Data_) -> + Col = remove_nulls(F, Data_), + Min = lists:nth(1, Col), + largest_bin(Min, Col). + +remove_nulls([FChar|_], Data_) -> + FNo = 1 + (FChar - $a), + [element(FNo, D) || D <- Data_, element(FNo, D) /= []]. + +largest_bin(X1, Data) -> + largest_bin_({X1, 1, X1, 1}, 2, Data). + +largest_bin_({LargestV, _, _, _}, Pos, Data) when Pos > length(Data) -> + LargestV; +largest_bin_({LargestV, LargestC, CurrentV, CurrentC}, Pos, Data) -> + case lists:nth(Pos, Data) of + V when V == CurrentV -> + largest_bin_({LargestV, LargestC, + CurrentV, CurrentC + 1}, Pos + 1, Data); + V when V > CurrentV, + CurrentC > LargestC -> + largest_bin_({CurrentV, CurrentC, %% now these be largest + V, 1}, Pos + 1, Data); + V when V > CurrentV, + CurrentC =< LargestC -> + largest_bin_({LargestV, LargestC, %% keep largest, reset current + V, 1}, Pos + 1, Data) + end. + + +%% supporting functions +%% ------------------------------- + +create_table(Client, Table) -> + DDL = " +create table " ++ Table ++ " +(a timestamp not null, + b double, + c sint64, + d sint64, + e sint64, + primary key ((quantum(a, 10, h)), a))", + {ok, {[], []}} = riakc_ts:query(Client, DDL), + ok. + +create_table_desc(Client, Table) -> + DDL = " +create table " ++ Table ++ " +(a timestamp not null, + b double, + c sint64, + d sint64, + e sint64, + primary key ((quantum(a, 10, h)), a desc))", + {ok, {[], []}} = riakc_ts:query(Client, DDL), + ok. + +data_generator(I) -> + {?TIMEBASE + (I + 1) * 1000, + 100 * math:cos(float(I) / 10 * math:pi()), + case I rem 5 of 0 -> []; _ -> I end, %% sprinkle some NULLs + I+1, + -I + }. + +make_data() -> + [data_generator(I) || I <- lists:seq(0, ?LIFESPAN)]. + +insert_data(_C, _Table, []) -> + ok; +insert_data(C, Table, Data) -> + Batch = lists:sublist(Data, 50), + ok = riakc_ts:put(C, Table, Batch), + case catch lists:nthtail(50, Data) of + Rest when is_list(Rest) -> + insert_data(C, Table, Rest); + _ -> + ok + end. + +make_query(Select) -> + make_query(?TABLE_A, Select, ""). +make_query(Table, Select) -> + make_query(Table, Select, ""). +make_query(Table, Select, Extra) -> + fmt("select ~s from ~s where a >= ~b and a <= ~b~s", + [Select, Table, + ?TIMEBASE, ?TIMEBASE * 1000, + [" " ++ Extra || Extra /= []]]). + + + +check_column(C, Col, Data) -> + SortedData = order_by(Col, Data), + Combos = + [{PercentileVariety, Parm, Parm_s} || + {Parm, Parm_s} <- [{0.24, "0.24"}, + {0.11, "0.11"}, + {0.0, "0.0"}, + {0.8, "0.8*1"}, + {1.0, "1/1.0"}, + {0.36, "(3.6/10)"}, + {0.40, "0.5 - 1 * 0.1"}], + PercentileVariety <- [percentile_cont, percentile_disc]], + Checker = + fun(Q, Need) -> + ct:log("Query \"~s\"", [Q]), + {ok, {_Cols, [{Got}]}} = + riakc_ts:query(C, Q, [], undefined, []), + case Got == Need of + true -> + ok; + false -> + ct:fail("Got ~p, Need ~p\n", [Got, Need]) + end + end, + ok == lists:foreach( + fun({PercentileFun, Pc, Pc_s}) -> + Need = apply(?MODULE, PercentileFun, [Col, Pc, SortedData]), + ok = Checker(make_query(?TABLE_A, fmt("~s(~s, ~s)", [PercentileFun, Col, Pc_s])), Need), + ok = Checker(make_query(?TABLE_D, fmt("~s(~s, ~s)", [PercentileFun, Col, Pc_s])), Need) + end, + Combos). + +order_by([FChar|_], Data) -> + FNo = 1 + (FChar - $a), + lists:sort( + fun(Row1, Row2) -> + element(FNo, Row1) =< element(FNo, Row2) + end, + Data). + + +%% intercepts + +load_intercept(Node, C, Intercept) -> + ok = rt_intercept:add(Node, Intercept), + %% when code changes underneath the riak_kv_qry_buffers + %% gen_server, it gets reinitialized. We need to probe it with a + %% dummy query until it becomes ready. + rt_intercept:wait_until_loaded(Node), + wait_until_qbuf_mgr_reinit(C). + +wait_until_qbuf_mgr_reinit(C) -> + ProbingQuery = make_query("*"), + case riakc_ts:query(C, ProbingQuery, [], undefined, []) of + {ok, _Data} -> + ok; + {error, {ErrCode, _NotReadyMessage}} + when ErrCode == ?E_QBUF_CREATE_ERROR; + ErrCode == ?E_QBUF_INTERNAL_ERROR -> + timer:sleep(100), + wait_until_qbuf_mgr_reinit(C) + end. + + +fmt(F, A) -> + lists:flatten(io_lib:format(F, A)). + +pseudo_random_data() -> + [float(X) || X <- [1,2,3,4,5,7,0,11,-4,2,2,19,3,11,17,4,9]]. + +externally_verified_percentiles() -> + %% these are generated with this script: + %% #!/bin/env python + %% import numpy as np + %% a = np.array([1,2,3,4,5,7,0,11,-4,2,2,19,3,11,17,4,9]) + %% for p in range(1,99): + %% disc = np.percentile(a, p, interpolation='lower') + %% cont = np.percentile(a, p, interpolation='linear') + %% print("{0:.3f}, {1:.3f},".format(disc, cont)) + + [{-4.000, -3.360}, + {-4.000, -2.720}, + {-4.000, -2.080}, + {-4.000, -1.440}, + {-4.000, -0.800}, + {-4.000, -0.160}, + {0.000, 0.120}, + {0.000, 0.280}, + {0.000, 0.440}, + {0.000, 0.600}, + {0.000, 0.760}, + {0.000, 0.920}, + {1.000, 1.080}, + {1.000, 1.240}, + {1.000, 1.400}, + {1.000, 1.560}, + {1.000, 1.720}, + {1.000, 1.880}, + {2.000, 2.000}, + {2.000, 2.000}, + {2.000, 2.000}, + {2.000, 2.000}, + {2.000, 2.000}, + {2.000, 2.000}, + {2.000, 2.000}, + {2.000, 2.000}, + {2.000, 2.000}, + {2.000, 2.000}, + {2.000, 2.000}, + {2.000, 2.000}, + {2.000, 2.000}, + {2.000, 2.120}, + {2.000, 2.280}, + {2.000, 2.440}, + {2.000, 2.600}, + {2.000, 2.760}, + {2.000, 2.920}, + {3.000, 3.000}, + {3.000, 3.000}, + {3.000, 3.000}, + {3.000, 3.000}, + {3.000, 3.000}, + {3.000, 3.000}, + {3.000, 3.040}, + {3.000, 3.200}, + {3.000, 3.360}, + {3.000, 3.520}, + {3.000, 3.680}, + {3.000, 3.840}, + {4.000, 4.000}, + {4.000, 4.000}, + {4.000, 4.000}, + {4.000, 4.000}, + {4.000, 4.000}, + {4.000, 4.000}, + {4.000, 4.000}, + {4.000, 4.120}, + {4.000, 4.280}, + {4.000, 4.440}, + {4.000, 4.600}, + {4.000, 4.760}, + {4.000, 4.920}, + {5.000, 5.160}, + {5.000, 5.480}, + {5.000, 5.800}, + {5.000, 6.120}, + {5.000, 6.440}, + {5.000, 6.760}, + {7.000, 7.080}, + {7.000, 7.400}, + {7.000, 7.720}, + {7.000, 8.040}, + {7.000, 8.360}, + {7.000, 8.680}, + {9.000, 9.000}, + {9.000, 9.320}, + {9.000, 9.640}, + {9.000, 9.960}, + {9.000, 10.280}, + {9.000, 10.600}, + {9.000, 10.920}, + {11.000, 11.000}, + {11.000, 11.000}, + {11.000, 11.000}, + {11.000, 11.000}, + {11.000, 11.000}, + {11.000, 11.000}, + {11.000, 11.480}, + {11.000, 12.440}, + {11.000, 13.400}, + {11.000, 14.360}, + {11.000, 15.320}, + {11.000, 16.280}, + {17.000, 17.080}, + {17.000, 17.400}, + {17.000, 17.720}, + {17.000, 18.040} + ]. diff --git a/tests/ts_simple_pb_security_SUITE.erl b/tests/ts_simple_pb_security_SUITE.erl index a4ca869ef..4d0f9f914 100644 --- a/tests/ts_simple_pb_security_SUITE.erl +++ b/tests/ts_simple_pb_security_SUITE.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2016 Basho Technologies, Inc. +%% Copyright (c) 2016-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -31,6 +31,9 @@ %%-------------------------------------------------------------------- suite() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + [{timetrap,{minutes,10}}]. init_per_suite(Config) -> diff --git a/tests/ts_updown_util.erl b/tests/ts_updown_util.erl index 7f5b62706..813eeda46 100644 --- a/tests/ts_updown_util.erl +++ b/tests/ts_updown_util.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2016 Basho Technologies, Inc. +%% Copyright (c) 2016, 2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -50,6 +50,7 @@ -module(ts_updown_util). -export([ + caps_to_ensure/1, convert_riak_conf_to_previous/1, maybe_shutdown_client_node/1, setup/1, @@ -581,31 +582,87 @@ wait_until_active_table(TargetNode, PrevClientNode, Table, N) -> make_msg(Format, Payload) -> list_to_binary(fmt(Format, Payload)). + +get_riak_release_in_slot(VsnSlot) -> + case rtdev:get_version(VsnSlot) of + unknown -> + ct:fail("Failed to determine riak version in '~s' slot", [VsnSlot]); + Known -> + case re:run(Known, "riak_ts-(\\d+)\\.(\\d+)\\.(\\d+)", [{capture, all_but_first, list}]) of + {match, [V1, V2, V3]} -> + {list_to_integer(V1), + list_to_integer(V2), + list_to_integer(V3)}; + nomatch -> + ct:fail("Failed to parse riak version in '~s' slot", [VsnSlot]) + end + end. + +%% --------------------------------- + +%% Dealing with case-by-case upgrade particulars, such as: +%% +%% * newly introduced keys in riak.conf that need to be deleted on +%% downgrade; +%% +%% * capabilities that need to be ensured before running the tests +%% (arguments to `wait_until_capability`). + %% We need to comment out those settings which appear in version %% 1.x. For version 1.x-1 to work with riak.conf initially created in %% 1.x, the offending settings need to be deleted. We do it here, by %% commenting them out. -%% riak.conf created under 1.6 cannot be read by 1.5 because of new keys: -%% riak_kv.query.timeseries.qbuf_inmem_max, -%% -%% riak.conf created under 1.5 cannot be read by 1.4 because of new keys: -%% riak_kv.query.timeseries.max_returned_data_size, -%% riak_kv.query.timeseries.qbuf_soft_watermark, -%% riak_kv.query.timeseries.qbuf_hard_watermark, -%% riak_kv.query.timeseries.qbuf_expire_ms, -%% riak_kv.query.timeseries.qbuf_incomplete_release_ms -%% convert_riak_conf_to_previous(Config) -> DatafPath = ?CFG(new_data_dir, Config), RiakConfPath = filename:join(DatafPath, "../etc/riak.conf"), - {ok, Content0} = file:read_file(RiakConfPath), - Content9 = - re:replace( - Content0, - <<"^riak_kv.query.timeseries.qbuf_inmem_max">>, - <<"#\\1">>, [global, multiline, {return, binary}]), - ok = file:write_file(RiakConfPath, Content9). + {ok, Contents0} = file:read_file(RiakConfPath), + Contents9 = + lists:foldl( + fun(KeyToDelete, Contents) -> + re:replace(Contents, ["^", KeyToDelete], "#\\1", + [global, multiline, {return, list}]) + end, + Contents0, + get_riak_conf_new_keys()), + ok = file:write_file(RiakConfPath, Contents9). + +%% When a new release is cut, register newly introduced keys here: +get_riak_conf_new_keys() -> + %% the current version may have not been tagged yet, so look at + %% previous version + case get_riak_release_in_slot(previous) of + {1, 5, _} -> + ["riak_kv.query.timeseries.qbuf_inmem_max"]; + {1, 4, _} -> + ["riak_kv.query.timeseries.max_returned_data_size", + "riak_kv.query.timeseries.qbuf_soft_watermark", + "riak_kv.query.timeseries.qbuf_hard_watermark", + "riak_kv.query.timeseries.qbuf_expire_ms", + "riak_kv.query.timeseries.qbuf_incomplete_release_ms"] + end. + +%% Wait for these capabilities to settle at these versions at the end +%% of upgrade/downgrade: +caps_to_ensure(full) -> + case get_riak_release_in_slot(previous) of + {1, 5, _} -> + []; %% no new caps in 1.6 since 1.5 + {1, 4, _} -> + [{{riak_kv, sql_select_version}, v3}, + {{riak_kv, riak_ql_ddl_rec_version}, v2}, + {{riak_kv, decode_query_results_at_vnode}, true}] + end; +caps_to_ensure(degraded) -> + case get_riak_release_in_slot(previous) of + {1, 5, _} -> + []; %% no new caps in 1.6 since 1.5 + {1, 4, _} -> + [{{riak_kv, sql_select_version}, v2}, + {{riak_kv, riak_ql_ddl_rec_version}, v1}, + {{riak_kv, decode_query_results_at_vnode}, false}] + end. + %% Keep the old convert_riak_conf_to_previous functions around, for %% reference and occasional test rerun diff --git a/tests/verify_aae.erl b/tests/verify_aae.erl index 8a2d61cb9..06e7ec9d2 100644 --- a/tests/verify_aae.erl +++ b/tests/verify_aae.erl @@ -47,7 +47,8 @@ -define(CFG, [{riak_kv, [ - {sweep_tick, 3000}, + {sweep_tick, 1023}, + {sweep_concurrency, 8}, % Speedy AAE configuration {anti_entropy, {on, []}}, {anti_entropy_build_limit, {100, 1000}}, @@ -136,7 +137,7 @@ verify_aae(Nodes) -> start_tree_rebuilds(Nodes) -> rpc:multicall(Nodes, application, set_env, [riak_kv, anti_entropy_expire, - 15 * 1000]). + 30 * 1000]). acc_preflists(Pl, PlCounts) -> lists:foldl(fun(Idx, D) -> diff --git a/tests/verify_api_timeouts.erl b/tests/verify_api_timeouts.erl index 0a6851cfd..1ddbe878f 100644 --- a/tests/verify_api_timeouts.erl +++ b/tests/verify_api_timeouts.erl @@ -8,6 +8,9 @@ -define(NUM_KEYS, 1000). confirm() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + %% test requires allow_mult=false b/c of rt:systest_read [Node] = rt:build_cluster(1), rt:wait_until_pingable(Node), diff --git a/tests/verify_capabilities.erl b/tests/verify_capabilities.erl index a1ff966df..22407f35b 100644 --- a/tests/verify_capabilities.erl +++ b/tests/verify_capabilities.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2012-2013 Basho Technologies, Inc. +%% Copyright (c) 2012-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -25,6 +25,9 @@ %% 1.4 {riak_kv, handoff_data_encoding} -> [encode_raw, encode_zlib] %% 1.3 {riak_kv, anti_entropy} -> [disabled, enabled_v1] confirm() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + lager:info("Deploying mixed set of nodes"), Legacy = case lists:member(legacy, rt:versions()) of true -> legacy; diff --git a/tests/verify_dt_data_upgrade.erl b/tests/verify_dt_data_upgrade.erl index ac9435676..907ce4b26 100644 --- a/tests/verify_dt_data_upgrade.erl +++ b/tests/verify_dt_data_upgrade.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2016 Basho Technologies, Inc. +%% Copyright (c) 2016-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -47,7 +47,7 @@ confirm() -> TestMetaData = riak_test_runner:metadata(), - OldVsn = proplists:get_value(upgrade_version, TestMetaData, lts), + OldVsn = proplists:get_value(upgrade_version, TestMetaData, legacy), NumNodes = 4, Vsns = [{OldVsn, ?CONFIG} || _ <- lists:seq(1, NumNodes)], @@ -116,7 +116,7 @@ confirm() -> ?assertEqual(FetchSet2, FetchSet3), %% Downgrade All Nodes and Compare - downgrade(Nodes, lts), + downgrade(Nodes, legacy), %% Create PB connection. Pid3 = rt:pbc(rt:select_random(Nodes)), diff --git a/tests/verify_job_enable_ac.erl b/tests/verify_job_enable_ac.erl index 0acf920cb..faa4091a3 100644 --- a/tests/verify_job_enable_ac.erl +++ b/tests/verify_job_enable_ac.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2016 Basho Technologies, Inc. +%% Copyright (c) 2016-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -36,6 +36,9 @@ [Class || {Class, Enabled} <- ?JOB_CLASS_DEFAULTS, Enabled]). confirm() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + lager:info("Deploying 1 node"), rt:set_backend(eleveldb), [Node] = rt:build_cluster(1, ?CFG), diff --git a/tests/verify_job_enable_rc.erl b/tests/verify_job_enable_rc.erl index 511690bd3..076302888 100644 --- a/tests/verify_job_enable_rc.erl +++ b/tests/verify_job_enable_rc.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2016 Basho Technologies, Inc. +%% Copyright (c) 2016-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -47,6 +47,9 @@ %% =================================================================== confirm() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + Configs = [ {current, {cuttlefish, ?COMMON_CONFIG ++ config(?TEST_OPS, Bool, [])}} diff --git a/tests/verify_listkeys.erl b/tests/verify_listkeys.erl index 33af44937..66a77f54a 100644 --- a/tests/verify_listkeys.erl +++ b/tests/verify_listkeys.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2012 Basho Technologies, Inc. +%% Copyright (c) 2012-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -29,6 +29,9 @@ -define(UNDEFINED_BUCKET_TYPE, <<"880bf69d-5dab-44ee-8762-d24c6f759ce1">>). confirm() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + [Node1, Node2, Node3, Node4] = Nodes = rt:deploy_nodes(4), ?assertEqual(ok, rt:wait_until_nodes_ready(Nodes)), diff --git a/tests/verify_listkeys_eqcfsm.erl b/tests/verify_listkeys_eqcfsm.erl index d8f5f01ee..66d209674 100644 --- a/tests/verify_listkeys_eqcfsm.erl +++ b/tests/verify_listkeys_eqcfsm.erl @@ -26,6 +26,9 @@ %% riak_test callback %% ==================================================================== confirm() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + ?assert(eqc:quickcheck(eqc:numtests(?NUM_TESTS, ?MODULE:prop_test()))), pass. %% ==================================================================== diff --git a/tests/verify_mr_prereduce_node_down.erl b/tests/verify_mr_prereduce_node_down.erl index 265dd7a29..796c7b2e5 100644 --- a/tests/verify_mr_prereduce_node_down.erl +++ b/tests/verify_mr_prereduce_node_down.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2013 Basho Technologies, Inc. +%% Copyright (c) 2013-2017 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -42,6 +42,9 @@ %% @doc riak_test callback confirm() -> + %% Allow listing of buckets and keys for testing + application:set_env(riakc, allow_listing, true), + NodeCount = 4, lager:info("Build ~b-node cluster", [NodeCount]), [Primary,ToKill|_] = rt:build_cluster(NodeCount), diff --git a/tests/verify_riak_stats.erl b/tests/verify_riak_stats.erl index 36831b473..7ef601ea1 100644 --- a/tests/verify_riak_stats.erl +++ b/tests/verify_riak_stats.erl @@ -415,6 +415,7 @@ common_stats() -> <<"asn1_version">>, <<"basho_stats_version">>, <<"bitcask_version">>, + <<"chronos_version">>, <<"clique_version">>, <<"cluster_info_version">>, <<"compiler_version">>, diff --git a/tests/yz_core_properties_create_unload.erl b/tests/yz_core_properties_create_unload.erl deleted file mode 100644 index 186227e64..000000000 --- a/tests/yz_core_properties_create_unload.erl +++ /dev/null @@ -1,170 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2014 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%%------------------------------------------------------------------- --module(yz_core_properties_create_unload). --compile(export_all). --include_lib("eunit/include/eunit.hrl"). - --define(CFG, [{riak_kv, - [ - %% allow AAE to build trees and exchange rapidly - {anti_entropy_build_limit, {1000, 1000}}, - {anti_entropy_concurrency, 64}, - {anti_entropy_tick, 1000} - ]}, - {yokozuna, - [ - {enabled, true}, - {anti_entropy_tick, 1000} - ]}]). --define(INDEX, <<"test_idx_core">>). --define(TYPE, <<"data">>). --define(BUCKET, {?TYPE, <<"test_bkt_core">>}). --define(SEQMAX, 100). - -confirm() -> - Cluster = rt:build_cluster(4, ?CFG), - rt:wait_for_cluster_service(Cluster, yokozuna), - - %% Generate keys, YZ only supports UTF-8 compatible keys - Keys = [<> || N <- lists:seq(1, ?SEQMAX), - not lists:any(fun(E) -> E > 127 end, - binary_to_list(<>))], - KeyCount = length(Keys), - - %% Randomly select a subset of the test nodes to remove - %% core.properties from - RandNodes = rt:random_sublist(Cluster, 3), - - %% Select one of the modified nodes as a client endpoint - Node = rt:select_random(RandNodes), - Pid = rt:pbc(Node), - riakc_pb_socket:set_options(Pid, [queue_if_disconnected]), - - %% Create a search index and associate with a bucket - lager:info("Create and set Index ~p for Bucket ~p~n", [?INDEX, ?BUCKET]), - _ = riakc_pb_socket:create_search_index(Pid, ?INDEX), - yokozuna_rt:wait_for_index(Cluster, ?INDEX), - - ok = rt:create_and_activate_bucket_type(Node, - ?TYPE, - [{search_index, ?INDEX}]), - - rt:wait_until_bucket_type_visible(Cluster, ?TYPE), - - %% Write keys and wait for soft commit - lager:info("Writing ~p keys", [KeyCount]), - [ok = rt:pbc_write(Pid, ?BUCKET, Key, Key, "text/plain") || Key <- Keys], - yokozuna_rt:commit(Cluster, ?INDEX), - - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount), - - - test_core_props_removal(Cluster, RandNodes, KeyCount, Pid), - test_remove_index_dirs(Cluster, RandNodes, KeyCount, Pid), - test_remove_segment_infos_and_rebuild(Cluster, RandNodes, KeyCount, Pid), - test_brutal_kill_and_delete_index_dirs(Cluster, RandNodes, KeyCount, Pid), - - riakc_pb_socket:stop(Pid), - - pass. - -test_core_props_removal(Cluster, RandNodes, KeyCount, Pid) -> - lager:info("Remove core.properties file in each index data dir"), - remove_core_props(RandNodes, ?INDEX), - - yokozuna_rt:check_exists(Cluster, ?INDEX), - - lager:info("Write one more piece of data"), - ok = rt:pbc_write(Pid, ?BUCKET, <<"foo">>, <<"foo">>, "text/plain"), - yokozuna_rt:commit(Cluster, ?INDEX), - - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount+1). - -test_remove_index_dirs(Cluster, RandNodes, KeyCount, Pid) -> - lager:info("Remove index directories on each node and let them recreate/reindex"), - yokozuna_rt:remove_index_dirs(RandNodes, ?INDEX, [riak_kv, yokozuna]), - - yokozuna_rt:check_exists(Cluster, ?INDEX), - - yokozuna_rt:expire_trees(Cluster), - yokozuna_rt:wait_for_aae(Cluster), - - lager:info("Write second piece of data"), - ok = rt:pbc_write(Pid, ?BUCKET, <<"food">>, <<"foody">>, "text/plain"), - yokozuna_rt:commit(Cluster, ?INDEX), - - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount+2). - -test_remove_segment_infos_and_rebuild(Cluster, RandNodes, KeyCount, Pid) -> - lager:info("Remove segment info files in each index data dir"), - remove_segment_infos(RandNodes, ?INDEX), - - lager:info("To fix, we remove index directories on each node and let them recreate/reindex"), - - yokozuna_rt:remove_index_dirs(RandNodes, ?INDEX, [riak_kv, yokozuna]), - - yokozuna_rt:check_exists(Cluster, ?INDEX), - - yokozuna_rt:expire_trees(Cluster), - yokozuna_rt:wait_for_aae(Cluster), - - lager:info("Write third piece of data"), - ok = rt:pbc_write(Pid, ?BUCKET, <<"baz">>, <<"bar">>, "text/plain"), - yokozuna_rt:commit(Cluster, ?INDEX), - - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount+3). - -test_brutal_kill_and_delete_index_dirs(Cluster, RandNodes, KeyCount, Pid) -> - lager:info("Remove index directories on each node and let them recreate/reindex"), - yokozuna_rt:brutal_kill_remove_index_dirs(RandNodes, ?INDEX, [riak_kv, yokozuna]), - - yokozuna_rt:check_exists(Cluster, ?INDEX), - - yokozuna_rt:expire_trees(Cluster), - yokozuna_rt:wait_for_aae(Cluster), - - lager:info("Write fourth piece of data"), - ok = rt:pbc_write(Pid, ?BUCKET, <<"food">>, <<"foody">>, "text/plain"), - yokozuna_rt:commit(Cluster, ?INDEX), - - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 4). - -%% @doc Remove core properties file on nodes. -remove_core_props(Nodes, IndexName) -> - IndexDirs = [rpc:call(Node, yz_index, index_dir, [IndexName]) || - Node <- Nodes], - PropsFiles = [filename:join([IndexDir, "core.properties"]) || - IndexDir <- IndexDirs], - lager:info("Remove core.properties files: ~p, on nodes: ~p~n", - [PropsFiles, Nodes]), - [file:delete(PropsFile) || PropsFile <- PropsFiles], - ok. - -%% @doc Remove lucence segment info files to check if reindexing will occur -%% on re-creation/re-indexing. -remove_segment_infos(Nodes, IndexName) -> - IndexDirs = [rpc:call(Node, yz_index, index_dir, [IndexName]) || - Node <- Nodes], - SiPaths = [binary_to_list(filename:join([IndexDir, "data/index/*.si"])) || - IndexDir <- IndexDirs], - SiFiles = lists:append([filelib:wildcard(Path) || Path <- SiPaths]), - lager:info("Remove segment info files: ~p, on in dirs: ~p~n", - [SiFiles, IndexDirs]), - [file:delete(SiFile) || SiFile <- SiFiles]. diff --git a/tests/yz_crdt.erl b/tests/yz_crdt.erl deleted file mode 100644 index f2c114cb3..000000000 --- a/tests/yz_crdt.erl +++ /dev/null @@ -1,564 +0,0 @@ --module(yz_crdt). - --compile(export_all). --compile({parse_transform, rt_intercept_pt}). - --include_lib("eunit/include/eunit.hrl"). - --define(HARNESS, (rt_config:get(rt_harness))). --define(INDEX, <<"maps">>). --define(TYPE, <<"maps">>). --define(KEY, <<"Chris Meiklejohn">>). --define(BUCKET, {?TYPE, <<"testbucket">>}). --define(GET(K,L), proplists:get_value(K, L)). --define(N, 3). - --define(CONF, - [ - {riak_core, - [{ring_creation_size, 8}] - }, - {riak_kv, - [{delete_mode, keep}, - {anti_entropy_build_limit, {100, 1000}}, - {anti_entropy_concurrency, 8}, - {anti_entropy_tick, 1000}]}, - {yokozuna, - [{enabled, true}] - }]). - -confirm() -> - rt:set_advanced_conf(all, ?CONF), - - %% Configure cluster. - Nodes = rt:build_cluster(5, ?CONF), - - Node = rt:select_random(Nodes), - - %% Create PB connection. - Pid = rt:pbc(Node), - riakc_pb_socket:set_options(Pid, [queue_if_disconnected]), - - %% Create index. - riakc_pb_socket:create_search_index(Pid, ?INDEX, <<"_yz_default">>, []), - - %% Create bucket type for maps. - rt:create_and_activate_bucket_type(Node, - ?TYPE, - [{datatype, map}, - {n_val, ?N}, - {search_index, ?INDEX}]), - - lager:info("Write some sample data"), - test_sample_data(Pid, Nodes, ?BUCKET, ?KEY, ?INDEX), - lager:info("Search and Validate our CRDT writes/updates."), - ok = rt:wait_until(fun() -> validate_sample_data(Pid, ?KEY, ?INDEX) - end), - - lager:info("Test setting the register of a map twice to different values." - " (The # of results should still be 1)"), - test_repeat_sets(Pid, Nodes, ?BUCKET, ?INDEX, ?KEY), - ok = rt:wait_until(fun() -> validate_test_repeat_set(Pid, ?INDEX) - end), - - lager:info("FYI: delete_mode is on keep here to make sure YZ handles" - " deletes correctly throughout."), - lager:info("Test varying deletes operations"), - test_and_validate_delete(Pid, Nodes, ?BUCKET, ?INDEX, ?KEY), - - lager:info("Test to make sure yz AAE handles deletes/removes correctly"), - test_and_validate_delete_aae(Pid, Nodes, ?BUCKET, ?INDEX), - - lager:info("Test to make sure siblings don't exist after partition"), - test_siblings(Nodes, ?BUCKET, ?INDEX), - lager:info("Verify counts and operations after heal + transfers + commits"), - ok = rt:wait_until(fun() -> validate_test_siblings(Pid, ?BUCKET, ?INDEX) - end), - - %% Stop PB connection - riakc_pb_socket:stop(Pid), - - pass. - -test_sample_data(Pid, Cluster, Bucket, Key, Index) -> - Map1 = riakc_map:update( - {<<"name">>, register}, - fun(R) -> - riakc_register:set(Key, R) - end, riakc_map:new()), - Map2 = riakc_map:update( - {<<"interests">>, set}, - fun(S) -> - riakc_set:add_element(<<"thing">>, S) end, - Map1), - - ok = riakc_pb_socket:update_type( - Pid, - Bucket, - Key, - riakc_map:to_op(Map2)), - - drain_and_commit(Cluster, Index). - -%% @doc Test setting the register of a map twice to different values. -%% The # of results should still be 1. -test_repeat_sets(Pid, Cluster, Bucket, Index, Key) -> - {ok, M1} = riakc_pb_socket:fetch_type(Pid, Bucket, Key), - M2 = riakc_map:update( - {<<"update">>, register}, - fun(R) -> - riakc_register:set(<<"foo">>, R) - end, M1), - ok = riakc_pb_socket:update_type( - Pid, - Bucket, - Key, - riakc_map:to_op(M2)), - M3 = riakc_map:update( - {<<"update">>, register}, - fun(R) -> - riakc_register:set(<<"bar">>, R) - end, M1), - ok = riakc_pb_socket:update_type( - Pid, - Bucket, - Key, - riakc_map:to_op(M3)), - - drain_and_commit(Cluster, Index). - -%% @doc Tests varying deletes of within a CRDT map and checks for correct counts -%% - Remove registers, remove and add elements within a set -%% - Delete the map (associated w/ a key) -%% - Recreate objects in the map and delete the map again -test_and_validate_delete(Pid, Cluster, Bucket, Index, Key) -> - {ok, M1} = riakc_pb_socket:fetch_type(Pid, Bucket, Key), - - lager:info("Remove register from map"), - M2 = riakc_map:erase({<<"name">>, register}, M1), - - lager:info("Delete element from set (in map) & Add element to set"), - M3 = riakc_map:update( - {<<"interests">>, set}, - fun(S) -> - riakc_set:del_element(<<"thing">>, - riakc_set:add_element(<<"roses">>, S)) - end, M2), - - ok = riakc_pb_socket:update_type( - Pid, - Bucket, - Key, - riakc_map:to_op(M3)), - - drain_and_commit(Cluster, Index), - - lager:info("Search deleted/erased name_register:*"), - search_and_validate_found(Pid, Index, <<"name_register:*">>, 0), - - lager:info("Add another element to set (in map)"), - M4 = riakc_map:update( - {<<"interests">>, set}, - fun(S) -> - riakc_set:add_element(<<"pans">>, S) - end, M3), - - ok = riakc_pb_socket:update_type( - Pid, - Bucket, - Key, - riakc_map:to_op(M4)), - - drain_and_commit(Cluster, Index), - - lager:info("Search deleted interests_set:thing*"), - search_and_validate_found(Pid, Index, <<"interests_set:thing*">>, 0), - - lager:info("Delete key for map"), - ?assertEqual(ok, riakc_pb_socket:delete(Pid, Bucket, Key)), - - drain_and_commit(Cluster, Index), - - ?assertEqual({error, {notfound, map}}, - riakc_pb_socket:fetch_type(Pid, Bucket, Key)), - - lager:info("Search deleted map: *:*"), - search_and_validate_found(Pid, Index, <<"*:*">>, 0), - - lager:info("Recreate object and check counts..."), - - lager:info("Set a new register for map"), - M5 = riakc_map:update( - {<<"name">>, register}, - fun(R) -> - riakc_register:set(<<"hello">>, R) - end, riakc_map:new()), - - ok = riakc_pb_socket:update_type( - Pid, - Bucket, - Key, - riakc_map:to_op(M5)), - - drain_and_commit(Cluster, Index), - - {ok, M6} = riakc_pb_socket:fetch_type(Pid, Bucket, Key), - Keys = riakc_map:fetch_keys(M6), - ?assertEqual(1, length(Keys)), - ?assert(riakc_map:is_key({<<"name">>, register}, M6)), - - lager:info("Search recreated map: *:*"), - search_and_validate_found(Pid, Index, <<"*:*">>, 1), - - lager:info("Delete key for map again"), - ?assertEqual(ok, riakc_pb_socket:delete(Pid, Bucket, Key)), - - drain_and_commit(Cluster, Index), - - ?assertEqual({error, {notfound, map}}, - riakc_pb_socket:fetch_type(Pid, Bucket, Key)), - - lager:info("Search ~p deleted map: *:*", [Key]), - search_and_validate_found(Pid, Index, <<"*:*">>, 0). - -%% @doc Tests key/map delete and AAE -%% - Use intercept to trap yz_kv:delete_operation to skip over -%% - Makes sure that yz AAE handles tombstone on expire/exchange -%% - Recreate objects and check -test_and_validate_delete_aae(Pid, Cluster, Bucket, Index) -> - Key1 = <<"ohyokozuna">>, - M1 = riakc_map:update( - {<<"name">>, register}, - fun(R) -> - riakc_register:set(<<"jokes are">>, R) - end, riakc_map:new()), - ok = riakc_pb_socket:update_type( - Pid, - Bucket, - Key1, - riakc_map:to_op(M1)), - - Key2 = <<"ohriaksearch">>, - M2 = riakc_map:update( - {<<"name">>, register}, - fun(R) -> - riakc_register:set(<<"better explained">>, R) - end, riakc_map:new()), - ok = riakc_pb_socket:update_type( - Pid, - Bucket, - Key2, - riakc_map:to_op(M2)), - - drain_and_commit(Cluster, Index), - - lager:info("Add and load handle_delete_operation intercept"), - - [make_intercepts_tab(ANode) || ANode <- Cluster], - - [rt_intercept:add(ANode, {yz_solrq_helper, [{{get_ops_for_no_sibling_deletes, 3}, - handle_get_ops_for_no_sibling_deletes}]}) - || ANode <- Cluster], - [true = rpc:call(ANode, ets, insert, [intercepts_tab, {del_put, 0}]) || - ANode <- Cluster], - [rt_intercept:wait_until_loaded(ANode) || ANode <- Cluster], - - lager:info("Delete key ~p for map", [Key2]), - ?assertEqual(ok, riakc_pb_socket:delete(Pid, Bucket, Key2)), - ?assertEqual({error, {notfound, map}}, - riakc_pb_socket:fetch_type(Pid, Bucket, Key2)), - - drain_and_commit(Cluster, Index), - - lager:info("Search all results, expect extra b/c tombstone" - " and we've modified the delete op : *:*"), - search_and_validate_found(Pid, Index, <<"*:*">>, 2), - - lager:info("Expire and re-check"), - yokozuna_rt:expire_trees(Cluster), - yokozuna_rt:wait_for_full_exchange_round(Cluster, erlang:now()), - drain_and_commit(Cluster, Index), - - lager:info("Search all results, expect removed tombstone b/c AAE" - " should clean it up: *:*"), - search_and_validate_found(Pid, Index, <<"*:*">>, 1), - - lager:info("Recreate object and check counts"), - - M3 = riakc_map:update( - {<<"name">>, register}, - fun(R) -> - riakc_register:set(<<"hello again, is it me you're" - "looking for">>, R) - end, riakc_map:new()), - - ok = riakc_pb_socket:update_type( - Pid, - Bucket, - Key2, - riakc_map:to_op(M3)), - - drain_and_commit(Cluster, Index), - - {ok, M4} = riakc_pb_socket:fetch_type(Pid, Bucket, Key2), - Keys = riakc_map:fetch_keys(M4), - ?assertEqual(1, length(Keys)), - ?assert(riakc_map:is_key({<<"name">>, register}, M4)), - - lager:info("Search recreated map: *:*"), - search_and_validate_found(Pid, Index, <<"*:*">>, 2). - -%% @doc Tests sibling handling/merge when there's a partition -%% - Write/remove from separate partitions -%% - Verify counts and that CRDTs have no siblings, vtags, -%% after healing partitions. The CRDT map merges so the search -%% results be consistent. -test_siblings(Cluster, Bucket, Index) -> - Key1 = <<"Movies">>, - Key2 = <<"Games">>, - Set1 = <<"directors">>, - Set2 = <<"characters">>, - - %% make siblings - {P1, P2} = lists:split(1, Cluster), - - %% Create an object in Partition 1 and siblings in Partition 2 - lager:info("Create partition: ~p | ~p", [P1, P2]), - Partition = rt:partition(P1, P2), - - %% PB connections for accessing each side - Pid1 = rt:pbc(hd(P1)), - Pid2 = rt:pbc(hd(P2)), - - riakc_pb_socket:set_options(Pid1, [queue_if_disconnected, auto_reconnect]), - riakc_pb_socket:set_options(Pid2, [queue_if_disconnected, auto_reconnect]), - - %% P1 writes - lager:info("Writing to Partition 1 Set 1: Key ~p | Director ~p", - [Key1, <<"Kubrick">>]), - M1 = riakc_map:update( - {Set1, set}, - fun(S) -> - riakc_set:add_element(<<"Kubrick">>, S) - end, riakc_map:new()), - ok = riakc_pb_socket:update_type( - Pid1, - Bucket, - Key1, - riakc_map:to_op(M1)), - - lager:info("Writing to Partition 1 Set 1: Key ~p | Director ~p", - [Key1, <<"Demme">>]), - M2 = riakc_map:update( - {Set1, set}, - fun(S) -> - riakc_set:add_element(<<"Demme">>, S) - end, riakc_map:new()), - ok = riakc_pb_socket:update_type( - Pid1, - Bucket, - Key1, - riakc_map:to_op(M2)), - - %% P2 Siblings - lager:info("Writing to Partition 2 Set 2: Key ~p | Char ~p", - [Key2, <<"Sonic">>]), - M3 = riakc_map:update( - {Set2, set}, - fun(S) -> - riakc_set:add_element(<<"Sonic">>, S) - end, riakc_map:new()), - ok = riakc_pb_socket:update_type( - Pid2, - Bucket, - Key2, - riakc_map:to_op(M3)), - - lager:info("Delete key from Partition 2: Key ~p", [Key2]), - ok = riakc_pb_socket:delete(Pid2, Bucket, Key2), - - lager:info("Writing to Partition 2 Set 2: after delete: Key ~p | Char" - " ~p", [Key2, <<"Crash">>]), - M4 = riakc_map:update( - {Set2, set}, - fun(S) -> - riakc_set:add_element(<<"Crash">>, S) - end, riakc_map:new()), - ok = riakc_pb_socket:update_type( - Pid2, - Bucket, - Key2, - riakc_map:to_op(M4)), - - lager:info("Writing to Partition 2 Set 2: Key ~p | Char ~p", - [Key2, <<"Mario">>]), - M5 = riakc_map:update( - {Set2, set}, - fun(S) -> - riakc_set:add_element(<<"Mario">>, S) - end, riakc_map:new()), - ok = riakc_pb_socket:update_type( - Pid2, - Bucket, - Key2, - riakc_map:to_op(M5)), - - lager:info("Writing to Partition 2 Set 1: Key ~p | Director ~p", - [Key1, <<"Klimov">>]), - M6 = riakc_map:update( - {Set1, set}, - fun(S) -> - riakc_set:add_element(<<"Klimov">>, S) - end, riakc_map:new()), - ok = riakc_pb_socket:update_type( - Pid2, - Bucket, - Key1, - riakc_map:to_op(M6)), - - rt:heal(Partition), - rt:wait_until_transfers_complete(Cluster), - - drain_and_commit(Cluster, Index). - -validate_sample_data(Pid, Key, Index) -> - try - Thing = <<"thing">>, - - {ok, {search_results, Results1a, _, Found1}} = riakc_pb_socket:search( - Pid, Index, <<"name_register:Chris*">>), - ?assertEqual(1, Found1), - - ?assertEqual(?GET(<<"name_register">>, ?GET(Index, Results1a)), - Key), - ?assertEqual(?GET(<<"interests_set">>, ?GET(Index, Results1a)), - Thing), - - {ok, {search_results, Results2a, _, Found2}} = riakc_pb_socket:search( - Pid, Index, <<"interests_set:thing*">>), - ?assertEqual(1, Found2), - ?assertEqual(?GET(<<"name_register">>, ?GET(Index, Results2a)), - Key), - ?assertEqual(?GET(<<"interests_set">>, ?GET(Index, Results2a)), - Thing), - - {ok, {search_results, Results3a, _, Found3}} = riakc_pb_socket:search( - Pid, Index, <<"_yz_rb:testbucket">>), - ?assertEqual(1, Found3), - ?assertEqual(?GET(<<"name_register">>, ?GET(Index, Results3a)), - Key), - ?assertEqual(?GET(<<"interests_set">>, ?GET(Index, Results3a)), - Thing), - - %% Redo queries and check if results are equal - {ok, {search_results, Results1b, _, _}} = riakc_pb_socket:search( - Pid, Index, <<"name_register:Chris*">>), - ?assertEqual(number_of_fields(Results1a, Index), - number_of_fields(Results1b, Index)), - - {ok, {search_results, Results2b, _, _}} = riakc_pb_socket:search( - Pid, Index, <<"interests_set:thing*">>), - ?assertEqual(number_of_fields(Results2a, Index), - number_of_fields(Results2b, Index)), - - {ok, {search_results, Results3b, _, _}} = riakc_pb_socket:search( - Pid, Index, <<"_yz_rb:testbucket">>), - ?assertEqual(number_of_fields(Results3a, Index), - number_of_fields(Results3b, Index)), - - true - catch Err:Reason -> - lager:info("Waiting for CRDT search results to converge. Error" - " was ~p.", [{Err, Reason}]), - false - end. - -validate_test_repeat_set(Pid, Index) -> - try - {ok, {search_results, _R, _, Found}} = riakc_pb_socket:search( - Pid, Index, - <<"update_register:*">>), - ?assertEqual(1, Found), - - true - catch Err:Reason -> - lager:info("Waiting for CRDT search results to converge. Error" - " was ~p.", [{Err, Reason}]), - false - end. - -validate_test_siblings(Pid, Bucket, Index) -> - try - Key1 = <<"Movies">>, - Key2 = <<"Games">>, - - {ok, MF1} = riakc_pb_socket:fetch_type(Pid, Bucket, Key1), - Keys = riakc_map:fetch_keys(MF1), - ?assertEqual(1, length(Keys)), - ?assert(riakc_map:is_key({<<"directors">>, set}, MF1)), - {ok, {search_results, Results1, _, _}} = riakc_pb_socket:search( - Pid, Index, - <<"directors_set:*">>), - lager:info("Search movies map directors_set:*: ~p~n", [Results1]), - ?assertEqual(3, length(proplists:lookup_all(<<"directors_set">>, - ?GET(?INDEX, Results1)))), - - {ok, MF2} = riakc_pb_socket:fetch_type(Pid, Bucket, Key2), - Keys2 = riakc_map:fetch_keys(MF2), - ?assertEqual(1, length(Keys2)), - ?assert(riakc_map:is_key({<<"characters">>, set}, MF2)), - {ok, {search_results, Results2, _, _}} = riakc_pb_socket:search( - Pid, Index, - <<"characters_set:*">>), - lager:info("Search games map characters_set:*: ~p~n", [Results2]), - ?assertEqual(2, length(proplists:lookup_all(<<"characters_set">>, - ?GET(?INDEX, Results2)))), - - {ok, {search_results, Results3, _, Found}} = riakc_pb_socket:search( - Pid, Index, - <<"_yz_vtag:*">>), - lager:info("Search vtags in search *:*: ~p~n", [Results3]), - ?assertEqual(0, Found), - true - catch Err:Reason -> - lager:info("Waiting for CRDT search results to converge. Error" - " was ~p.", [{Err, Reason}]), - false - end. - -%% @private -drain_and_commit(Cluster, Index) -> - yokozuna_rt:drain_solrqs(Cluster), - yokozuna_rt:commit(Cluster, Index). - -%% @private -number_of_fields(Resp, Index) -> - length(?GET(Index, Resp)). - -%% @private -make_intercepts_tab(Node) -> - SupPid = rpc:call(Node, erlang, whereis, [sasl_safe_sup]), - intercepts_tab = rpc:call(Node, ets, new, [intercepts_tab, [named_table, - public, set, {heir, SupPid, {}}]]). - -%% @private -search_and_validate_found(Pid, Index, Search, ExpectedCount) -> - ok = rt:wait_until( - fun() -> - try - {ok, {search_results, Results2, _, F}} = - riakc_pb_socket:search(Pid, Index, Search), - ?assertEqual(ExpectedCount, F), - true - catch Err:Reason -> - lager:info( - "Waiting for CRDT search results to converge." - " Index: ~p" - " Search: ~p" - " Error: ~p", - [Index, Search, {Err, Reason}] - ), - false - end - end). - diff --git a/tests/yz_default_bucket_type_upgrade.erl b/tests/yz_default_bucket_type_upgrade.erl deleted file mode 100644 index 0152d692e..000000000 --- a/tests/yz_default_bucket_type_upgrade.erl +++ /dev/null @@ -1,98 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2015 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%%-------------------------------------------------------------------- - -%% @doc Test that checks to make sure that default bucket_types -%% do not lose data when expiring/clearing AAE trees when -%% trees are rebuilt for comparison. -%% @end - - --module(yz_default_bucket_type_upgrade). --compile(export_all). --include_lib("eunit/include/eunit.hrl"). --include_lib("riakc/include/riakc.hrl"). - --define(N, 3). --define(YZ_CAP, {yokozuna, handle_legacy_default_bucket_type_aae}). --define(INDEX, <<"test_upgrade_idx">>). --define(BUCKET, <<"test_upgrade_bucket">>). --define(SEQMAX, 2000). --define(CFG, - [{riak_core, - [ - {ring_creation_size, 16}, - {default_bucket_props, - [ - {n_val, ?N}, - {allow_mult, true}, - {dvv_enabled, true} - ]} - ]}, - {riak_kv, - [ - {anti_entropy_build_limit, {100, 1000}}, - {anti_entropy_concurrency, 8} - ] - }, - {yokozuna, - [ - {anti_entropy_tick, 1000}, - {enabled, true} - ]} - ]). - -confirm() -> - %% This test explicitly requires an upgrade from 2.0.5 to test a - %% new capability - OldVsn = "2.0.5", - - [_, Node|_] = Cluster = rt:build_cluster(lists:duplicate(4, {OldVsn, ?CFG})), - rt:wait_for_cluster_service(Cluster, yokozuna), - - [rt:assert_capability(ANode, ?YZ_CAP, {unknown_capability, ?YZ_CAP}) || ANode <- Cluster], - - GenKeys = yokozuna_rt:gen_keys(?SEQMAX), - KeyCount = length(GenKeys), - lager:info("KeyCount ~p", [KeyCount]), - - OldPid = rt:pbc(Node), - - yokozuna_rt:write_data(Cluster, OldPid, ?INDEX, ?BUCKET, GenKeys), - yokozuna_rt:commit(Cluster, ?INDEX), - - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount), - - %% Upgrade - yokozuna_rt:rolling_upgrade(Cluster, current), - - [rt:assert_capability(ANode, ?YZ_CAP, v1) || ANode <- Cluster], - [rt:assert_supported(rt:capability(ANode, all), ?YZ_CAP, [v1, v0]) || ANode <- Cluster], - - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount), - - lager:info("Write one more piece of data"), - Pid = rt:pbc(Node), - ok = rt:pbc_write(Pid, ?BUCKET, <<"foo">>, <<"foo">>, "text/plain"), - yokozuna_rt:commit(Cluster, ?INDEX), - - yokozuna_rt:expire_trees(Cluster), - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 1), - - pass. diff --git a/tests/yz_ensemble.erl b/tests/yz_ensemble.erl deleted file mode 100644 index f4a604592..000000000 --- a/tests/yz_ensemble.erl +++ /dev/null @@ -1,117 +0,0 @@ --module(yz_ensemble). --compile(export_all). --include_lib("eunit/include/eunit.hrl"). - --define(CFG, - [ - {riak_core, - [ - {ring_creation_size, 8} - ]}, - {yokozuna, - [ - {enabled, true} - ]} - ]). - -confirm() -> - NumNodes = 3, - NVal = 3, - ConfigB = ensemble_util:fast_config(NVal), - Config = ConfigB ++ [{yokozuna, [{enabled, true}]}], - lager:info("Building cluster and waiting for ensemble to stablize"), - Nodes = build_cluster_with_yz_support(NumNodes, Config, NVal), - rt:wait_for_cluster_service(Nodes, yokozuna), - vnode_util:load(Nodes), - Node = hd(Nodes), - - lager:info("Creating/activating 'strong' bucket type"), - rt:create_and_activate_bucket_type(Node, <<"strong">>, - [{consistent, true}, {n_val, NVal}]), - - Bucket = {<<"strong">>, <<"test">>}, - Index = <<"testi">>, - create_index(Node, Index), - set_bucket_props(Node, Bucket, Index), - - verify_ensemble_delete_support(Nodes, Bucket, Index), - - pass. - - -%% @private -%% @doc Populates then deletes from SC bucket -verify_ensemble_delete_support(Cluster, Bucket, Index) -> - %% Yz only supports UTF-8 compatible keys - Keys = [<> || N <- lists:seq(1,2000), - not lists:any(fun(E) -> E > 127 end,binary_to_list(<>))], - - PBC = rt:pbc(hd(Cluster)), - - lager:info("Writing ~p keys", [length(Keys)]), - [ok = rt:pbc_write(PBC, Bucket, Key, Key, "text/plain") || Key <- Keys], - yokozuna_rt:commit(Cluster, Index), - - %% soft commit wait, then check that last key is indexed - lager:info("Search for keys to verify they exist"), - LKey = lists:last(Keys), - rt:wait_until(fun() -> - {M, _} = riakc_pb_socket:search(PBC, Index, query_value(LKey)), - ok == M - end), - [{ok, _} = - riakc_pb_socket:search(PBC, Index, query_value(Key)) || Key <- Keys], - - lager:info("Deleting keys"), - [riakc_pb_socket:delete(PBC, Bucket, Key) || Key <- Keys], - yokozuna_rt:commit(Cluster, Index), - rt:wait_until(fun() -> - case riakc_pb_socket:search(PBC, Index, query_value(LKey)) of - {ok,{search_results,Res,_,_}} -> - lager:info("RES: ~p ~p~n", [Res, LKey]), - Res == []; - S -> - lager:info("OTHER: ~p ~p~n", [S, LKey]), - false - end - end), - [ {ok,{search_results,[],_,_}} = - riakc_pb_socket:search(PBC, Index, query_value(Key)) || Key <- Keys], - - ok. - - -%% @private -%% @doc build a cluster from ensemble_util + yz support -%% -%% NOTE: There's a timing issue that causes join_cluster to hang the r_t -%% node when adding yokozuna and ensemble support. Waiting for yokozuna -%% to load on each node allows join_cluster to complete consistently -build_cluster_with_yz_support(Num, Config, NVal) -> - Nodes = rt:deploy_nodes(Num, Config), - [rt:wait_for_cluster_service([N], yokozuna) || N <- Nodes], - Node = hd(Nodes), - rt:join_cluster(Nodes), - ensemble_util:wait_until_cluster(Nodes), - ensemble_util:wait_for_membership(Node), - ensemble_util:wait_until_stable(Node, NVal), - Nodes. - -%% @private -%% @doc Builds a simple riak key query -query_value(Value) -> - V2 = iolist_to_binary(re:replace(Value, "\"", "%22")), - V3 = iolist_to_binary(re:replace(V2, "\\\\", "%5C")), - <<"_yz_rk:\"",V3/binary,"\"">>. - -%% pulled from yz_rt - -%% @private -create_index(Node, Index) -> - lager:info("Creating index ~s [~p]", [Index, Node]), - ok = rpc:call(Node, yz_index, create, [Index]). - -%% @private -set_bucket_props(Node, Bucket, Index) -> - Props = [{search_index, Index}], - rpc:call(Node, riak_core_bucket, set_bucket, [Bucket, Props]). diff --git a/tests/yz_extractors.erl b/tests/yz_extractors.erl deleted file mode 100644 index 29d51c4bb..000000000 --- a/tests/yz_extractors.erl +++ /dev/null @@ -1,441 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2015 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%%------------------------------------------------------------------- - -%% @doc Test that checks if we're caching the extractor map and that -%% creating custom extractors is doable via protobufs. -%% @end - --module(yz_extractors). --compile(export_all). --include_lib("eunit/include/eunit.hrl"). --include_lib("riakc/include/riakc.hrl"). - --define(FMT(S, Args), lists:flatten(io_lib:format(S, Args))). --define(TYPE1, <<"extractors_in_paradise">>). --define(TYPE2, <<"extractors_in_paradiso">>). --define(INDEX1, <<"test_idx1">>). --define(BUCKET1, {?TYPE1, <<"test_bkt1">>}). --define(INDEX2, <<"test_idx2">>). --define(BUCKET2, {?TYPE2, <<"test_bkt2">>}). --define(TYPE3, <<"type3">>). --define(BUCKET3, {?TYPE3, <<"test_bkt3">>}). --define(INDEX3, <<"test_idx3">>). --define(SCHEMANAME, <<"test">>). --define(TEST_SCHEMA, -<<" - - - - - - - - - - - - - - - - - - -_yz_id - - - - - - - - - - - - - - - - - - - - - -">>). --define(TEST_SCHEMA_UPGRADE, -<<" - - - - - - - - - - - - - - - - - - - -_yz_id - - - - - - - - - - - - - - - - - - - - - -">>). --define(YZ_CAP, {yokozuna, extractor_map_in_cmd}). --define(GET_MAP_RING_MFA, {yz_extractor, get_map, 1}). --define(GET_MAP_MFA, {yz_extractor, get_map, 0}). --define(GET_MAP_READTHROUGH_MFA, {yz_extractor, get_map_read_through, 0}). --define(YZ_META_EXTRACTORS, {yokozuna, extractors}). --define(YZ_EXTRACTOR_MAP, yokozuna_extractor_map). --define(NEW_EXTRACTOR, {"application/httpheader", yz_noop_extractor}). --define(EXTRACTOR_CT, element(1, ?NEW_EXTRACTOR)). --define(EXTRACTOR_MOD, element(2, ?NEW_EXTRACTOR)). --define(DEFAULT_MAP, [{default, yz_noop_extractor}, - {"application/json",yz_json_extractor}, - {"application/riak_counter", yz_dt_extractor}, - {"application/riak_map", yz_dt_extractor}, - {"application/riak_set", yz_dt_extractor}, - {"application/xml",yz_xml_extractor}, - {"text/plain",yz_text_extractor}, - {"text/xml",yz_xml_extractor} - ]). --define(EXTRACTMAPEXPECT, lists:sort(?DEFAULT_MAP ++ [?NEW_EXTRACTOR])). --define(SEQMAX, 20). --define(NVAL, 3). --define(CFG, - [ - {riak_kv, - [ - %% allow AAE to build trees and exchange rapidly - {anti_entropy_build_limit, {100, 1000}}, - {anti_entropy_concurrency, 8}, - {anti_entropy_tick, 1000}, - %% but start with AAE turned off so as not to interfere with earlier parts of the test - {anti_entropy, {off, []}} - ]}, - {yokozuna, - [ - {enabled, true} - ]} - ]). - -confirm() -> - %% This test explicitly requires an upgrade from 2.0.5 to test a - %% new capability - OldVsn = "2.0.5", - - [_, Node|_] = Cluster = rt:build_cluster(lists:duplicate(4, {OldVsn, ?CFG})), - rt:wait_for_cluster_service(Cluster, yokozuna), - - [rt:assert_capability(ANode, ?YZ_CAP, {unknown_capability, ?YZ_CAP}) || ANode <- Cluster], - - OldPid = rt:pbc(Node), - - GenKeys = yokozuna_rt:gen_keys(?SEQMAX), - KeyCount = length(GenKeys), - - rt:count_calls(Cluster, [?GET_MAP_RING_MFA, ?GET_MAP_MFA]), - - yokozuna_rt:write_data(Cluster, OldPid, ?INDEX1, - {?SCHEMANAME, ?TEST_SCHEMA}, ?BUCKET1, GenKeys), - - ok = rt:stop_tracing(), - - {ok, BProps} = riakc_pb_socket:get_bucket(OldPid, ?BUCKET1), - N = proplists:get_value(n_val, BProps), - - riakc_pb_socket:stop(OldPid), - - PrevGetMapRingCC = rt:get_call_count(Cluster, ?GET_MAP_RING_MFA), - PrevGetMapCC = rt:get_call_count(Cluster, ?GET_MAP_MFA), - ?assertEqual(KeyCount * N, PrevGetMapRingCC), - ?assertEqual(KeyCount * N, PrevGetMapCC), - - %% test query count - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX1, KeyCount), - - {RingVal1, MDVal1} = get_ring_and_cmd_vals(Node, ?YZ_META_EXTRACTORS, - ?YZ_EXTRACTOR_MAP), - - ?assertEqual(undefined, MDVal1), - %% In previous version, Ring only gets map metadata if a non-default - %% extractor is registered - ?assertEqual(undefined, RingVal1), - - ?assertEqual(?DEFAULT_MAP, get_map(Node)), - - %% %% Custom Register - ExtractMap = register_extractor(Node, ?EXTRACTOR_CT, ?EXTRACTOR_MOD), - - ?assertEqual(?EXTRACTMAPEXPECT, ExtractMap), - - %% Upgrade - yokozuna_rt:rolling_upgrade(Cluster, current), - - [rt:wait_until_ready(ANode) || ANode <- Cluster], - - [rt:assert_capability(ANode, ?YZ_CAP, true) || ANode <- Cluster], - [rt:assert_supported(rt:capability(ANode, all), ?YZ_CAP, [true, false]) || - ANode <- Cluster], - - %% test query count again - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX1, KeyCount), - - Pid = rt:pbc(Node), - - rt:count_calls(Cluster, [?GET_MAP_RING_MFA, ?GET_MAP_MFA, - ?GET_MAP_READTHROUGH_MFA]), - - yokozuna_rt:write_data(Cluster, Pid, ?INDEX2, {?SCHEMANAME, ?TEST_SCHEMA}, - ?BUCKET2, GenKeys), - yokozuna_rt:commit(Cluster, ?INDEX2), - - ok = rt:stop_tracing(), - - riakc_pb_socket:stop(Pid), - - CurrGetMapRingCC = rt:get_call_count(Cluster, ?GET_MAP_RING_MFA), - CurrGetMapCC = rt:get_call_count(Cluster, ?GET_MAP_MFA), - CurrGetMapRTCC = rt:get_call_count(Cluster, ?GET_MAP_READTHROUGH_MFA), - - lager:info("Number of calls to get the map from the ring - current: ~p~n, previous: ~p~n", - [CurrGetMapRingCC, PrevGetMapRingCC]), - ?assert(CurrGetMapRingCC < PrevGetMapRingCC), - lager:info("Number of calls to get the map - current: ~p~n, previous: ~p~n", - [CurrGetMapCC, PrevGetMapCC]), - ?assert(CurrGetMapCC =< PrevGetMapCC), - lager:info("Number of calls to get_map_read_through/0: ~p~n, Number of calls to get_map/0: ~p~n", - [CurrGetMapRTCC, CurrGetMapCC]), - ?assert(CurrGetMapRTCC =< CurrGetMapCC), - - {_RingVal2, MDVal2} = get_ring_and_cmd_vals(Node, ?YZ_META_EXTRACTORS, - ?YZ_EXTRACTOR_MAP), - - ?assertEqual(?EXTRACTMAPEXPECT, MDVal2), - ?assertEqual(?EXTRACTMAPEXPECT, get_map(Node)), - - Packet = <<"GET http://www.google.com HTTP/1.1\n">>, - test_extractor_works(Cluster, Packet), - test_extractor_with_aae_expire(Cluster, ?INDEX2, ?BUCKET2, Packet), - test_bad_extraction(Cluster), - - pass. - -%%%=================================================================== -%%% Private -%%%=================================================================== - -get_ring_and_cmd_vals(Node, Prefix, Key) -> - Ring = rpc:call(Node, yz_misc, get_ring, [transformed]), - MDVal = metadata_get(Node, Prefix, Key), - RingVal = ring_meta_get(Node, Key, Ring), - {RingVal, MDVal}. - -metadata_get(Node, Prefix, Key) -> - rpc:call(Node, riak_core_metadata, get, [Prefix, Key, []]). - -ring_meta_get(Node, Key, Ring) -> - rpc:call(Node, riak_core_ring, get_meta, [Key, Ring]). - -register_extractor(Node, MimeType, Mod) -> - rpc:call(Node, yz_extractor, register, [MimeType, Mod]). - -get_map(Node) -> - rpc:call(Node, yz_extractor, get_map, []). - -verify_extractor(Node, PacketData, Mod) -> - rpc:call(Node, yz_extractor, run, [PacketData, Mod]). - -bucket_url({Host,Port}, {BType, BName}, Key) -> - ?FMT("http://~s:~B/types/~s/buckets/~s/keys/~s", - [Host, Port, BType, BName, Key]). - -test_extractor_works(Cluster, Packet) -> - [rt_intercept:add(ANode, {yz_noop_extractor, - [{{extract, 1}, extract_httpheader}]}) || - ANode <- Cluster], - [rt_intercept:wait_until_loaded(ANode) || ANode <- Cluster], - - ExpectedExtraction = [{method, 'GET'}, - {host, <<"www.google.com">>}, - {uri, <<"/">>}], - ?assertEqual(ExpectedExtraction, - verify_extractor(rt:select_random(Cluster), Packet, ?EXTRACTOR_MOD)). - -test_extractor_with_aae_expire(Cluster, Index, Bucket, Packet) -> - %% Now make sure we register extractor across all nodes - [register_extractor(ANode, ?EXTRACTOR_CT, ?EXTRACTOR_MOD) || - ANode <- Cluster], - - Key = <<"google">>, - - {Host, Port} = rt:select_random(yokozuna_rt:host_entries( - rt:connection_info( - Cluster))), - URL = bucket_url({Host, Port}, Bucket, - mochiweb_util:quote_plus(Key)), - - CT = ?EXTRACTOR_CT, - {ok, "204", _, _} = yokozuna_rt:http( - put, URL, [{"Content-Type", CT}], Packet), - - yokozuna_rt:commit(Cluster, Index), - - ANode = rt:select_random(Cluster), - yokozuna_rt:search_expect(ANode, Index, <<"host">>, - <<"www*">>, 1), - - rpc:multicall(Cluster, riak_kv_entropy_manager, enable, []), - - yokozuna_rt:expire_trees(Cluster), - yokozuna_rt:wait_for_full_exchange_round(Cluster, erlang:now()), - - yokozuna_rt:search_expect(ANode, Index, <<"host">>, - <<"www*">>, 1), - - APid = rt:pbc(rt:select_random(Cluster)), - yokozuna_rt:override_schema(APid, Cluster, Index, ?SCHEMANAME, - ?TEST_SCHEMA_UPGRADE), - - {ok, "200", RHeaders, _} = yokozuna_rt:http(get, URL, [{"Content-Type", CT}], - [], []), - VC = proplists:get_value("X-Riak-Vclock", RHeaders), - - {ok, "204", _, _} = yokozuna_rt:http( - put, URL, [{"Content-Type", CT}, {"X-Riak-Vclock", VC}], - Packet), - yokozuna_rt:commit(Cluster, Index), - - yokozuna_rt:search_expect(ANode, Index, <<"method">>, - <<"GET">>, 1), - - yokozuna_rt:expire_trees(Cluster), - yokozuna_rt:wait_for_full_exchange_round(Cluster, erlang:now()), - - yokozuna_rt:search_expect(ANode, Index, <<"method">>, - <<"GET">>, 1), - riakc_pb_socket:stop(APid). - -test_bad_extraction(Cluster) -> - %% Previous test enabled AAE, which makes the number of repairs here not consistent - %% Turn off AAE again just to make the test deterministic. - rpc:multicall(Cluster, riak_kv_entropy_manager, disable, []), - %% - %% register the no-op extractor on all the nodes with a content type - %% - [register_extractor(ANode, "application/bad-extractor", yz_noop_extractor) || - ANode <- Cluster], - %% - %% Set up the intercepts so that they extract non-unicode data - %% - [rt_intercept:add(ANode, {yz_noop_extractor, - [{{extract, 1}, extract_non_unicode_data}]}) || - ANode <- Cluster], - [rt_intercept:wait_until_loaded(ANode) || ANode <- Cluster], - %% - %% create and wire up the bucket to the Solr index/core - %% - yokozuna_rt:create_indexed_bucket_type(Cluster, ?TYPE3, ?INDEX3, ?SCHEMANAME), - %% - %% Grab the stats before - %% - {PreviousFailCount, PreviousErrorThresholdCount} = get_error_stats(Cluster), - %% - %% Put some data into Riak. This should cause the intercepted no-op - %% extractor to generate an object to be written into Solr that contains - %% non-unicode data. - {Host, Port} = rt:select_random( - yokozuna_rt:host_entries(rt:connection_info(Cluster))), - Key = <<"test_bad_extraction">>, - URL = bucket_url({Host, Port}, ?BUCKET3, Key), - Headers = [{"Content-Type", "application/bad-extractor"}], - Data = <<"blahblahblahblah">>, - {ok, "204", _, _} = yokozuna_rt:http(put, URL, Headers, Data), - %% - %% The put should pass, but because it's "bad data", there should - %% be no data in Riak. - %% - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX3, 0), - %% - %% Verify the stats. There should be one more index failure, - %% but there should be more more "melts" (error threshold failures) - %% - yokozuna_rt:wait_until( - Cluster, - fun(_Node) -> - check_error_stats(Cluster, PreviousFailCount, PreviousErrorThresholdCount) - end - ), - ok. - -check_error_stats(Cluster, PreviousFailCount, PreviousErrorThresholdCount) -> - {FailCount, ErrorThresholdCount} = get_error_stats(Cluster), - lager:info( - "PreviousFailCount: ~p FailCount: ~p;" - " PreviousErrorThresholdCount: ~p; ErrorThresholdCount: ~p", - [PreviousFailCount, FailCount, - PreviousErrorThresholdCount, ErrorThresholdCount] - ), - PreviousFailCount + ?NVAL == FailCount - andalso PreviousErrorThresholdCount == ErrorThresholdCount. - - -get_error_stats(Cluster) -> - AllStats = [rpc:call(Node, yz_stat, get_stats, []) || Node <- Cluster], - { - lists:sum([get_count([index, bad_entry], count, Stats) || Stats <- AllStats]), - lists:sum([get_count([search_index_error_threshold_failure_count], value, Stats) || Stats <- AllStats]) - }. - -get_count(StatName, Type, Stats) -> - proplists:get_value( - Type, - proplists:get_value( - yz_stat:stat_name(StatName), - Stats - ) - ). diff --git a/tests/yz_handoff.erl b/tests/yz_handoff.erl deleted file mode 100644 index 7ae32ad24..000000000 --- a/tests/yz_handoff.erl +++ /dev/null @@ -1,206 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2014 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%%------------------------------------------------------------------- --module(yz_handoff). --compile(export_all). --include_lib("eunit/include/eunit.hrl"). --include_lib("riakc/include/riakc.hrl"). - --define(GET(K,L), proplists:get_value(K, L)). --define(FMT(S, Args), lists:flatten(io_lib:format(S, Args))). --define(INDEX, <<"test_idx">>). --define(BUCKET, <<"test_bkt">>). --define(NUMRUNSTATES, 1). --define(SEQMAX, 1000). --define(TESTCYCLE, 20). --define(N, 3). --define(CFG, - [ - {riak_core, - [ - {ring_creation_size, 16}, - {n_val, ?N}, - {handoff_concurrency, 10}, - {vnode_management_timer, 1000} - ]}, - {riak_kv, - [ - %% allow AAE to build trees and exchange rapidly - {anti_entropy_build_limit, {100, 1000}}, - {anti_entropy_concurrency, 8}, - {handoff_rejected_max, infinity} - ]}, - {yokozuna, - [ - {anti_entropy_tick, 1000}, - {enabled, true} - ]} - ]). - --record(trial_state, { - solr_url_before, - solr_url_after, - leave_node, - join_node, - admin_node}). - -confirm() -> - %% Setup cluster initially - [Node1, Node2, _Node3, _Node4, _Node5] = Nodes = rt:build_cluster(5, ?CFG), - - rt:wait_for_cluster_service(Nodes, yokozuna), - - ConnInfo = ?GET(Node2, rt:connection_info([Node2])), - {Host, Port} = ?GET(http, ConnInfo), - Shards = [{N, node_solr_port(N)} || N <- Nodes], - - %% Generate keys, YZ only supports UTF-8 compatible keys - Keys = [<> || N <- lists:seq(1, ?SEQMAX), - not lists:any(fun(E) -> E > 127 end, - binary_to_list(<>))], - KeyCount = length(Keys), - - Pid = rt:pbc(Node2), - yokozuna_rt:write_data(Nodes, Pid, ?INDEX, ?BUCKET, Keys), - - %% Separate out shards for multiple runs - [Shard1|Shards2Rest] = Shards, - {_, SolrPort1} = Shard1, - [{_, SolrPort2}|_] = Shards2Rest, - SolrURL = internal_solr_url(Host, SolrPort1, ?INDEX, Shards), - BucketURL = bucket_keys_url(Host, Port, ?BUCKET), - SearchURL = search_url(Host, Port, ?INDEX), - - lager:info("Verify Replicas Count = (3 * docs/keys) count"), - verify_count(SolrURL, (KeyCount * ?N)), - - States = [#trial_state{solr_url_before = SolrURL, - solr_url_after = internal_solr_url(Host, SolrPort2, ?INDEX, Shards2Rest), - leave_node = Node1}, - #trial_state{solr_url_before = internal_solr_url(Host, SolrPort2, ?INDEX, Shards2Rest), - solr_url_after = SolrURL, - join_node = Node1, - admin_node = Node2}], - - %% Run set of leave/join trials and count/test #'s from the cluster - [[begin - check_data(Nodes, KeyCount, BucketURL, SearchURL, State), - check_counts(Pid, KeyCount, BucketURL) - end || State <- States] - || _ <- lists:seq(1,?NUMRUNSTATES)], - - pass. - -%%%=================================================================== -%%% Private -%%%=================================================================== - -node_solr_port(Node) -> - {ok, P} = riak_core_util:safe_rpc(Node, application, get_env, - [yokozuna, solr_port]), - P. - -internal_solr_url(Host, Port, Index) -> - ?FMT("http://~s:~B/internal_solr/~s", [Host, Port, Index]). -internal_solr_url(Host, Port, Index, Shards) -> - Ss = [internal_solr_url(Host, ShardPort, Index) - || {_, ShardPort} <- Shards], - ?FMT("http://~s:~B/internal_solr/~s/select?wt=json&q=*:*&shards=~s", - [Host, Port, Index, string:join(Ss, ",")]). - -%% @private -bucket_keys_url(Host, Port, BName) -> - ?FMT("http://~s:~B/buckets/~s/keys?keys=true", [Host, Port, BName]). - -%% @private -search_url(Host, Port, Index) -> - ?FMT("http://~s:~B/solr/~s/select?wt=json&q=*:*", [Host, Port, Index]). - -verify_count(Url, ExpectedCount) -> - AreUp = - fun() -> - {ok, "200", _, DBody} = yokozuna_rt:http(get, Url, [], [], "", 60000), - FoundCount = get_count(DBody), - lager:info("FoundCount: ~b, ExpectedCount: ~b", - [FoundCount, ExpectedCount]), - ExpectedCount =:= FoundCount - end, - ?assertEqual(ok, rt:wait_until(AreUp)), - ok. - -get_count(Resp) -> - Struct = mochijson2:decode(Resp), - kvc:path([<<"response">>, <<"numFound">>], Struct). - -get_keys_count(BucketURL) -> - {ok, "200", _, RBody} = yokozuna_rt:http(get, BucketURL, [], []), - Struct = mochijson2:decode(RBody), - length(kvc:path([<<"keys">>], Struct)). - -check_counts(Pid, InitKeyCount, BucketURL) -> - PBCounts = [begin {ok, Resp} = riakc_pb_socket:search( - Pid, ?INDEX, <<"*:*">>), - Resp#search_results.num_found - end || _ <- lists:seq(1,?TESTCYCLE)], - HTTPCounts = [begin {ok, "200", _, RBody} = yokozuna_rt:http( - get, BucketURL, [], []), - Struct = mochijson2:decode(RBody), - length(kvc:path([<<"keys">>], Struct)) - end || _ <- lists:seq(1,?TESTCYCLE)], - MinPBCount = lists:min(PBCounts), - MinHTTPCount = lists:min(HTTPCounts), - lager:info("Before-Node-Leave PB: ~b, After-Node-Leave PB: ~b", - [InitKeyCount, MinPBCount]), - ?assertEqual(InitKeyCount, MinPBCount), - lager:info("Before-Node-Leave PB: ~b, After-Node-Leave HTTP: ~b", - [InitKeyCount, MinHTTPCount]), - ?assertEqual(InitKeyCount, MinHTTPCount). - -check_data(Cluster, KeyCount, BucketURL, SearchURL, S) -> - CheckCount = KeyCount * ?N, - KeysBefore = get_keys_count(BucketURL), - - UpdatedCluster = leave_or_join(Cluster, S), - - yokozuna_rt:wait_for_aae(UpdatedCluster), - - KeysAfter = get_keys_count(BucketURL), - lager:info("KeysBefore: ~b, KeysAfter: ~b", [KeysBefore, KeysAfter]), - ?assertEqual(KeysBefore, KeysAfter), - - lager:info("Verify Search Docs Count =:= key count"), - lager:info("Run Search URL: ~s", [SearchURL]), - verify_count(SearchURL, KeysAfter), - lager:info("Verify Replicas Count = (3 * docs/keys) count"), - lager:info("Run Search URL: ~s", [S#trial_state.solr_url_after]), - verify_count(S#trial_state.solr_url_after, CheckCount). - -leave_or_join(Cluster, S=#trial_state{join_node=undefined}) -> - Node = S#trial_state.leave_node, - rt:leave(Node), - ?assertEqual(ok, rt:wait_until_unpingable(Node)), - Cluster -- [Node]; -leave_or_join(Cluster, S=#trial_state{leave_node=undefined}) -> - Node = S#trial_state.join_node, - NodeAdmin = S#trial_state.admin_node, - ok = rt:start_and_wait(Node), - ok = rt:join(Node, NodeAdmin), - ?assertEqual(ok, rt:wait_until_nodes_ready(Cluster)), - ?assertEqual(ok, rt:wait_until_no_pending_changes(Cluster)), - Cluster ++ [Node]. diff --git a/tests/yz_schema_change_reset.erl b/tests/yz_schema_change_reset.erl deleted file mode 100644 index 3efda592f..000000000 --- a/tests/yz_schema_change_reset.erl +++ /dev/null @@ -1,305 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2015 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%%-------------------------------------------------------------------- --module(yz_schema_change_reset). --compile(export_all). --include_lib("eunit/include/eunit.hrl"). --include_lib("riakc/include/riakc.hrl"). - --define(GET(K,L), proplists:get_value(K, L)). --define(INDEX, <<"test_schema_change_reset">>). --define(TYPE, <<"test_schema_change">>). --define(BUCKET1, <<"test_schema_change_reset">>). --define(BUCKET2, {?TYPE, <<"test_schema_change_reset_2">>}). --define(SCHEMANAME, <<"test">>). - --define(TEST_SCHEMA, -<<" - - - - - - - - - - - - - - - - -_yz_id - - - - - - - - - - - - - - - - - - - - -">>). --define(TEST_SCHEMA_UPDATE, -<<" - - - - - - - - - - - - - - - - - - - -_yz_id - - - - - - - - - - - - - - - - - - - -">>). - --define(SEQMAX, 20). --define(CFG, - [{riak_core, - [ - {ring_creation_size, 16} - ]}, - {riak_kv, [ - {anti_entropy_concurrency, 8}, - {anti_entropy_build_limit, {100, 1000}} - ]}, - {yokozuna, - [ - {anti_entropy_tick, 1000}, - {enabled, true} - ]} - ]). - -confirm() -> - [Node1|_RestNodes] = Cluster = rt:build_cluster(4, ?CFG), - rt:wait_for_cluster_service(Cluster, yokozuna), - - GenKeys = yokozuna_rt:gen_keys(?SEQMAX), - KeyCount = length(GenKeys), - lager:info("KeyCount ~p", [KeyCount]), - - Pid = rt:pbc(rt:select_random(Cluster)), - - lager:info("Write initial data to index ~p with schema ~p", - [?INDEX, ?SCHEMANAME]), - - yokozuna_rt:write_data(Cluster, Pid, ?INDEX, - {?SCHEMANAME, ?TEST_SCHEMA}, - ?BUCKET1, GenKeys), - lager:info("Create and activate map-based bucket type ~s and tie it to search_index ~s", - [?TYPE, ?INDEX]), - rt:create_and_activate_bucket_type(Node1, ?TYPE, [{datatype, map}, - {search_index, ?INDEX}]), - - lager:info("Write and check age at integer per original schema"), - - NewObj1A = riakc_obj:new(?BUCKET1, <<"keyA">>, - <<"{\"age\":26}">>, - "application/json"), - - NewObj1B = riakc_obj:new(?BUCKET1, <<"keyB">>, - <<"{\"age\":99}">>, - "application/json"), - - {ok, _ObjA} = riakc_pb_socket:put(Pid, NewObj1A, [return_head]), - yokozuna_rt:commit(Cluster, ?INDEX), - {ok, _ObjB} = riakc_pb_socket:put(Pid, NewObj1B, [return_head]), - yokozuna_rt:commit(Cluster, ?INDEX), - - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 2), - - yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, - <<"age:26">>, {<<"age">>, <<"26">>}, []), - yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, - <<"age:99">>, {<<"age">>, <<"99">>}, []), - - Map1 = riakc_map:update( - {<<"0_foo">>, register}, - fun(R) -> - riakc_register:set(<<"44ab">>, R) - end, riakc_map:new()), - ok = riakc_pb_socket:update_type( - Pid, - ?BUCKET2, - <<"keyMap1">>, - riakc_map:to_op(Map1)), - - {ok, Map2} = riakc_pb_socket:fetch_type(Pid, ?BUCKET2, <<"keyMap1">>), - Map3 = riakc_map:update( - {<<"1_baz">>, counter}, - fun(R) -> - riakc_counter:increment(10, R) - end, Map2), - ok = riakc_pb_socket:update_type( - Pid, - ?BUCKET2, - <<"keyMap1">>, - riakc_map:to_op(Map3)), - - yokozuna_rt:commit(Cluster, ?INDEX), - yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, - <<"0_foo_register:44ab">>, - {<<"0_foo_register">>, <<"44ab">>}, - []), - - lager:info("Expire and re-check count before updating schema"), - - yokozuna_rt:expire_trees(Cluster), - yokozuna_rt:wait_for_aae(Cluster), - - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 3), - - yokozuna_rt:override_schema(Pid, Cluster, ?INDEX, ?SCHEMANAME, ?TEST_SCHEMA_UPDATE), - - lager:info("Write and check hello_i at integer per schema update"), - - NewObj2 = riakc_obj:new(?BUCKET1, <<"key2">>, - <<"{\"hello_i\":36}">>, - "application/json"), - - {ok, _Obj2} = riakc_pb_socket:put(Pid, NewObj2, [return_head]), - yokozuna_rt:commit(Cluster, ?INDEX), - - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 4), - yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, - <<"hello_i:36">>, {<<"hello_i">>, <<"36">>}, []), - - lager:info("Write and check age at string per schema update"), - - NewObj3 = riakc_obj:new(?BUCKET1, <<"key3">>, - <<"{\"age\":\"3jlkjkl\"}">>, - "application/json"), - - {ok, _Obj3} = riakc_pb_socket:put(Pid, NewObj3, [return_head]), - yokozuna_rt:commit(Cluster, ?INDEX), - - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 5), - yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, - <<"age:3jlkjkl">>, {<<"age">>, <<"3jlkjkl">>}, - []), - - lager:info("Expire and re-check count to make sure we're correctly indexed - by the new schema"), - - yokozuna_rt:expire_trees(Cluster), - yokozuna_rt:wait_for_aae(Cluster), - - yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 5), - - ANode = rt:select_random(Cluster), - yokozuna_rt:search_expect(ANode, ?INDEX, <<"age">>, <<"*">>, 3), - - lager:info("Re-Put because AAE won't find a diff even though the types - have changed, as it only compares based on bkey currently. - Also, this re-put will work as we have a default bucket (type) - with allow_mult=false... no siblings"), - - {ok, _Obj4} = riakc_pb_socket:put(Pid, NewObj1A, [return_head]), - yokozuna_rt:commit(Cluster, ?INDEX), - - yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, - <<"age:26">>, {<<"age">>, <<"26">>}, []), - - lager:info("Re-Put Map data by dec/inc counter to account for *change* and - allow previously unindexed counter to be searchable"), - - {ok, Map4} = riakc_pb_socket:fetch_type(Pid, ?BUCKET2, <<"keyMap1">>), - Map5 = riakc_map:update( - {<<"1_baz">>, counter}, - fun(R) -> - riakc_counter:decrement(0, R), - riakc_counter:increment(0, R) - end, Map4), - ok = riakc_pb_socket:update_type( - Pid, - ?BUCKET2, - <<"keyMap1">>, - riakc_map:to_op(Map5)), - - yokozuna_rt:commit(Cluster, ?INDEX), - yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, - <<"0_foo_register:44ab">>, - {<<"0_foo_register">>, <<"44ab">>}, - []), - yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, - <<"1_baz_counter:10">>, - {<<"1_baz_counter">>, <<"10">>}, - []), - - lager:info("Test nested json searches w/ unsearched fields ignored"), - - NewObj5 = riakc_obj:new(?BUCKET1, <<"key4">>, - <<"{\"quip\":\"blashj3\", - \"paths\":{\"quip\":\"88\"}}">>, - "application/json"), - {ok, _Obj5} = riakc_pb_socket:put(Pid, NewObj5, [return_head]), - - yokozuna_rt:commit(Cluster, ?INDEX), - yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, - <<"paths.quip:88">>, - {<<"paths.quip">>, <<"88">>}, - []), - - riakc_pb_socket:stop(Pid), - - pass. -