diff --git a/CHANGES.md b/CHANGES.md index 5fdafd8..4c56656 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Added +### Changed + +## [v0.6.0] - 2019-06-19 + +### Added + +- Add federated search #30 - Add cluster manager (#48) - Add KVS HTTP handlers #46 diff --git a/Makefile b/Makefile index 484515f..0b5d1d8 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,7 @@ PACKAGES = $(shell $(GO) list ./... | grep -v '/vendor/') PROTOBUFS = $(shell find . -name '*.proto' -print0 | xargs -0 -n1 dirname | sort | uniq | grep -v /vendor/) TARGET_PACKAGES = $(shell find . -name 'main.go' -print0 | xargs -0 -n1 dirname | sort | uniq | grep -v /vendor/) +# TARGET_PACKAGES = $(shell find . -name 'main.go' -print0 | xargs -0 -n1 dirname | sort | uniq | grep -v /vendor/ | grep blast-manager) ifeq ($(VERSION),) VERSION = latest diff --git a/README.md b/README.md index f23cf81..539f8e7 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,7 @@ You can see the binary file when build successful like so: ```bash $ ls ./bin -blast-indexer +blast blastd ``` @@ -233,12 +233,21 @@ $ make \ -## Starting Blast index node +## Starting Blast in standalone mode -Running a Blast index node is easy. Start Blast data node like so: +Running a Blast in standalone mode is easy. Start a indexer like so: ```bash -$ ./bin/blast-indexer start --node-id=indexer1 --data-dir=/tmp/blast/indexer1 --bind-addr=:6060 --grpc-addr=:5050 --http-addr=:8080 --index-mapping-file=./example/index_mapping.json +$ ./bin/blastd \ + indexer \ + --node-id=indexer1 \ + --bind-addr=:5000 \ + --grpc-addr=:5001 \ + --http-addr=:5002 \ + --data-dir=/tmp/blast/indexer1 \ + --index-mapping-file=./example/index_mapping.json \ + --index-type=upside_down \ + --index-storage-type=boltdb ``` Please refer to following document for details of index mapping: @@ -255,15 +264,13 @@ You can now put, get, search and delete the documents via CLI. For document indexing, execute the following command: ```bash -$ cat ./example/doc_enwiki_1.json | xargs -0 ./bin/blast-indexer index --grpc-addr=:5050 --id=enwiki_1 +$ cat ./example/doc_enwiki_1.json | xargs -0 ./bin/blast set document --grpc-addr=:5001 enwiki_1 ``` You can see the result in JSON format. The result of the above command is: ```bash -{ - "count": 1 -} +1 ``` @@ -272,7 +279,7 @@ You can see the result in JSON format. The result of the above command is: Getting a document is as following: ```bash -$ ./bin/blast-indexer get --grpc-addr=:5050 --id=enwiki_1 +$ ./bin/blast get document --grpc-addr=:5001 enwiki_1 ``` You can see the result in JSON format. The result of the above command is: @@ -280,7 +287,6 @@ You can see the result in JSON format. The result of the above command is: ```json { "_type": "enwiki", - "contributor": "unknown", "text_en": "A search engine is an information retrieval system designed to help find information stored on a computer system. The search results are usually presented in a list and are commonly called hits. Search engines help to minimize the time required to find information and the amount of information which must be consulted, akin to other techniques for managing information overload. The most public, visible form of a search engine is a Web search engine which searches for information on the World Wide Web.", "timestamp": "2018-07-04T05:41:00Z", "title_en": "Search engine (computing)" @@ -293,7 +299,7 @@ You can see the result in JSON format. The result of the above command is: Searching documents is as like following: ```bash -$ cat ./example/search_request.json | xargs -0 ./bin/blast-indexer search --grpc-addr=:5050 +$ cat ./example/search_request.json | xargs -0 ./bin/blast search --grpc-addr=:5001 ``` You can see the result in JSON format. The result of the above command is: @@ -351,9 +357,9 @@ You can see the result in JSON format. The result of the above command is: }, "hits": [ { - "index": "/tmp/blast/index1/index", + "index": "/tmp/blast/indexer1/index", "id": "enwiki_1", - "score": 0.09634961191421738, + "score": 0.09703538256409851, "locations": { "text_en": { "search": [ @@ -411,7 +417,6 @@ You can see the result in JSON format. The result of the above command is: ], "fields": { "_type": "enwiki", - "contributor": "unknown", "text_en": "A search engine is an information retrieval system designed to help find information stored on a computer system. The search results are usually presented in a list and are commonly called hits. Search engines help to minimize the time required to find information and the amount of information which must be consulted, akin to other techniques for managing information overload. The most public, visible form of a search engine is a Web search engine which searches for information on the World Wide Web.", "timestamp": "2018-07-04T05:41:00Z", "title_en": "Search engine (computing)" @@ -419,20 +424,14 @@ You can see the result in JSON format. The result of the above command is: } ], "total_hits": 1, - "max_score": 0.09634961191421738, - "took": 362726, + "max_score": 0.09703538256409851, + "took": 201951, "facets": { "Contributor count": { "field": "contributor", - "total": 1, - "missing": 0, - "other": 0, - "terms": [ - { - "term": "unknown", - "count": 1 - } - ] + "total": 0, + "missing": 1, + "other": 0 }, "Timestamp range": { "field": "timestamp", @@ -465,15 +464,13 @@ Please refer to following document for details of search request and result: Deleting a document is as following: ```bash -$ ./bin/blast-indexer delete --grpc-addr=:5050 --id=enwiki_1 +$ ./bin/blast delete document --grpc-addr=:5001 enwiki_1 ``` You can see the result in JSON format. The result of the above command is: ```bash -{ - "count": 1 -} +1 ``` @@ -482,15 +479,13 @@ You can see the result in JSON format. The result of the above command is: Indexing documents in bulk, run the following command: ```bash -$ cat ./example/docs_wiki.json | xargs -0 ./bin/blast-indexer index --grpc-addr=:5050 +$ cat ./example/bulk_index_wiki.json | xargs -0 ./bin/blast set document --grpc-addr=:5001 ``` You can see the result in JSON format. The result of the above command is: ```bash -{ - "count": 4 -} +4 ``` @@ -499,21 +494,19 @@ You can see the result in JSON format. The result of the above command is: Deleting documents in bulk, run the following command: ```bash -$ cat ./example/docs_wiki.json | xargs -0 ./bin/blast-indexer delete --grpc-addr=:5050 +$ cat ./example/bulk_delete_wiki.json | xargs -0 ./bin/blast delete document --grpc-addr=:5001 ``` You can see the result in JSON format. The result of the above command is: ```bash -{ - "count": 4 -} +4 ``` ## Using HTTP REST API -Also you can do above commands via HTTP REST API that listened port 8080. +Also you can do above commands via HTTP REST API that listened port 5002. ### Indexing a document via HTTP REST API @@ -521,7 +514,7 @@ Also you can do above commands via HTTP REST API that listened port 8080. Indexing a document via HTTP is as following: ```bash -$ curl -s -X PUT 'http://127.0.0.1:8080/documents/enwiki_1' -d @./example/doc_enwiki_1.json +$ curl -X PUT 'http://127.0.0.1:5002/documents/enwiki_1' -d @./example/doc_enwiki_1.json ``` @@ -530,7 +523,7 @@ $ curl -s -X PUT 'http://127.0.0.1:8080/documents/enwiki_1' -d @./example/doc_en Getting a document via HTTP is as following: ```bash -$ curl -s -X GET 'http://127.0.0.1:8080/documents/enwiki_1' +$ curl -X GET 'http://127.0.0.1:5002/documents/enwiki_1' ``` @@ -539,7 +532,7 @@ $ curl -s -X GET 'http://127.0.0.1:8080/documents/enwiki_1' Searching documents via HTTP is as following: ```bash -$ curl -X POST 'http://127.0.0.1:8080/search' -d @./example/search_request.json +$ curl -X POST 'http://127.0.0.1:5002/search' -d @./example/search_request.json ``` @@ -548,7 +541,7 @@ $ curl -X POST 'http://127.0.0.1:8080/search' -d @./example/search_request.json Deleting a document via HTTP is as following: ```bash -$ curl -X DELETE 'http://127.0.0.1:8080/documents/enwiki_1' +$ curl -X DELETE 'http://127.0.0.1:5002/documents/enwiki_1' ``` @@ -557,7 +550,7 @@ $ curl -X DELETE 'http://127.0.0.1:8080/documents/enwiki_1' Indexing documents in bulk via HTTP is as following: ```bash -$ curl -s -X PUT 'http://127.0.0.1:8080/documents' -d @./example/docs_wiki.json +$ curl -X PUT 'http://127.0.0.1:5002/documents' -d @./example/bulk_index_wiki.json ``` @@ -566,56 +559,92 @@ $ curl -s -X PUT 'http://127.0.0.1:8080/documents' -d @./example/docs_wiki.json Deleting documents in bulk via HTTP is as following: ```bash -$ curl -X DELETE 'http://127.0.0.1:8080/documents' -d @./example/docs_wiki.json +$ curl -X DELETE 'http://127.0.0.1:5002/documents' -d @./example/bulk_delete_wiki.json ``` -## Bringing up a cluster +## Starting Blast in cluster mode -Blast is easy to bring up the cluster. Blast data node is already running, but that is not fault tolerant. If you need to increase the fault tolerance, bring up 2 more data nodes like so: +Blast can easily bring up a cluster. Running a Blast in standalone is not fault tolerant. If you need to improve fault tolerance, start two more indexers as follows: + +First of all, start a indexer in standalone. + +```bash +$ ./bin/blastd \ + indexer \ + --node-id=indexer1 \ + --bind-addr=:5000 \ + --grpc-addr=:5001 \ + --http-addr=:5002 \ + --data-dir=/tmp/blast/indexer1 \ + --index-mapping-file=./example/index_mapping.json \ + --index-type=upside_down \ + --index-storage-type=boltdb +``` + +Then, start two more indexers. ```bash -$ ./bin/blast-indexer start --node-id=indexer2 --data-dir=/tmp/blast/indexer2 --bind-addr=:6061 --grpc-addr=:5051 --http-addr=:8081 --index-mapping-file=./example/index_mapping.json --join-addr=:5050 -$ ./bin/blast-indexer start --node-id=indexer3 --data-dir=/tmp/blast/indexer3 --bind-addr=:6062 --grpc-addr=:5052 --http-addr=:8082 --index-mapping-file=./example/index_mapping.json --join-addr=:5050 +$ ./bin/blastd \ + indexer \ + --peer-addr=:5001 \ + --node-id=indexer2 \ + --bind-addr=:5010 \ + --grpc-addr=:5011 \ + --http-addr=:5012 \ + --data-dir=/tmp/blast/indexer2 + +$ ./bin/blastd \ + indexer \ + --peer-addr=:5001 \ + --node-id=indexer3 \ + --bind-addr=:5020 \ + --grpc-addr=:5021 \ + --http-addr=:5022 \ + --data-dir=/tmp/blast/indexer3 ``` _Above example shows each Blast node running on the same host, so each node must listen on different ports. This would not be necessary if each node ran on a different host._ -This instructs each new node to join an existing node, each node recognizes the joining clusters when started. -So you have a 3-node cluster. That way you can tolerate the failure of 1 node. You can check the peers with the following command: +This instructs each new node to join an existing node, specifying `--peer-addr=:5001`. Each node recognizes the joining clusters when started. +So you have a 3-node cluster. That way you can tolerate the failure of 1 node. You can check the peers in the cluster with the following command: + ```bash -$ ./bin/blast-indexer cluster --grpc-addr=:5050 +$ ./bin/blast get cluster --grpc-addr=:5001 ``` You can see the result in JSON format. The result of the above command is: ```json { - "nodes": [ - { - "id": "index1", - "bind_addr": ":6060", - "grpc_addr": ":5050", - "http_addr": ":8080", - "leader": true, - "data_dir": "/tmp/blast/index1" - }, - { - "id": "index2", - "bind_addr": ":6061", + "indexer1": { + "metadata": { + "bind_addr": ":5050", + "data_dir": "/tmp/blast/indexer1", "grpc_addr": ":5051", - "http_addr": ":8081", - "data_dir": "/tmp/blast/index2" + "http_addr": ":5052" }, - { - "id": "index3", - "bind_addr": ":6062", - "grpc_addr": ":5052", - "http_addr": ":8082", - "data_dir": "/tmp/blast/index3" - } - ] + "state": "Leader" + }, + "indexer2": { + "metadata": { + "bind_addr": ":5060", + "data_dir": "/tmp/blast/indexer2", + "grpc_addr": ":5061", + "http_addr": ":5062" + }, + "state": "Follower" + }, + "indexer3": { + "metadata": { + "bind_addr": ":5070", + "data_dir": "/tmp/blast/indexer3", + "grpc_addr": ":5071", + "http_addr": ":5072" + }, + "state": "Follower" + } } ``` @@ -624,13 +653,13 @@ Recommend 3 or more odd number of nodes in the cluster. In failure scenarios, da The following command indexes documents to any node in the cluster: ```bash -$ cat ./example/doc_enwiki_1.json | xargs -0 ./bin/blast-indexer index --grpc-addr=:5050 enwiki_1 +$ cat ./example/doc_enwiki_1.json | xargs -0 ./bin/blast set document --grpc-addr=:5001 enwiki_1 ``` So, you can get the document from the node specified by the above command as follows: ```bash -$ ./bin/blast-indexer get --grpc-addr=:5050 enwiki_1 +$ ./bin/blast get document --grpc-addr=:5001 enwiki_1 ``` You can see the result in JSON format. The result of the above command is: @@ -648,8 +677,8 @@ You can see the result in JSON format. The result of the above command is: You can also get the same document from other nodes in the cluster as follows: ```bash -$ ./bin/blast-indexer get --grpc-addr=:5051 enwiki_1 -$ ./bin/blast-indexer get --grpc-addr=:5052 enwiki_1 +$ ./bin/blast get document --grpc-addr=:5011 enwiki_1 +$ ./bin/blast get document --grpc-addr=:5021 enwiki_1 ``` You can see the result in JSON format. The result of the above command is: @@ -665,6 +694,144 @@ You can see the result in JSON format. The result of the above command is: ``` +## Starting Blast in federated mode (experimental) + +Running a Blast in cluster mode allows you to replicate the index among indexers in a cluster to improve fault tolerance. +However, as the index grows, performance degradation can become an issue. Therefore, instead of providing a large single physical index, it is better to distribute indices across multiple indexers. +Blast provides a federated mode to enable distributed search and indexing. + +Blast provides the following type of node for federation: +- manager: Manager manage common index mappings to index across multiple indexers. It also manages information and status of clusters that participate in the federation. +- dispatcher: Dispatcher is responsible for distributed search or indexing of each indexer. In the case of a index request, send document to each cluster based on the document ID. And in the case of a search request, the same query is sent to each cluster, then the search results are merged and returned to the client. + +### Bring up the manager cluster. + +Manager can also bring up a cluster like an indexer. Specify a common index mapping for federation at startup. + +```bash +$ ./bin/blastd \ + manager \ + --node-id=manager1 \ + --bind-addr=:15000 \ + --grpc-addr=:15001 \ + --http-addr=:15002 \ + --data-dir=/tmp/blast/manager1 \ + --index-mapping-file=./example/index_mapping.json \ + --index-type=upside_down \ + --index-storage-type=boltdb + +$ ./bin/blastd \ + manager \ + --peer-addr=:15001 \ + --node-id=manager2 \ + --bind-addr=:15010 \ + --grpc-addr=:15011 \ + --http-addr=:15012 \ + --data-dir=/tmp/blast/manager2 + +$ ./bin/blastd \ + manager \ + --peer-addr=:15001 \ + --node-id=manager3 \ + --bind-addr=:15020 \ + --grpc-addr=:15021 \ + --http-addr=:15022 \ + --data-dir=/tmp/blast/manager3 +``` + +### Bring up the indexer cluster. + +Federated mode differs from cluster mode that it specifies the manager in start up to bring up indexer cluster. +The following example starts two 3-node clusters. + +```bash +$ ./bin/blastd \ + indexer \ + --manager-addr=:15001 \ + --cluster-id=cluster1 \ + --node-id=indexer1 \ + --bind-addr=:5000 \ + --grpc-addr=:5001 \ + --http-addr=:5002 \ + --data-dir=/tmp/blast/indexer1 + +$ ./bin/blastd \ + indexer \ + --manager-addr=:15001 \ + --cluster-id=cluster1 \ + --node-id=indexer2 \ + --bind-addr=:5010 \ + --grpc-addr=:5011 \ + --http-addr=:5012 \ + --data-dir=/tmp/blast/indexer2 + +$ ./bin/blastd \ + indexer \ + --manager-addr=:15001 \ + --cluster-id=cluster1 \ + --node-id=indexer3 \ + --bind-addr=:5020 \ + --grpc-addr=:5021 \ + --http-addr=:5022 \ + --data-dir=/tmp/blast/indexer3 + +$ ./bin/blastd \ + indexer \ + --manager-addr=:15001 \ + --cluster-id=cluster2 \ + --node-id=indexer4 \ + --bind-addr=:5030 \ + --grpc-addr=:5031 \ + --http-addr=:5032 \ + --data-dir=/tmp/blast/indexer4 + +$ ./bin/blastd \ + indexer \ + --manager-addr=:15001 \ + --cluster-id=cluster2 \ + --node-id=indexer5 \ + --bind-addr=:5040 \ + --grpc-addr=:5041 \ + --http-addr=:5042 \ + --data-dir=/tmp/blast/indexer5 + +$ ./bin/blastd \ + indexer \ + --manager-addr=:15001 \ + --cluster-id=cluster2 \ + --node-id=indexer6 \ + --bind-addr=:5050 \ + --grpc-addr=:5051 \ + --http-addr=:5052 \ + --data-dir=/tmp/blast/indexer6 +``` + +### Start up the dispatcher. + +Finally, start the dispatcher with a manager that manages the target federation so that it can perform distributed search and indexing. + +```bash +$ ./bin/blastd \ + dispatcher \ + --manager-addr=:15001 \ + --grpc-addr=:25001 \ + --http-addr=:25002 +``` + +```bash +$ cat ./example/bulk_index_wiki.json | xargs -0 ./bin/blast set document --grpc-addr=:25001 +``` + +```bash +$ cat ./example/search_request.json | xargs -0 ./bin/blast search --grpc-addr=:25001 +``` + +```bash +$ cat ./example/bulk_delete_wiki.json | xargs -0 ./bin/blast delete document --grpc-addr=:25001 +``` + + + ## Blast on Docker ### Building Docker container image on localhost @@ -695,20 +862,20 @@ $ docker pull mosuka/blast:latest ``` -### Running Blast index node on Docker +### Running Indexer on Docker Running a Blast data node on Docker. Start Blast data node like so: ```bash $ docker run --rm --name blast-indexer1 \ - -p 5050:5050 \ -p 6060:6060 \ + -p 7070:7070 \ -p 8080:8080 \ -v $(pwd)/example:/opt/blast/example \ mosuka/blast:latest blast-indexer start \ --node-id=blast-indexer1 \ --bind-addr=:6060 \ - --grpc-addr=:5050 \ + --grpc-addr=:7070 \ --http-addr=:8080 \ --data-dir=/tmp/blast/indexer1 \ --index-mapping-file=/opt/blast/example/index_mapping.json \ @@ -718,7 +885,7 @@ $ docker run --rm --name blast-indexer1 \ You can execute the command in docker container as follows: ```bash -$ docker exec -it blast-indexer1 blast-indexer node --grpc-addr=:5050 +$ docker exec -it blast-indexer1 blast-indexer node --grpc-addr=:7070 ``` @@ -752,7 +919,7 @@ $ ./WikiExtractor.py -o ~/tmp/enwiki --json ~/tmp/enwiki-20190101-pages-articles ### Indexing wikipedia dump -```bash +```shell $ for FILE in $(find ~/tmp/enwiki -type f -name '*' | sort) do echo "Indexing ${FILE}" diff --git a/cmd/blast-indexer/get.go b/cmd/blast-indexer/get.go deleted file mode 100644 index 9ace247..0000000 --- a/cmd/blast-indexer/get.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) 2019 Minoru Osuka -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "encoding/json" - "errors" - "fmt" - "os" - - "github.com/mosuka/blast/indexer" - "github.com/mosuka/blast/protobuf" - pbindex "github.com/mosuka/blast/protobuf/index" - "github.com/urfave/cli" -) - -func execGet(c *cli.Context) error { - grpcAddr := c.String("grpc-addr") - id := c.String("id") - if id == "" { - err := errors.New("arguments are not correct") - return err - } - - doc := &pbindex.Document{ - Id: id, - } - - client, err := indexer.NewGRPCClient(grpcAddr) - if err != nil { - return err - } - defer func() { - err := client.Close() - if err != nil { - fmt.Fprintln(os.Stderr, err) - } - }() - - resp, err := client.Get(doc) - if err != nil { - return err - } - - // Any -> map[string]interface{} - var fieldsMap *map[string]interface{} - fieldsInstance, err := protobuf.MarshalAny(resp.Fields) - if err != nil { - return err - } - if fieldsInstance == nil { - return errors.New("nil") - } - fieldsMap = fieldsInstance.(*map[string]interface{}) - - // map[string]interface -> []byte - fieldsBytes, err := json.MarshalIndent(fieldsMap, "", " ") - if err != nil { - return err - } - - fmt.Fprintln(os.Stdout, fmt.Sprintf("%v\n", string(fieldsBytes))) - - return nil -} diff --git a/cmd/blast-indexer/index.go b/cmd/blast-indexer/index.go deleted file mode 100644 index b835af4..0000000 --- a/cmd/blast-indexer/index.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright (c) 2019 Minoru Osuka -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "encoding/json" - "errors" - "fmt" - "os" - - "github.com/golang/protobuf/ptypes/any" - "github.com/mosuka/blast/indexer" - "github.com/mosuka/blast/protobuf" - pbindex "github.com/mosuka/blast/protobuf/index" - "github.com/urfave/cli" -) - -func execIndex(c *cli.Context) error { - grpcAddr := c.String("grpc-addr") - id := c.String("id") - - if c.NArg() == 0 { - err := errors.New("arguments are not correct") - return err - } - - // create documents - docs := make([]*pbindex.Document, 0) - - if id == "" { - // documents - docsStr := c.Args().Get(0) - - var docMaps []map[string]interface{} - err := json.Unmarshal([]byte(docsStr), &docMaps) - if err != nil { - return err - } - - for _, docMap := range docMaps { - // map[string]interface{} -> Any - fieldsAny := &any.Any{} - err = protobuf.UnmarshalAny(docMap["fields"], fieldsAny) - if err != nil { - return err - } - - // create document - doc := &pbindex.Document{ - Id: docMap["id"].(string), - Fields: fieldsAny, - } - - docs = append(docs, doc) - } - } else { - // document - fields := c.Args().Get(0) - - // string -> map[string]interface{} - var fieldsMap map[string]interface{} - err := json.Unmarshal([]byte(fields), &fieldsMap) - if err != nil { - return err - } - - // map[string]interface{} -> Any - fieldsAny := &any.Any{} - err = protobuf.UnmarshalAny(fieldsMap, fieldsAny) - if err != nil { - return err - } - - // create document - doc := &pbindex.Document{ - Id: id, - Fields: fieldsAny, - } - - docs = append(docs, doc) - } - - // create gRPC client - client, err := indexer.NewGRPCClient(grpcAddr) - if err != nil { - return err - } - defer func() { - err := client.Close() - if err != nil { - fmt.Fprintln(os.Stderr, err) - } - }() - - // index documents in bulk - result, err := client.Index(docs) - if err != nil { - return err - } - - resultBytes, err := json.MarshalIndent(result, "", " ") - if err != nil { - return err - } - - fmt.Fprintln(os.Stdout, fmt.Sprintf("%v\n", string(resultBytes))) - - return nil -} diff --git a/cmd/blast-indexer/join.go b/cmd/blast-indexer/join.go deleted file mode 100644 index ad03957..0000000 --- a/cmd/blast-indexer/join.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) 2019 Minoru Osuka -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "errors" - "fmt" - "os" - - "github.com/mosuka/blast/indexer" - "github.com/mosuka/blast/protobuf/raft" - "github.com/urfave/cli" -) - -func execJoin(c *cli.Context) error { - grpcAddr := c.String("grpc-addr") - - id := c.Args().Get(0) - if id == "" { - err := errors.New("id argument must be set") - return err - } - - addr := c.Args().Get(1) - if addr == "" { - err := errors.New("address argument must be set") - return err - } - - client, err := indexer.NewGRPCClient(grpcAddr) - if err != nil { - return err - } - defer func() { - err := client.Close() - if err != nil { - fmt.Fprintln(os.Stderr, err) - } - }() - - node := &raft.Node{ - Id: id, - BindAddr: addr, - } - - err = client.Join(node) - if err != nil { - return err - } - - return nil -} diff --git a/cmd/blast-indexer/leave.go b/cmd/blast-indexer/leave.go deleted file mode 100644 index 075563d..0000000 --- a/cmd/blast-indexer/leave.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) 2019 Minoru Osuka -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "errors" - "fmt" - "os" - - "github.com/mosuka/blast/indexer" - "github.com/mosuka/blast/protobuf/raft" - "github.com/urfave/cli" -) - -func execLeave(c *cli.Context) error { - grpcAddr := c.String("grpc-addr") - - id := c.Args().Get(0) - if id == "" { - err := errors.New("id argument must be set") - return err - } - - client, err := indexer.NewGRPCClient(grpcAddr) - if err != nil { - return err - } - defer func() { - err := client.Close() - if err != nil { - fmt.Fprintln(os.Stderr, err) - } - }() - - node := &raft.Node{ - Id: id, - } - - err = client.Leave(node) - if err != nil { - return err - } - - return nil -} diff --git a/cmd/blast-indexer/main.go b/cmd/blast-indexer/main.go deleted file mode 100644 index f55b158..0000000 --- a/cmd/blast-indexer/main.go +++ /dev/null @@ -1,295 +0,0 @@ -// Copyright (c) 2019 Minoru Osuka -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "fmt" - "os" - "path" - - "github.com/mosuka/blast/version" - "github.com/urfave/cli" -) - -func main() { - app := cli.NewApp() - app.Name = path.Base(os.Args[0]) - app.Usage = "Blast indexer" - app.Version = version.Version - app.Authors = []cli.Author{ - { - Name: "mosuka", - Email: "minoru.osuka@gmail.com", - }, - } - app.Commands = []cli.Command{ - { - Name: "start", - Usage: "Start index server", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "node-id, n", - Value: "", - Usage: "Node ID", - }, - cli.StringFlag{ - Name: "bind-addr, b", - Value: ":6060", - Usage: "Raft bind address", - }, - cli.StringFlag{ - Name: "grpc-addr, g", - Value: ":5050", - Usage: "gRPC Server listen address", - }, - cli.StringFlag{ - Name: "http-addr, H", - Value: ":8080", - Usage: "HTTP server listen address", - }, - cli.StringFlag{ - Name: "data-dir, d", - Value: "/tmp/blast-index", - Usage: "Data directory", - }, - cli.StringFlag{ - Name: "join-addr, j", - Value: "", - Usage: "Existing gRPC server listen address to join to the cluster", - }, - cli.StringFlag{ - Name: "index-mapping-file, m", - Value: "", - Usage: "Path to a file containing a JSON representation of an index mapping to use", - }, - cli.StringFlag{ - Name: "index-storage-type, s", - Value: "boltdb", - Usage: "Index storage type to use", - }, - cli.StringFlag{ - Name: "log-level, L", - Value: "INFO", - Usage: "Log level", - }, - cli.StringFlag{ - Name: "log-file, F", - Value: os.Stderr.Name(), - Usage: "Log file", - }, - cli.IntFlag{ - Name: "log-max-size, S", - Value: 500, - Usage: "Max size of a log file (megabytes)", - }, - cli.IntFlag{ - Name: "log-max-backups, B", - Value: 3, - Usage: "Max backup count of log files", - }, - cli.IntFlag{ - Name: "log-max-age, A", - Value: 30, - Usage: "Max age of a log file (days)", - }, - cli.BoolFlag{ - Name: "log-compress, C", - Usage: "Compress a log file", - }, - cli.StringFlag{ - Name: "http-access-log-file", - Value: os.Stderr.Name(), - Usage: "HTTP access log file", - }, - cli.IntFlag{ - Name: "http-access-log-max-size", - Value: 500, - Usage: "Max size of a HTTP access log file (megabytes)", - }, - cli.IntFlag{ - Name: "http-access-log-max-backups", - Value: 3, - Usage: "Max backup count of HTTP access log files", - }, - cli.IntFlag{ - Name: "http-access-log-max-age", - Value: 30, - Usage: "Max age of a HTTP access log file (days)", - }, - cli.BoolFlag{ - Name: "http-access-log-compress", - Usage: "Compress a HTTP access log", - }, - }, - Action: execStart, - }, - { - Name: "join", - Usage: "Join a node to the cluster", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "grpc-addr, g", - Value: ":5050", - Usage: "gRPC address to connect to", - }, - }, - ArgsUsage: "[id] [addr]", - Action: execJoin, - }, - { - Name: "leave", - Usage: "Leave a node from the cluster", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "grpc-addr, g", - Value: ":5050", - Usage: "gRPC address to connect to", - }, - }, - ArgsUsage: "[id]", - Action: execLeave, - }, - { - Name: "node", - Usage: "Get node", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "grpc-addr, g", - Value: ":5050", - Usage: "gRPC address to connect to", - }, - }, - ArgsUsage: "[id]", - Action: execNode, - }, - { - Name: "cluster", - Usage: "Get cluster", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "grpc-addr, g", - Value: ":5050", - Usage: "gRPC address to connect to", - }, - }, - Action: execCluster, - }, - { - Name: "snapshot", - Usage: "Create snapshot manually", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "grpc-addr, g", - Value: ":5050", - Usage: "gRPC address to connect to", - }, - }, - Action: execSnapshot, - }, - { - Name: "get", - Usage: "get document", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "grpc-addr, g", - Value: ":5050", - Usage: "gRPC address to connect to", - }, - cli.StringFlag{ - Name: "id, i", - Value: "", - Usage: "document id", - }, - }, - Action: execGet, - }, - { - Name: "index", - Usage: "Index documents in bulk", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "grpc-addr, g", - Value: ":5050", - Usage: "gRPC address to connect to", - }, - cli.StringFlag{ - Name: "id, i", - Value: "", - Usage: "document id", - }, - }, - ArgsUsage: "[documents | fields]", - Action: execIndex, - }, - { - Name: "delete", - Usage: "Delete documents in bulk", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "grpc-addr, g", - Value: ":5050", - Usage: "address to connect to", - }, - cli.StringFlag{ - Name: "id, i", - Value: "", - Usage: "document id", - }, - }, - ArgsUsage: "[documents]", - Action: execDelete, - }, - { - Name: "search", - Usage: "Search documents", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "grpc-addr, g", - Value: ":5050", - Usage: "gRPC address to connect to", - }, - }, - ArgsUsage: "[search request]", - Action: execSearch, - }, - { - Name: "stats", - Usage: "Get a index stats", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "grpc-addr, g", - Value: ":5050", - Usage: "address to connect to", - }, - }, - Action: execStats, - }, - } - - cli.HelpFlag = cli.BoolFlag{ - Name: "help, h", - Usage: "Show this message", - } - - cli.VersionFlag = cli.BoolFlag{ - Name: "version, v", - Usage: "Print the version", - } - - err := app.Run(os.Args) - if err != nil { - fmt.Fprintln(os.Stderr, err) - } -} diff --git a/cmd/blast-indexer/start.go b/cmd/blast-indexer/start.go deleted file mode 100644 index 7ce31de..0000000 --- a/cmd/blast-indexer/start.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright (c) 2019 Minoru Osuka -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "os" - "os/signal" - "syscall" - - "github.com/mosuka/blast/indexer" - "github.com/mosuka/logutils" - "github.com/urfave/cli" -) - -func execStart(c *cli.Context) error { - nodeId := c.String("node-id") - bindAddr := c.String("bind-addr") - grpcAddr := c.String("grpc-addr") - httpAddr := c.String("http-addr") - dataDir := c.String("data-dir") - joinAddr := c.String("join-addr") - - indexMappingFile := c.String("index-mapping-file") - indexStorageType := c.String("index-storage-type") - - logLevel := c.String("log-level") - logFilename := c.String("log-file") - logMaxSize := c.Int("log-max-size") - logMaxBackups := c.Int("log-max-backups") - logMaxAge := c.Int("log-max-age") - logCompress := c.Bool("log-compress") - - httpAccessLogFilename := c.String("http-access-log-file") - httpAccessLogMaxSize := c.Int("http-access-log-max-size") - httpAccessLogMaxBackups := c.Int("http-access-log-max-backups") - httpAccessLogMaxAge := c.Int("http-access-log-max-age") - httpAccessLogCompress := c.Bool("http-access-log-compress") - - // Create logger - logger := logutils.NewLogger( - logLevel, - logFilename, - logMaxSize, - logMaxBackups, - logMaxAge, - logCompress, - ) - - // Create HTTP access logger - httpAccessLogger := logutils.NewApacheCombinedLogger( - httpAccessLogFilename, - httpAccessLogMaxSize, - httpAccessLogMaxBackups, - httpAccessLogMaxAge, - httpAccessLogCompress, - ) - - svr, err := indexer.NewServer(nodeId, bindAddr, grpcAddr, httpAddr, dataDir, joinAddr, indexMappingFile, indexStorageType, logger, httpAccessLogger) - if err != nil { - return err - } - - quitCh := make(chan os.Signal, 1) - signal.Notify(quitCh, os.Kill, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) - - go svr.Start() - - <-quitCh - - svr.Stop() - - return nil -} diff --git a/cmd/blast-manager/get.go b/cmd/blast-manager/get.go deleted file mode 100644 index b3b4734..0000000 --- a/cmd/blast-manager/get.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright (c) 2019 Minoru Osuka -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "encoding/json" - "errors" - "fmt" - "os" - - "github.com/mosuka/blast/manager" - "github.com/mosuka/blast/protobuf" - pbfederation "github.com/mosuka/blast/protobuf/management" - "github.com/urfave/cli" -) - -func execGet(c *cli.Context) error { - grpcAddr := c.String("grpc-addr") - - key := c.String("key") - if key == "" { - err := errors.New("key argument must be set") - return err - } - - req := &pbfederation.KeyValuePair{ - Key: key, - } - - client, err := manager.NewGRPCClient(grpcAddr) - if err != nil { - return err - } - defer func() { - err := client.Close() - if err != nil { - fmt.Fprintln(os.Stderr, err) - } - }() - - resp, err := client.Get(req) - if err != nil { - return err - } - - // Any -> map[string]interface{} - var fieldsMap *map[string]interface{} - fieldsInstance, err := protobuf.MarshalAny(resp.Value) - if err != nil { - return err - } - if fieldsInstance == nil { - return errors.New("nil") - } - fieldsMap = fieldsInstance.(*map[string]interface{}) - - // map[string]interface -> []byte - valueBytes, err := json.MarshalIndent(fieldsMap, "", " ") - if err != nil { - return err - } - - fmt.Fprintln(os.Stdout, fmt.Sprintf("%v\n", string(valueBytes))) - - return nil -} diff --git a/cmd/blast-manager/leave.go b/cmd/blast-manager/leave.go deleted file mode 100644 index c4298b3..0000000 --- a/cmd/blast-manager/leave.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) 2019 Minoru Osuka -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "errors" - "fmt" - "os" - - "github.com/mosuka/blast/manager" - "github.com/mosuka/blast/protobuf/raft" - "github.com/urfave/cli" -) - -func execLeave(c *cli.Context) error { - grpcAddr := c.String("grpc-addr") - - id := c.Args().Get(0) - if id == "" { - err := errors.New("id argument must be set") - return err - } - - node := &raft.Node{ - Id: id, - } - - client, err := manager.NewGRPCClient(grpcAddr) - if err != nil { - return err - } - defer func() { - err := client.Close() - if err != nil { - fmt.Fprintln(os.Stderr, err) - } - }() - - err = client.Leave(node) - if err != nil { - return err - } - - return nil -} diff --git a/cmd/blast-manager/main.go b/cmd/blast-manager/main.go deleted file mode 100644 index f5b8b41..0000000 --- a/cmd/blast-manager/main.go +++ /dev/null @@ -1,258 +0,0 @@ -// Copyright (c) 2019 Minoru Osuka -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "fmt" - "os" - "path" - - "github.com/mosuka/blast/version" - "github.com/urfave/cli" -) - -func main() { - app := cli.NewApp() - app.Name = path.Base(os.Args[0]) - app.Usage = "Blast manager" - app.Version = version.Version - app.Authors = []cli.Author{ - { - Name: "mosuka", - Email: "minoru.osuka@gmail.com", - }, - } - app.Commands = []cli.Command{ - { - Name: "start", - Usage: "Start federation server", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "node-id, n", - Value: "", - Usage: "Node ID", - }, - cli.StringFlag{ - Name: "bind-addr, b", - Value: ":6060", - Usage: "Raft bind address", - }, - cli.StringFlag{ - Name: "grpc-addr, g", - Value: ":5050", - Usage: "gRPC Server listen address", - }, - cli.StringFlag{ - Name: "http-addr, H", - Value: ":8080", - Usage: "HTTP server listen address", - }, - cli.StringFlag{ - Name: "data-dir, d", - Value: "./", - Usage: "Data directory", - }, - cli.StringFlag{ - Name: "join-addr, j", - Value: "", - Usage: "Existing gRPC server listen address to join to the cluster", - }, - cli.StringFlag{ - Name: "log-level", - Value: "INFO", - Usage: "Log level", - }, - cli.StringFlag{ - Name: "log-file, L", - Value: os.Stderr.Name(), - Usage: "Log file", - }, - cli.IntFlag{ - Name: "log-max-size, S", - Value: 500, - Usage: "Max size of a log file (megabytes)", - }, - cli.IntFlag{ - Name: "log-max-backups, B", - Value: 3, - Usage: "Max backup count of log files", - }, - cli.IntFlag{ - Name: "log-max-age, A", - Value: 30, - Usage: "Max age of a log file (days)", - }, - cli.BoolFlag{ - Name: "log-compress, C", - Usage: "Compress a log file", - }, - cli.StringFlag{ - Name: "http-access-log-file", - Value: os.Stderr.Name(), - Usage: "HTTP access log file", - }, - cli.IntFlag{ - Name: "http-access-log-max-size", - Value: 500, - Usage: "Max size of a HTTP access log file (megabytes)", - }, - cli.IntFlag{ - Name: "http-access-log-max-backups", - Value: 3, - Usage: "Max backup count of HTTP access log files", - }, - cli.IntFlag{ - Name: "http-access-log-max-age", - Value: 30, - Usage: "Max age of a HTTP access log file (days)", - }, - cli.BoolFlag{ - Name: "http-access-log-compress", - Usage: "Compress a HTTP access log", - }, - }, - Action: execStart, - }, - { - Name: "join", - Usage: "Join a node to the cluster", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "grpc-addr, g", - Value: ":5050", - Usage: "gRPC address to connect to", - }, - }, - ArgsUsage: "[id] [addr]", - Action: execJoin, - }, - { - Name: "leave", - Usage: "Leave a node from the cluster", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "grpc-addr, g", - Value: ":5050", - Usage: "address to connect to", - }, - }, - ArgsUsage: "[id]", - Action: execLeave, - }, - { - Name: "node", - Usage: "Get node", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "grpc-addr, g", - Value: ":5050", - Usage: "gRPC address to connect to", - }, - }, - ArgsUsage: "[id]", - Action: execNode, - }, - { - Name: "cluster", - Usage: "Get cluster", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "grpc-addr, g", - Value: ":5050", - Usage: "gRPC address to connect to", - }, - }, - Action: execCluster, - }, - { - Name: "snapshot", - Usage: "Create snapshot manually", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "grpc-addr, g", - Value: ":5050", - Usage: "address to connect to", - }, - }, - Action: execSnapshot, - }, - { - Name: "get", - Usage: "Get a value by key", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "grpc-addr, g", - Value: ":5050", - Usage: "gRPC address to connect to", - }, - cli.StringFlag{ - Name: "key, k", - Value: "", - Usage: "Key", - }, - }, - Action: execGet, - }, - { - Name: "set", - Usage: "Set a value by key", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "grpc-addr, g", - Value: ":5050", - Usage: "gRPC address to connect to", - }, - cli.StringFlag{ - Name: "key, k", - Value: "", - Usage: "key", - }, - }, - ArgsUsage: "[value]", - Action: execSet, - }, - { - Name: "delete", - Usage: "Delete a value by key", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "grpc-addr, g", - Value: ":5050", - Usage: "gRPC address to connect to", - }, - cli.StringFlag{ - Name: "key, k", - Value: "", - Usage: "Key", - }, - }, - Action: execDelete, - }, - } - - cli.HelpFlag = cli.BoolFlag{ - Name: "help, h", - Usage: "Show this message", - } - cli.VersionFlag = cli.BoolFlag{ - Name: "version, v", - Usage: "Print the version", - } - - err := app.Run(os.Args) - if err != nil { - fmt.Fprintln(os.Stderr, err) - } -} diff --git a/cmd/blast-manager/set.go b/cmd/blast-manager/set.go deleted file mode 100644 index bba8bc7..0000000 --- a/cmd/blast-manager/set.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (c) 2019 Minoru Osuka -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "encoding/json" - "errors" - "fmt" - "os" - - "github.com/golang/protobuf/ptypes/any" - "github.com/mosuka/blast/manager" - "github.com/mosuka/blast/protobuf" - pbfederation "github.com/mosuka/blast/protobuf/management" - "github.com/urfave/cli" -) - -func execSet(c *cli.Context) error { - grpcAddr := c.String("grpc-addr") - - key := c.String("key") - if key == "" { - err := errors.New("key argument must be set") - return err - } - - value := c.Args().Get(0) - if value == "" { - err := errors.New("value argument must be set") - return err - } - - // string -> map[string]interface{} - var valueMap map[string]interface{} - err := json.Unmarshal([]byte(value), &valueMap) - if err != nil { - return err - } - - // map[string]interface{} -> Any - valueAny := &any.Any{} - err = protobuf.UnmarshalAny(valueMap, valueAny) - if err != nil { - return err - } - - // create PutRequest - req := &pbfederation.KeyValuePair{ - Key: key, - Value: valueAny, - } - - client, err := manager.NewGRPCClient(grpcAddr) - if err != nil { - return err - } - defer func() { - err := client.Close() - if err != nil { - fmt.Fprintln(os.Stderr, err) - } - }() - - err = client.Set(req) - if err != nil { - return err - } - - return nil -} diff --git a/cmd/blast/delete_document.go b/cmd/blast/delete_document.go new file mode 100644 index 0000000..558e959 --- /dev/null +++ b/cmd/blast/delete_document.go @@ -0,0 +1,70 @@ +// Copyright (c) 2019 Minoru Osuka +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/mosuka/blast/grpc" + "github.com/urfave/cli" +) + +func execDeleteDocument(c *cli.Context) error { + grpcAddr := c.String("grpc-addr") + + // create documents + ids := make([]string, 0) + + // documents + idsStr := c.Args().Get(0) + + err := json.Unmarshal([]byte(idsStr), &ids) + if err != nil { + switch err.(type) { + case *json.SyntaxError: + ids = append(ids, idsStr) + default: + return err + } + } + + // create client + client, err := grpc.NewClient(grpcAddr) + if err != nil { + return err + } + defer func() { + err := client.Close() + if err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + } + }() + + result, err := client.DeleteDocument(ids) + if err != nil { + return err + } + + resultBytes, err := json.MarshalIndent(result, "", " ") + if err != nil { + return err + } + + _, _ = fmt.Fprintln(os.Stdout, fmt.Sprintf("%v", string(resultBytes))) + + return nil +} diff --git a/cmd/blast-indexer/snapshot.go b/cmd/blast/delete_node.go similarity index 79% rename from cmd/blast-indexer/snapshot.go rename to cmd/blast/delete_node.go index ee86995..d2d7566 100644 --- a/cmd/blast-indexer/snapshot.go +++ b/cmd/blast/delete_node.go @@ -18,25 +18,27 @@ import ( "fmt" "os" - "github.com/mosuka/blast/indexer" + "github.com/mosuka/blast/grpc" "github.com/urfave/cli" ) -func execSnapshot(c *cli.Context) error { +func execDeleteNode(c *cli.Context) error { grpcAddr := c.String("grpc-addr") - client, err := indexer.NewGRPCClient(grpcAddr) + nodeId := c.Args().Get(0) + + client, err := grpc.NewClient(grpcAddr) if err != nil { return err } defer func() { err := client.Close() if err != nil { - fmt.Fprintln(os.Stderr, err) + _, _ = fmt.Fprintln(os.Stderr, err) } }() - err = client.Snapshot() + err = client.DeleteNode(nodeId) if err != nil { return err } diff --git a/cmd/blast-manager/delete.go b/cmd/blast/delete_state.go similarity index 74% rename from cmd/blast-manager/delete.go rename to cmd/blast/delete_state.go index e96301c..6194481 100644 --- a/cmd/blast-manager/delete.go +++ b/cmd/blast/delete_state.go @@ -19,36 +19,31 @@ import ( "fmt" "os" - "github.com/mosuka/blast/manager" - pbfederation "github.com/mosuka/blast/protobuf/management" + "github.com/mosuka/blast/grpc" "github.com/urfave/cli" ) -func execDelete(c *cli.Context) error { +func execDeleteState(c *cli.Context) error { grpcAddr := c.String("grpc-addr") - key := c.String("key") + key := c.Args().Get(0) if key == "" { err := errors.New("key argument must be set") return err } - req := &pbfederation.KeyValuePair{ - Key: key, - } - - client, err := manager.NewGRPCClient(grpcAddr) + client, err := grpc.NewClient(grpcAddr) if err != nil { return err } defer func() { err := client.Close() if err != nil { - fmt.Fprintln(os.Stderr, err) + _, _ = fmt.Fprintln(os.Stderr, err) } }() - err = client.Delete(req) + err = client.DeleteState(key) if err != nil { return err } diff --git a/cmd/blast-indexer/cluster.go b/cmd/blast/get_cluster.go similarity index 81% rename from cmd/blast-indexer/cluster.go rename to cmd/blast/get_cluster.go index 5a4667b..1c123af 100644 --- a/cmd/blast-indexer/cluster.go +++ b/cmd/blast/get_cluster.go @@ -19,21 +19,21 @@ import ( "fmt" "os" - "github.com/mosuka/blast/indexer" + "github.com/mosuka/blast/grpc" "github.com/urfave/cli" ) -func execCluster(c *cli.Context) error { +func execGetCluster(c *cli.Context) error { grpcAddr := c.String("grpc-addr") - client, err := indexer.NewGRPCClient(grpcAddr) + client, err := grpc.NewClient(grpcAddr) if err != nil { return err } defer func() { err := client.Close() if err != nil { - fmt.Fprintln(os.Stderr, err) + _, _ = fmt.Fprintln(os.Stderr, err) } }() @@ -47,7 +47,7 @@ func execCluster(c *cli.Context) error { return err } - fmt.Fprintln(os.Stdout, fmt.Sprintf("%v\n", string(clusterBytes))) + _, _ = fmt.Fprintln(os.Stdout, fmt.Sprintf("%v", string(clusterBytes))) return nil } diff --git a/cmd/blast-manager/join.go b/cmd/blast/get_document.go similarity index 68% rename from cmd/blast-manager/join.go rename to cmd/blast/get_document.go index 471b7de..98b31de 100644 --- a/cmd/blast-manager/join.go +++ b/cmd/blast/get_document.go @@ -15,50 +15,45 @@ package main import ( + "encoding/json" "errors" "fmt" "os" - "github.com/mosuka/blast/manager" - "github.com/mosuka/blast/protobuf/raft" + "github.com/mosuka/blast/grpc" "github.com/urfave/cli" ) -func execJoin(c *cli.Context) error { +func execGetDocument(c *cli.Context) error { grpcAddr := c.String("grpc-addr") - id := c.Args().Get(0) if id == "" { - err := errors.New("id argument must be set") - return err - } - - addr := c.Args().Get(1) - if addr == "" { - err := errors.New("address argument must be set") + err := errors.New("arguments are not correct") return err } - node := &raft.Node{ - Id: id, - BindAddr: addr, - } - - client, err := manager.NewGRPCClient(grpcAddr) + client, err := grpc.NewClient(grpcAddr) if err != nil { return err } defer func() { err := client.Close() if err != nil { - fmt.Fprintln(os.Stderr, err) + _, _ = fmt.Fprintln(os.Stderr, err) } }() - err = client.Join(node) + fields, err := client.GetDocument(id) if err != nil { return err } + fieldsBytes, err := json.MarshalIndent(fields, "", " ") + if err != nil { + return err + } + + _, _ = fmt.Fprintln(os.Stdout, fmt.Sprintf("%v", string(fieldsBytes))) + return nil } diff --git a/cmd/blast/get_metadata.go b/cmd/blast/get_metadata.go new file mode 100644 index 0000000..57e165e --- /dev/null +++ b/cmd/blast/get_metadata.go @@ -0,0 +1,55 @@ +// Copyright (c) 2019 Minoru Osuka +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/mosuka/blast/grpc" + "github.com/urfave/cli" +) + +func execGetMetadata(c *cli.Context) error { + grpcAddr := c.String("grpc-addr") + + nodeId := c.Args().Get(0) + + client, err := grpc.NewClient(grpcAddr) + if err != nil { + return err + } + defer func() { + err := client.Close() + if err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + } + }() + + metadata, err := client.GetNodeMetadata(nodeId) + if err != nil { + return err + } + + metadataBytes, err := json.MarshalIndent(metadata, "", " ") + if err != nil { + return err + } + + _, _ = fmt.Fprintln(os.Stdout, fmt.Sprintf("%v", string(metadataBytes))) + + return nil +} diff --git a/cmd/blast-indexer/node.go b/cmd/blast/get_node.go similarity index 71% rename from cmd/blast-indexer/node.go rename to cmd/blast/get_node.go index 1df4a65..c7c8271 100644 --- a/cmd/blast-indexer/node.go +++ b/cmd/blast/get_node.go @@ -19,35 +19,37 @@ import ( "fmt" "os" - "github.com/mosuka/blast/indexer" + "github.com/mosuka/blast/grpc" "github.com/urfave/cli" ) -func execNode(c *cli.Context) error { +func execGetNode(c *cli.Context) error { grpcAddr := c.String("grpc-addr") - client, err := indexer.NewGRPCClient(grpcAddr) + nodeId := c.Args().Get(0) + + client, err := grpc.NewClient(grpcAddr) if err != nil { return err } defer func() { err := client.Close() if err != nil { - fmt.Fprintln(os.Stderr, err) + _, _ = fmt.Fprintln(os.Stderr, err) } }() - nodes, err := client.GetNode() + metadata, err := client.GetNode(nodeId) if err != nil { return err } - nodesBytes, err := json.MarshalIndent(nodes, "", " ") + metadataBytes, err := json.MarshalIndent(metadata, "", " ") if err != nil { return err } - fmt.Fprintln(os.Stdout, fmt.Sprintf("%v\n", string(nodesBytes))) + _, _ = fmt.Fprintln(os.Stdout, fmt.Sprintf("%v", string(metadataBytes))) return nil } diff --git a/cmd/blast-manager/cluster.go b/cmd/blast/get_nodestate.go similarity index 70% rename from cmd/blast-manager/cluster.go rename to cmd/blast/get_nodestate.go index c10ea3b..99dd948 100644 --- a/cmd/blast-manager/cluster.go +++ b/cmd/blast/get_nodestate.go @@ -19,35 +19,37 @@ import ( "fmt" "os" - "github.com/mosuka/blast/manager" + "github.com/mosuka/blast/grpc" "github.com/urfave/cli" ) -func execCluster(c *cli.Context) error { +func execGetNodeState(c *cli.Context) error { grpcAddr := c.String("grpc-addr") - client, err := manager.NewGRPCClient(grpcAddr) + nodeId := c.Args().Get(0) + + client, err := grpc.NewClient(grpcAddr) if err != nil { return err } defer func() { err := client.Close() if err != nil { - fmt.Fprintln(os.Stderr, err) + _, _ = fmt.Fprintln(os.Stderr, err) } }() - cluster, err := client.GetCluster() + metadata, err := client.GetNodeState(nodeId) if err != nil { return err } - clusterBytes, err := json.MarshalIndent(cluster, "", " ") + metadataBytes, err := json.MarshalIndent(metadata, "", " ") if err != nil { return err } - fmt.Fprintln(os.Stdout, fmt.Sprintf("%v\n", string(clusterBytes))) + _, _ = fmt.Fprintln(os.Stdout, fmt.Sprintf("%v", string(metadataBytes))) return nil } diff --git a/cmd/blast-manager/node.go b/cmd/blast/get_state.go similarity index 72% rename from cmd/blast-manager/node.go rename to cmd/blast/get_state.go index c756776..e667916 100644 --- a/cmd/blast-manager/node.go +++ b/cmd/blast/get_state.go @@ -19,35 +19,36 @@ import ( "fmt" "os" - "github.com/mosuka/blast/manager" + "github.com/mosuka/blast/grpc" "github.com/urfave/cli" ) -func execNode(c *cli.Context) error { +func execGetState(c *cli.Context) error { grpcAddr := c.String("grpc-addr") - client, err := manager.NewGRPCClient(grpcAddr) + key := c.Args().Get(0) + + client, err := grpc.NewClient(grpcAddr) if err != nil { return err } defer func() { err := client.Close() if err != nil { - fmt.Fprintln(os.Stderr, err) + _, _ = fmt.Fprintln(os.Stderr, err) } }() - nodes, err := client.GetNode() + value, err := client.GetState(key) if err != nil { return err } - nodesBytes, err := json.MarshalIndent(nodes, "", " ") + valueBytes, err := json.MarshalIndent(value, "", " ") if err != nil { return err } - - fmt.Fprintln(os.Stdout, fmt.Sprintf("%v\n", string(nodesBytes))) + _, _ = fmt.Fprintln(os.Stdout, fmt.Sprintf("%v", string(valueBytes))) return nil } diff --git a/cmd/blast/livenessprobe.go b/cmd/blast/livenessprobe.go new file mode 100644 index 0000000..b801fd8 --- /dev/null +++ b/cmd/blast/livenessprobe.go @@ -0,0 +1,47 @@ +// Copyright (c) 2019 Minoru Osuka +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + + "github.com/mosuka/blast/grpc" + "github.com/urfave/cli" +) + +func execLivenessProbe(c *cli.Context) error { + grpcAddr := c.String("grpc-addr") + + client, err := grpc.NewClient(grpcAddr) + if err != nil { + return err + } + defer func() { + err := client.Close() + if err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + } + }() + + state, err := client.LivenessProbe() + if err != nil { + return err + } + + _, _ = fmt.Fprintln(os.Stdout, fmt.Sprintf("%v", state)) + + return nil +} diff --git a/cmd/blast/main.go b/cmd/blast/main.go new file mode 100644 index 0000000..269aa1b --- /dev/null +++ b/cmd/blast/main.go @@ -0,0 +1,306 @@ +// Copyright (c) 2019 Minoru Osuka +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + "path" + + "github.com/mosuka/blast/version" + "github.com/urfave/cli" +) + +func main() { + app := cli.NewApp() + app.Name = path.Base(os.Args[0]) + app.Usage = "blast" + app.Version = version.Version + app.Authors = []cli.Author{ + { + Name: "mosuka", + Email: "minoru.osuka@gmail.com", + }, + } + app.Commands = []cli.Command{ + { + Name: "livenessprobe", + Usage: "liveness probe", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "grpc-addr", + Value: ":5001", + Usage: "address to connect to", + }, + }, + Action: execLivenessProbe, + }, + { + Name: "readinessprobe", + Usage: "readiness probe", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "grpc-addr", + Value: ":5001", + Usage: "address to connect to", + }, + }, + Action: execReadinessProbe, + }, + { + Name: "get", + Usage: "get", + Subcommands: []cli.Command{ + { + Name: "metadata", + Usage: "get node metadata", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "grpc-addr", + Value: ":5001", + Usage: "address to connect to", + }, + }, + ArgsUsage: "[id]", + Action: execGetMetadata, + }, + { + Name: "nodestate", + Usage: "get node state", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "grpc-addr", + Value: ":5001", + Usage: "address to connect to", + }, + }, + ArgsUsage: "[id]", + Action: execGetNodeState, + }, + { + Name: "node", + Usage: "get node", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "grpc-addr", + Value: ":5001", + Usage: "address to connect to", + }, + }, + ArgsUsage: "[id]", + Action: execGetNode, + }, + { + Name: "cluster", + Usage: "get cluster", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "grpc-addr", + Value: ":5001", + Usage: "address to connect to", + }, + }, + Action: execGetCluster, + }, + { + Name: "state", + Usage: "get state", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "grpc-addr", + Value: ":5001", + Usage: "address to connect to", + }, + }, + ArgsUsage: "[key]", + Action: execGetState, + }, + { + Name: "document", + Usage: "get document", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "grpc-addr", + Value: ":5001", + Usage: "address to connect to", + }, + }, + ArgsUsage: "[id]", + Action: execGetDocument, + }, + }, + }, + { + Name: "set", + Usage: "set", + Subcommands: []cli.Command{ + { + Name: "node", + Usage: "set node", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "grpc-addr", + Value: ":5001", + Usage: "address to connect to", + }, + }, + ArgsUsage: "[id] [metadata]", + Action: execSetNode, + }, + { + Name: "state", + Usage: "set state", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "grpc-addr", + Value: ":5001", + Usage: "address to connect to", + }, + }, + ArgsUsage: "[key] [value]", + Action: execSetState, + }, + { + Name: "document", + Usage: "set document", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "grpc-addr", + Value: ":5001", + Usage: "address to connect to", + }, + }, + ArgsUsage: "[documents | [id] [fields]]", + Action: execSetDocument, + }, + }, + }, + { + Name: "delete", + Usage: "delete", + Subcommands: []cli.Command{ + { + Name: "node", + Usage: "delete node", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "grpc-addr", + Value: ":5001", + Usage: "address to connect to", + }, + }, + ArgsUsage: "[id]", + Action: execDeleteNode, + }, + { + Name: "state", + Usage: "delete state", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "grpc-addr", + Value: ":5001", + Usage: "address to connect to", + }, + }, + ArgsUsage: "[key] [value]", + Action: execDeleteState, + }, + { + Name: "document", + Usage: "delete document", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "grpc-addr", + Value: ":5001", + Usage: "address to connect to", + }, + }, + ArgsUsage: "[id] ...", + Action: execDeleteDocument, + }, + }, + }, + { + Name: "watch", + Usage: "watch", + Subcommands: []cli.Command{ + { + Name: "cluster", + Usage: "watch cluster", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "grpc-addr", + Value: ":5001", + Usage: "address to connect to", + }, + }, + Action: execWatchCluster, + }, + { + Name: "state", + Usage: "watch state", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "grpc-addr", + Value: ":5001", + Usage: "address to connect to", + }, + }, + ArgsUsage: "[key]", + Action: execWatchState, + }, + }, + }, + { + Name: "snapshot", + Usage: "snapshot data", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "grpc-addr", + Value: ":5001", + Usage: "address to connect to", + }, + }, + Action: execSnapshot, + }, + { + Name: "search", + Usage: "search documents", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "grpc-addr", + Value: ":5001", + Usage: "address to connect to", + }, + }, + ArgsUsage: "[search request]", + Action: execSearch, + }, + } + + cli.HelpFlag = cli.BoolFlag{ + Name: "help, h", + Usage: "Show this message", + } + cli.VersionFlag = cli.BoolFlag{ + Name: "version, v", + Usage: "Print the version", + } + + err := app.Run(os.Args) + if err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + } +} diff --git a/cmd/blast/readinessprobe.go b/cmd/blast/readinessprobe.go new file mode 100644 index 0000000..36bf970 --- /dev/null +++ b/cmd/blast/readinessprobe.go @@ -0,0 +1,47 @@ +// Copyright (c) 2019 Minoru Osuka +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + + "github.com/mosuka/blast/grpc" + "github.com/urfave/cli" +) + +func execReadinessProbe(c *cli.Context) error { + grpcAddr := c.String("grpc-addr") + + client, err := grpc.NewClient(grpcAddr) + if err != nil { + return err + } + defer func() { + err := client.Close() + if err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + } + }() + + state, err := client.ReadinessProbe() + if err != nil { + return err + } + + _, _ = fmt.Fprintln(os.Stdout, fmt.Sprintf("%v", state)) + + return nil +} diff --git a/cmd/blast-indexer/search.go b/cmd/blast/search.go similarity index 88% rename from cmd/blast-indexer/search.go rename to cmd/blast/search.go index 136db9e..174aa83 100644 --- a/cmd/blast-indexer/search.go +++ b/cmd/blast/search.go @@ -21,7 +21,7 @@ import ( "os" "github.com/blevesearch/bleve" - "github.com/mosuka/blast/indexer" + "github.com/mosuka/blast/grpc" "github.com/urfave/cli" ) @@ -43,14 +43,14 @@ func execSearch(c *cli.Context) error { } } - client, err := indexer.NewGRPCClient(grpcAddr) + client, err := grpc.NewClient(grpcAddr) if err != nil { return err } defer func() { err := client.Close() if err != nil { - fmt.Fprintln(os.Stderr, err) + _, _ = fmt.Fprintln(os.Stderr, err) } }() @@ -64,7 +64,7 @@ func execSearch(c *cli.Context) error { return err } - fmt.Fprintln(os.Stdout, fmt.Sprintf("%v\n", string(jsonBytes))) + _, _ = fmt.Fprintln(os.Stdout, fmt.Sprintf("%v", string(jsonBytes))) return nil } diff --git a/cmd/blast-indexer/delete.go b/cmd/blast/set_document.go similarity index 55% rename from cmd/blast-indexer/delete.go rename to cmd/blast/set_document.go index f586ce3..fded562 100644 --- a/cmd/blast-indexer/delete.go +++ b/cmd/blast/set_document.go @@ -20,72 +20,71 @@ import ( "fmt" "os" - "github.com/mosuka/blast/indexer" - pbindex "github.com/mosuka/blast/protobuf/index" + "github.com/mosuka/blast/grpc" "github.com/urfave/cli" ) -func execDelete(c *cli.Context) error { +func execSetDocument(c *cli.Context) error { grpcAddr := c.String("grpc-addr") - id := c.String("id") // create documents - docs := make([]*pbindex.Document, 0) - - if id == "" { - if c.NArg() == 0 { - err := errors.New("arguments are not correct") - return err - } + docs := make([]map[string]interface{}, 0) + if c.NArg() == 1 { // documents docsStr := c.Args().Get(0) - var docMaps []map[string]interface{} - err := json.Unmarshal([]byte(docsStr), &docMaps) + err := json.Unmarshal([]byte(docsStr), &docs) if err != nil { return err } - - for _, docMap := range docMaps { - // create document - doc := &pbindex.Document{ - Id: docMap["id"].(string), - } - - docs = append(docs, doc) + } else if c.NArg() == 2 { + // document + id := c.Args().Get(0) + fieldsSrc := c.Args().Get(1) + + // string -> map[string]interface{} + var fields map[string]interface{} + err := json.Unmarshal([]byte(fieldsSrc), &fields) + if err != nil { + return err } - } else { - doc := &pbindex.Document{ - Id: id, + + // create document + doc := map[string]interface{}{ + "id": id, + "fields": fields, } docs = append(docs, doc) + } else { + return errors.New("argument error") } - // create client - client, err := indexer.NewGRPCClient(grpcAddr) + // create gRPC client + client, err := grpc.NewClient(grpcAddr) if err != nil { return err } defer func() { err := client.Close() if err != nil { - fmt.Fprintln(os.Stderr, err) + _, _ = fmt.Fprintln(os.Stderr, err) } }() - result, err := client.Delete(docs) + // index documents in bulk + count, err := client.IndexDocument(docs) if err != nil { return err } - resultBytes, err := json.MarshalIndent(result, "", " ") + resultBytes, err := json.MarshalIndent(count, "", " ") if err != nil { return err } - fmt.Fprintln(os.Stdout, fmt.Sprintf("%v\n", string(resultBytes))) + _, _ = fmt.Fprintln(os.Stdout, fmt.Sprintf("%v", string(resultBytes))) return nil } diff --git a/cmd/blast/set_node.go b/cmd/blast/set_node.go new file mode 100644 index 0000000..a32052a --- /dev/null +++ b/cmd/blast/set_node.go @@ -0,0 +1,56 @@ +// Copyright (c) 2019 Minoru Osuka +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/mosuka/blast/grpc" + "github.com/urfave/cli" +) + +func execSetNode(c *cli.Context) error { + grpcAddr := c.String("grpc-addr") + + nodeId := c.Args().Get(0) + + metadataStr := c.Args().Get(1) + + var metadata map[string]interface{} + err := json.Unmarshal([]byte(metadataStr), &metadata) + if err != nil { + return err + } + + peerClient, err := grpc.NewClient(grpcAddr) + if err != nil { + return err + } + defer func() { + err := peerClient.Close() + if err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + } + }() + + err = peerClient.SetNode(nodeId, metadata) + if err != nil { + return err + } + + return nil +} diff --git a/cmd/blast/set_state.go b/cmd/blast/set_state.go new file mode 100644 index 0000000..6f7a334 --- /dev/null +++ b/cmd/blast/set_state.go @@ -0,0 +1,70 @@ +// Copyright (c) 2019 Minoru Osuka +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "os" + + "github.com/mosuka/blast/grpc" + "github.com/urfave/cli" +) + +func execSetState(c *cli.Context) error { + grpcAddr := c.String("grpc-addr") + + key := c.Args().Get(0) + if key == "" { + err := errors.New("key argument must be set") + return err + } + + valueStr := c.Args().Get(1) + if valueStr == "" { + err := errors.New("value argument must be set") + return err + } + + var value interface{} + err := json.Unmarshal([]byte(valueStr), &value) + if err != nil { + switch err.(type) { + case *json.SyntaxError: + value = valueStr + default: + return err + } + } + + client, err := grpc.NewClient(grpcAddr) + if err != nil { + return err + } + defer func() { + err := client.Close() + if err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + } + }() + + err = client.SetState(key, value) + if err != nil { + return err + } + + return nil +} diff --git a/cmd/blast-manager/snapshot.go b/cmd/blast/snapshot.go similarity index 88% rename from cmd/blast-manager/snapshot.go rename to cmd/blast/snapshot.go index 5719978..7e3ec46 100644 --- a/cmd/blast-manager/snapshot.go +++ b/cmd/blast/snapshot.go @@ -18,21 +18,21 @@ import ( "fmt" "os" - "github.com/mosuka/blast/manager" + "github.com/mosuka/blast/grpc" "github.com/urfave/cli" ) func execSnapshot(c *cli.Context) error { grpcAddr := c.String("grpc-addr") - client, err := manager.NewGRPCClient(grpcAddr) + client, err := grpc.NewClient(grpcAddr) if err != nil { return err } defer func() { err := client.Close() if err != nil { - fmt.Fprintln(os.Stderr, err) + _, _ = fmt.Fprintln(os.Stderr, err) } }() diff --git a/cmd/blast-indexer/stats.go b/cmd/blast/watch_cluster.go similarity index 55% rename from cmd/blast-indexer/stats.go rename to cmd/blast/watch_cluster.go index cdc8cf7..cb2e267 100644 --- a/cmd/blast-indexer/stats.go +++ b/cmd/blast/watch_cluster.go @@ -18,50 +18,65 @@ import ( "encoding/json" "errors" "fmt" + "io" + "log" "os" - "github.com/mosuka/blast/indexer" + "github.com/mosuka/blast/grpc" "github.com/mosuka/blast/protobuf" "github.com/urfave/cli" ) -func execStats(c *cli.Context) error { +func execWatchCluster(c *cli.Context) error { grpcAddr := c.String("grpc-addr") - client, err := indexer.NewGRPCClient(grpcAddr) + client, err := grpc.NewClient(grpcAddr) if err != nil { return err } defer func() { err := client.Close() if err != nil { - fmt.Fprintln(os.Stderr, err) + _, _ = fmt.Fprintln(os.Stderr, err) } }() - resp, err := client.GetIndexStats() + err = execGetCluster(c) if err != nil { return err } - // Any -> map[string]interface{} - var statsMap *map[string]interface{} - statsInstance, err := protobuf.MarshalAny(resp.Stats) + watchClient, err := client.WatchCluster() if err != nil { return err } - if statsInstance == nil { - return errors.New("nil") - } - statsMap = statsInstance.(*map[string]interface{}) - // map[string]interface -> []byte - fieldsBytes, err := json.MarshalIndent(statsMap, "", " ") - if err != nil { - return err - } + for { + resp, err := watchClient.Recv() + if err == io.EOF { + break + } + if err != nil { + log.Println(err.Error()) + break + } - fmt.Fprintln(os.Stdout, fmt.Sprintf("%v\n", string(fieldsBytes))) + cluster, err := protobuf.MarshalAny(resp.Cluster) + if err != nil { + return err + } + if cluster == nil { + return errors.New("nil") + } + + var clusterBytes []byte + clusterMap := *cluster.(*map[string]interface{}) + clusterBytes, err = json.MarshalIndent(clusterMap, "", " ") + if err != nil { + return err + } + _, _ = fmt.Fprintln(os.Stdout, fmt.Sprintf("%v", string(clusterBytes))) + } return nil } diff --git a/cmd/blast/watch_state.go b/cmd/blast/watch_state.go new file mode 100644 index 0000000..3d5bb3c --- /dev/null +++ b/cmd/blast/watch_state.go @@ -0,0 +1,87 @@ +// Copyright (c) 2019 Minoru Osuka +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "log" + "os" + + "github.com/mosuka/blast/grpc" + "github.com/mosuka/blast/protobuf" + "github.com/urfave/cli" +) + +func execWatchState(c *cli.Context) error { + grpcAddr := c.String("grpc-addr") + + key := c.Args().Get(0) + + client, err := grpc.NewClient(grpcAddr) + if err != nil { + return err + } + defer func() { + err := client.Close() + if err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + } + }() + + watchClient, err := client.WatchState(key) + if err != nil { + return err + } + + for { + resp, err := watchClient.Recv() + if err == io.EOF { + break + } + if err != nil { + log.Println(err.Error()) + break + } + + value, err := protobuf.MarshalAny(resp.Value) + if err != nil { + return err + } + if value == nil { + return errors.New("nil") + } + + var valueBytes []byte + switch value.(type) { + case *map[string]interface{}: + valueMap := *value.(*map[string]interface{}) + valueBytes, err = json.MarshalIndent(valueMap, "", " ") + if err != nil { + return err + } + _, _ = fmt.Fprintln(os.Stdout, fmt.Sprintf("%s %s %v", resp.Command.String(), resp.Key, string(valueBytes))) + case *string: + valueStr := *value.(*string) + _, _ = fmt.Fprintln(os.Stdout, fmt.Sprintf("%s %s %s", resp.Command.String(), resp.Key, valueStr)) + default: + _, _ = fmt.Fprintln(os.Stdout, fmt.Sprintf("%s %s %v", resp.Command.String(), resp.Key, &value)) + } + } + + return nil +} diff --git a/cmd/blast-manager/start.go b/cmd/blastd/dispatcher.go similarity index 59% rename from cmd/blast-manager/start.go rename to cmd/blastd/dispatcher.go index 43ebca8..c9b64b3 100644 --- a/cmd/blast-manager/start.go +++ b/cmd/blastd/dispatcher.go @@ -19,34 +19,31 @@ import ( "os/signal" "syscall" + "github.com/mosuka/blast/dispatcher" "github.com/mosuka/logutils" - - "github.com/mosuka/blast/manager" "github.com/urfave/cli" ) -func execStart(c *cli.Context) error { - nodeId := c.String("node-id") - bindAddr := c.String("bind-addr") - grpcAddr := c.String("grpc-addr") - httpAddr := c.String("http-addr") - dataDir := c.String("data-dir") - joinAddr := c.String("join-addr") +func startDispatcher(c *cli.Context) error { + logLevel := c.GlobalString("log-level") + logFilename := c.GlobalString("log-file") + logMaxSize := c.GlobalInt("log-max-size") + logMaxBackups := c.GlobalInt("log-max-backups") + logMaxAge := c.GlobalInt("log-max-age") + logCompress := c.GlobalBool("log-compress") - logLevel := c.String("log-level") - logFilename := c.String("log-file") - logMaxSize := c.Int("log-max-size") - logMaxBackups := c.Int("log-max-backups") - logMaxAge := c.Int("log-max-age") - logCompress := c.Bool("log-compress") + httpAccessLogFilename := c.GlobalString("http-access-log-file") + httpAccessLogMaxSize := c.GlobalInt("http-access-log-max-size") + httpAccessLogMaxBackups := c.GlobalInt("http-access-log-max-backups") + httpAccessLogMaxAge := c.GlobalInt("http-access-log-max-age") + httpAccessLogCompress := c.GlobalBool("http-access-log-compress") - httpAccessLogFilename := c.String("http-access-log-file") - httpAccessLogMaxSize := c.Int("http-access-log-max-size") - httpAccessLogMaxBackups := c.Int("http-access-log-max-backups") - httpAccessLogMaxAge := c.Int("http-access-log-max-age") - httpAccessLogCompress := c.Bool("http-access-log-compress") + managerAddr := c.String("manager-addr") + + grpcAddr := c.String("grpc-addr") + httpAddr := c.String("http-addr") - // Create logger + // create logger logger := logutils.NewLogger( logLevel, logFilename, @@ -56,7 +53,7 @@ func execStart(c *cli.Context) error { logCompress, ) - // Create HTTP access logger + // create HTTP access logger httpAccessLogger := logutils.NewApacheCombinedLogger( httpAccessLogFilename, httpAccessLogMaxSize, @@ -65,7 +62,7 @@ func execStart(c *cli.Context) error { httpAccessLogCompress, ) - svr, err := manager.NewServer(nodeId, bindAddr, grpcAddr, httpAddr, dataDir, joinAddr, logger, httpAccessLogger) + svr, err := dispatcher.NewServer(managerAddr, grpcAddr, httpAddr, logger, httpAccessLogger) if err != nil { return err } diff --git a/cmd/blastd/indexer.go b/cmd/blastd/indexer.go new file mode 100644 index 0000000..c2855f3 --- /dev/null +++ b/cmd/blastd/indexer.go @@ -0,0 +1,150 @@ +// Copyright (c) 2019 Minoru Osuka +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "io/ioutil" + "os" + "os/signal" + "syscall" + + "github.com/blevesearch/bleve/mapping" + "github.com/mosuka/blast/indexer" + "github.com/mosuka/logutils" + "github.com/urfave/cli" +) + +func startIndexer(c *cli.Context) error { + logLevel := c.GlobalString("log-level") + logFilename := c.GlobalString("log-file") + logMaxSize := c.GlobalInt("log-max-size") + logMaxBackups := c.GlobalInt("log-max-backups") + logMaxAge := c.GlobalInt("log-max-age") + logCompress := c.GlobalBool("log-compress") + + httpAccessLogFilename := c.GlobalString("http-access-log-file") + httpAccessLogMaxSize := c.GlobalInt("http-access-log-max-size") + httpAccessLogMaxBackups := c.GlobalInt("http-access-log-max-backups") + httpAccessLogMaxAge := c.GlobalInt("http-access-log-max-age") + httpAccessLogCompress := c.GlobalBool("http-access-log-compress") + + managerAddr := c.String("manager-addr") + clusterId := c.String("cluster-id") + + nodeId := c.String("node-id") + bindAddr := c.String("bind-addr") + grpcAddr := c.String("grpc-addr") + httpAddr := c.String("http-addr") + dataDir := c.String("data-dir") + peerAddr := c.String("peer-addr") + + indexMappingFile := c.String("index-mapping-file") + indexType := c.String("index-type") + indexStorageType := c.String("index-storage-type") + + // create logger + logger := logutils.NewLogger( + logLevel, + logFilename, + logMaxSize, + logMaxBackups, + logMaxAge, + logCompress, + ) + + // create HTTP access logger + httpAccessLogger := logutils.NewApacheCombinedLogger( + httpAccessLogFilename, + httpAccessLogMaxSize, + httpAccessLogMaxBackups, + httpAccessLogMaxAge, + httpAccessLogCompress, + ) + + // metadata + metadata := map[string]interface{}{ + "bind_addr": bindAddr, + "grpc_addr": grpcAddr, + "http_addr": httpAddr, + "data_dir": dataDir, + } + + // index mapping + indexMapping := mapping.NewIndexMapping() + if indexMappingFile != "" { + _, err := os.Stat(indexMappingFile) + if err == nil { + // read index mapping file + f, err := os.Open(indexMappingFile) + if err != nil { + return err + } + defer func() { + _ = f.Close() + }() + + b, err := ioutil.ReadAll(f) + if err != nil { + return err + } + + err = json.Unmarshal(b, indexMapping) + if err != nil { + return err + } + } else if os.IsNotExist(err) { + return err + } + } + err := indexMapping.Validate() + if err != nil { + return err + } + + // IndexMappingImpl -> JSON + indexMappingJSON, err := json.Marshal(indexMapping) + if err != nil { + return err + } + // JSON -> map[string]interface{} + var indexMappingMap map[string]interface{} + err = json.Unmarshal(indexMappingJSON, &indexMappingMap) + if err != nil { + return err + } + + indexConfig := map[string]interface{}{ + "index_mapping": indexMappingMap, + "index_type": indexType, + "index_storage_type": indexStorageType, + } + + svr, err := indexer.NewServer(managerAddr, clusterId, nodeId, metadata, peerAddr, indexConfig, logger, httpAccessLogger) + if err != nil { + return err + } + + quitCh := make(chan os.Signal, 1) + signal.Notify(quitCh, os.Kill, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + + go svr.Start() + + <-quitCh + + svr.Stop() + + return nil +} diff --git a/cmd/blastd/main.go b/cmd/blastd/main.go new file mode 100644 index 0000000..ccb949a --- /dev/null +++ b/cmd/blastd/main.go @@ -0,0 +1,245 @@ +// Copyright (c) 2019 Minoru Osuka +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + "path" + + "github.com/blevesearch/bleve" + "github.com/mosuka/blast/version" + "github.com/urfave/cli" +) + +func main() { + app := cli.NewApp() + app.Name = path.Base(os.Args[0]) + app.Usage = "blastd" + app.Version = version.Version + app.Flags = []cli.Flag{ + cli.StringFlag{ + Name: "log-level", + Value: "INFO", + Usage: "Log level", + }, + cli.StringFlag{ + Name: "log-file", + Value: os.Stderr.Name(), + Usage: "Log file", + }, + cli.IntFlag{ + Name: "log-max-size", + Value: 500, + Usage: "Max size of a log file (megabytes)", + }, + cli.IntFlag{ + Name: "log-max-backups", + Value: 3, + Usage: "Max backup count of log files", + }, + cli.IntFlag{ + Name: "log-max-age", + Value: 30, + Usage: "Max age of a log file (days)", + }, + cli.BoolFlag{ + Name: "log-compress", + Usage: "Compress a log file", + }, + cli.StringFlag{ + Name: "http-access-log-file", + Value: os.Stderr.Name(), + Usage: "HTTP access log file", + }, + cli.IntFlag{ + Name: "http-access-log-max-size", + Value: 500, + Usage: "Max size of a HTTP access log file (megabytes)", + }, + cli.IntFlag{ + Name: "http-access-log-max-backups", + Value: 3, + Usage: "Max backup count of HTTP access log files", + }, + cli.IntFlag{ + Name: "http-access-log-max-age", + Value: 30, + Usage: "Max age of a HTTP access log file (days)", + }, + cli.BoolFlag{ + Name: "http-access-log-compress", + Usage: "Compress a HTTP access log", + }, + } + app.Authors = []cli.Author{ + { + Name: "mosuka", + Email: "minoru.osuka@gmail.com", + }, + } + app.Commands = []cli.Command{ + { + Name: "indexer", + Usage: "Start indexer", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "manager-addr", + Value: "", + Usage: "Manager address", + }, + cli.StringFlag{ + Name: "cluster-id", + Value: "default", + Usage: "Cluster ID", + }, + cli.StringFlag{ + Name: "node-id", + Value: "indexer1", + Usage: "Node ID", + }, + cli.StringFlag{ + Name: "bind-addr", + Value: ":5000", + Usage: "Raft bind address", + }, + cli.StringFlag{ + Name: "grpc-addr", + Value: ":5001", + Usage: "gRPC Server listen address", + }, + cli.StringFlag{ + Name: "http-addr", + Value: ":5002", + Usage: "HTTP server listen address", + }, + cli.StringFlag{ + Name: "data-dir", + Value: "/tmp/blast-index", + Usage: "Data directory", + }, + cli.StringFlag{ + Name: "peer-addr", + Value: "", + Usage: "Existing gRPC server listen address to join to the cluster", + }, + cli.StringFlag{ + Name: "index-mapping-file", + Value: "", + Usage: "Path to a file containing a JSON representation of an index mapping to use", + }, + cli.StringFlag{ + Name: "index-type", + Value: bleve.Config.DefaultIndexType, + Usage: "Index storage type to use", + }, + cli.StringFlag{ + Name: "index-storage-type", + Value: bleve.Config.DefaultKVStore, + Usage: "Index storage type to use", + }, + }, + Action: startIndexer, + }, + { + Name: "manager", + Usage: "Start manager", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "node-id", + Value: "", + Usage: "Node ID", + }, + cli.StringFlag{ + Name: "bind-addr", + Value: ":15000", + Usage: "Raft bind address", + }, + cli.StringFlag{ + Name: "grpc-addr", + Value: ":15001", + Usage: "gRPC Server listen address", + }, + cli.StringFlag{ + Name: "http-addr", + Value: ":15002", + Usage: "HTTP server listen address", + }, + cli.StringFlag{ + Name: "data-dir", + Value: "./", + Usage: "Data directory", + }, + cli.StringFlag{ + Name: "peer-addr", + Value: "", + Usage: "Existing gRPC server listen address to join to the cluster", + }, + cli.StringFlag{ + Name: "index-mapping-file", + Value: "", + Usage: "Path to a file containing a JSON representation of an index mapping to use", + }, + cli.StringFlag{ + Name: "index-type", + Value: bleve.Config.DefaultIndexType, + Usage: "Index storage type to use", + }, + cli.StringFlag{ + Name: "index-storage-type", + Value: bleve.Config.DefaultKVStore, + Usage: "Index storage type to use", + }, + }, + Action: startManager, + }, + { + Name: "dispatcher", + Usage: "Start dispatcher", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "manager-addr", + Value: ":15001", + Usage: "Manager address", + }, + cli.StringFlag{ + Name: "grpc-addr", + Value: ":25001", + Usage: "gRPC Server listen address", + }, + cli.StringFlag{ + Name: "http-addr", + Value: ":25002", + Usage: "HTTP server listen address", + }, + }, + Action: startDispatcher, + }, + } + + cli.HelpFlag = cli.BoolFlag{ + Name: "help, h", + Usage: "Show this message", + } + cli.VersionFlag = cli.BoolFlag{ + Name: "version, v", + Usage: "Print the version", + } + + err := app.Run(os.Args) + if err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + } +} diff --git a/cmd/blastd/manager.go b/cmd/blastd/manager.go new file mode 100644 index 0000000..893cf76 --- /dev/null +++ b/cmd/blastd/manager.go @@ -0,0 +1,147 @@ +// Copyright (c) 2019 Minoru Osuka +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "io/ioutil" + "os" + "os/signal" + "syscall" + + "github.com/blevesearch/bleve/mapping" + "github.com/mosuka/blast/manager" + "github.com/mosuka/logutils" + "github.com/urfave/cli" +) + +func startManager(c *cli.Context) error { + logLevel := c.GlobalString("log-level") + logFilename := c.GlobalString("log-file") + logMaxSize := c.GlobalInt("log-max-size") + logMaxBackups := c.GlobalInt("log-max-backups") + logMaxAge := c.GlobalInt("log-max-age") + logCompress := c.GlobalBool("log-compress") + + httpAccessLogFilename := c.GlobalString("http-access-log-file") + httpAccessLogMaxSize := c.GlobalInt("http-access-log-max-size") + httpAccessLogMaxBackups := c.GlobalInt("http-access-log-max-backups") + httpAccessLogMaxAge := c.GlobalInt("http-access-log-max-age") + httpAccessLogCompress := c.GlobalBool("http-access-log-compress") + + nodeId := c.String("node-id") + bindAddr := c.String("bind-addr") + grpcAddr := c.String("grpc-addr") + httpAddr := c.String("http-addr") + dataDir := c.String("data-dir") + peerAddr := c.String("peer-addr") + + indexMappingFile := c.String("index-mapping-file") + indexType := c.String("index-type") + indexStorageType := c.String("index-storage-type") + + // create logger + logger := logutils.NewLogger( + logLevel, + logFilename, + logMaxSize, + logMaxBackups, + logMaxAge, + logCompress, + ) + + // create HTTP access logger + httpAccessLogger := logutils.NewApacheCombinedLogger( + httpAccessLogFilename, + httpAccessLogMaxSize, + httpAccessLogMaxBackups, + httpAccessLogMaxAge, + httpAccessLogCompress, + ) + + // metadata + metadata := map[string]interface{}{ + "bind_addr": bindAddr, + "grpc_addr": grpcAddr, + "http_addr": httpAddr, + "data_dir": dataDir, + } + + // index mapping + indexMapping := mapping.NewIndexMapping() + if indexMappingFile != "" { + _, err := os.Stat(indexMappingFile) + if err == nil { + // read index mapping file + f, err := os.Open(indexMappingFile) + if err != nil { + return err + } + defer func() { + _ = f.Close() + }() + + b, err := ioutil.ReadAll(f) + if err != nil { + return err + } + + err = json.Unmarshal(b, indexMapping) + if err != nil { + return err + } + } else if os.IsNotExist(err) { + return err + } + } + err := indexMapping.Validate() + if err != nil { + return err + } + + // IndexMappingImpl -> JSON + indexMappingJSON, err := json.Marshal(indexMapping) + if err != nil { + return err + } + // JSON -> map[string]interface{} + var indexMappingMap map[string]interface{} + err = json.Unmarshal(indexMappingJSON, &indexMappingMap) + if err != nil { + return err + } + + indexConfig := map[string]interface{}{ + "index_mapping": indexMappingMap, + "index_type": indexType, + "index_storage_type": indexStorageType, + } + + svr, err := manager.NewServer(nodeId, metadata, peerAddr, indexConfig, logger, httpAccessLogger) + if err != nil { + return err + } + + quitCh := make(chan os.Signal, 1) + signal.Notify(quitCh, os.Kill, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + + go svr.Start() + + <-quitCh + + svr.Stop() + + return nil +} diff --git a/dispatcher/grpc_service.go b/dispatcher/grpc_service.go new file mode 100644 index 0000000..18bd545 --- /dev/null +++ b/dispatcher/grpc_service.go @@ -0,0 +1,799 @@ +// Copyright (c) 2019 Minoru Osuka +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dispatcher + +import ( + "context" + "errors" + "hash/fnv" + "io" + "log" + "math/rand" + "reflect" + "sort" + "sync" + "time" + + "github.com/blevesearch/bleve" + "github.com/blevesearch/bleve/search" + "github.com/golang/protobuf/ptypes/any" + "github.com/hashicorp/raft" + "github.com/mosuka/blast/grpc" + "github.com/mosuka/blast/protobuf" + "github.com/mosuka/blast/sortutils" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type GRPCService struct { + *grpc.Service + + managerAddr string + + logger *log.Logger + + managers map[string]interface{} + managerClients map[string]*grpc.Client + watchManagersStopCh chan struct{} + watchManagersDoneCh chan struct{} + + indexers map[string]interface{} + indexerClients map[string]map[string]*grpc.Client + watchIndexersStopCh chan struct{} + watchIndexersDoneCh chan struct{} +} + +func NewGRPCService(managerAddr string, logger *log.Logger) (*GRPCService, error) { + return &GRPCService{ + managerAddr: managerAddr, + logger: logger, + + managers: make(map[string]interface{}, 0), + managerClients: make(map[string]*grpc.Client, 0), + + indexers: make(map[string]interface{}, 0), + indexerClients: make(map[string]map[string]*grpc.Client, 0), + }, nil +} + +func (s *GRPCService) Start() error { + s.logger.Print("[INFO] start watching managers") + go s.startWatchManagers(500 * time.Millisecond) + + s.logger.Print("[INFO] start watching indexers") + go s.startWatchIndexers(500 * time.Millisecond) + + return nil +} + +func (s *GRPCService) Stop() error { + s.logger.Print("[INFO] stop watching managers") + s.stopWatchManagers() + + s.logger.Print("[INFO] stop watching indexers") + s.stopWatchIndexers() + + return nil +} + +func (s *GRPCService) getManagerClient() (*grpc.Client, error) { + var client *grpc.Client + + for id, node := range s.managers { + state := node.(map[string]interface{})["state"].(string) + if state != raft.Shutdown.String() { + + if _, exist := s.managerClients[id]; exist { + client = s.managerClients[id] + break + } else { + s.logger.Printf("[DEBUG] %v does not exist", id) + } + } + } + + if client == nil { + return nil, errors.New("client does not exist") + } + + return client, nil +} + +func (s *GRPCService) getInitialManagers(managerAddr string) (map[string]interface{}, error) { + client, err := grpc.NewClient(s.managerAddr) + defer func() { + err := client.Close() + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + return + }() + if err != nil { + s.logger.Printf("[ERR] %v", err) + return nil, err + } + + managers, err := client.GetCluster() + if err != nil { + s.logger.Printf("[ERR] %v", err) + return nil, err + } + + return managers, nil +} + +func (s *GRPCService) startWatchManagers(checkInterval time.Duration) { + s.logger.Printf("[INFO] start watching a cluster") + + s.watchManagersStopCh = make(chan struct{}) + s.watchManagersDoneCh = make(chan struct{}) + + defer func() { + close(s.watchManagersDoneCh) + }() + + var err error + + // get initial managers + s.managers, err = s.getInitialManagers(s.managerAddr) + if err != nil { + s.logger.Printf("[ERR] %v", err) + return + } + s.logger.Printf("[DEBUG] %v", s.managers) + + // create clients for managers + for id, node := range s.managers { + metadata := node.(map[string]interface{})["metadata"].(map[string]interface{}) + + s.logger.Printf("[DEBUG] create client for %s", metadata["grpc_addr"].(string)) + + client, err := grpc.NewClient(metadata["grpc_addr"].(string)) + if err != nil { + s.logger.Printf("[ERR] %v", err) + continue + } + s.managerClients[id] = client + } + + for { + select { + case <-s.watchManagersStopCh: + s.logger.Print("[DEBUG] receive request that stop watching managers") + return + default: + // get active client for manager + client, err := s.getManagerClient() + if err != nil { + s.logger.Printf("[ERR] %v", err) + continue + } + + // create stream + stream, err := client.WatchCluster() + if err != nil { + st, _ := status.FromError(err) + switch st.Code() { + case codes.Canceled: + s.logger.Printf("[DEBUG] %v", err) + default: + s.logger.Printf("[ERR] %v", err) + } + continue + } + + // wait for receive cluster updates from stream + s.logger.Print("[DEBUG] wait for receive cluster updates from stream") + resp, err := stream.Recv() + if err == io.EOF { + continue + } + if err != nil { + st, _ := status.FromError(err) + switch st.Code() { + case codes.Canceled: + s.logger.Printf("[DEBUG] %v", err) + default: + s.logger.Printf("[ERR] %v", err) + } + continue + } + + // get current manager cluster + cluster, err := protobuf.MarshalAny(resp.Cluster) + if err != nil { + s.logger.Printf("[ERR] %v", err) + continue + } + if cluster == nil { + s.logger.Print("[ERR] nil") + continue + } + managers := *cluster.(*map[string]interface{}) + + // compare previous manager with current manager + if !reflect.DeepEqual(s.managers, managers) { + s.logger.Printf("[INFO] %v", managers) + + // close the client for left manager node + for id := range s.managers { + if _, managerExists := managers[id]; !managerExists { + if _, clientExists := s.managerClients[id]; clientExists { + client := s.managerClients[id] + + s.logger.Printf("[DEBUG] close client for %s", client.GetAddress()) + err = client.Close() + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + + delete(s.managerClients, id) + } + } + } + + // keep current manager cluster + s.managers = managers + } + } + } +} + +func (s *GRPCService) stopWatchManagers() { + // close clients + s.logger.Printf("[INFO] close manager clients") + for _, client := range s.managerClients { + s.logger.Printf("[DEBUG] close manager client for %s", client.GetAddress()) + err := client.Close() + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + } + + // stop watching managers + if s.watchManagersStopCh != nil { + s.logger.Printf("[INFO] stop watching managers") + close(s.watchManagersStopCh) + } + + // wait for stop watching managers has done + s.logger.Printf("[INFO] wait for stop watching managers has done") + <-s.watchManagersDoneCh +} + +func (s *GRPCService) startWatchIndexers(checkInterval time.Duration) { + s.logger.Printf("[INFO] start watching a cluster") + + s.watchIndexersStopCh = make(chan struct{}) + s.watchIndexersDoneCh = make(chan struct{}) + + defer func() { + close(s.watchIndexersDoneCh) + }() + + // wait for manager available + s.logger.Print("[INFO] wait for manager clients are available") + for { + if len(s.managerClients) > 0 { + s.logger.Print("[INFO] manager clients are available") + break + } + time.Sleep(100 * time.Millisecond) + } + + // get active client for manager + client, err := s.getManagerClient() + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + + // get initial indexers + clusters, err := client.GetState("/cluster_config/clusters/") + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + if clusters == nil { + s.logger.Print("[ERR] nil") + } + s.indexers = *clusters.(*map[string]interface{}) + + // create clients for indexer + for clusterId, ins := range s.indexers { + cluster := ins.(map[string]interface{}) + for nodeId, node := range cluster["nodes"].(map[string]interface{}) { + metadata := node.(map[string]interface{})["metadata"].(map[string]interface{}) + + s.logger.Printf("[DEBUG] create indexer client for %s at %s", metadata["grpc_addr"].(string), clusterId) + + client, err := grpc.NewClient(metadata["grpc_addr"].(string)) + if err != nil { + s.logger.Printf("[ERR] %v", err) + continue + } + + if _, exist := s.indexerClients[clusterId]; !exist { + s.indexerClients[clusterId] = make(map[string]*grpc.Client) + } + + s.indexerClients[clusterId][nodeId] = client + } + } + + for { + select { + case <-s.watchIndexersStopCh: + s.logger.Print("[DEBUG] receive request that stop watching indexers") + return + default: + // get active client for manager + client, err = s.getManagerClient() + if err != nil { + s.logger.Printf("[ERR] %v", err) + continue + } + + // create stream + stream, err := client.WatchState("/cluster_config/clusters/") + if err != nil { + st, _ := status.FromError(err) + switch st.Code() { + case codes.Canceled: + s.logger.Printf("[DEBUG] %s: %v", client.GetAddress(), err) + default: + s.logger.Printf("[ERR] %s: %v", client.GetAddress(), err) + } + continue + } + + // wait for receive cluster updates from stream + s.logger.Print("[DEBUG] wait for receive cluster updates from stream") + resp, err := stream.Recv() + if err == io.EOF { + continue + } + if err != nil { + st, _ := status.FromError(err) + switch st.Code() { + case codes.Canceled: + s.logger.Printf("[DEBUG] %v", err) + default: + s.logger.Printf("[ERR] %v", err) + } + continue + } + log.Printf("[DEBUG] %v", resp) + + // get current indexer cluster + cluster, err := client.GetState("/cluster_config/clusters/") + if err != nil { + s.logger.Printf("[ERR] %v", err) + continue + } + if cluster == nil { + s.logger.Print("[ERR] nil") + continue + } + indexers := *cluster.(*map[string]interface{}) + + // compare previous manager with current manager + if !reflect.DeepEqual(s.indexers, indexers) { + s.logger.Printf("[INFO] %v", indexers) + + } + } + } +} + +func (s *GRPCService) stopWatchIndexers() { + // close clients + s.logger.Printf("[INFO] close indexer clients") + for clusterId, cluster := range s.indexerClients { + for _, client := range cluster { + s.logger.Printf("[DEBUG] close indexer client for %s at %s", client.GetAddress(), clusterId) + err := client.Close() + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + } + } + + // stop watching managers + if s.watchIndexersStopCh != nil { + s.logger.Printf("[INFO] stop watching indexers") + close(s.watchIndexersStopCh) + } + + // wait for stop watching indexers has done + s.logger.Printf("[INFO] wait for stop watching indexers has done") + <-s.watchIndexersDoneCh +} + +func (s *GRPCService) getIndexerClients() map[string]*grpc.Client { + indexerClients := make(map[string]*grpc.Client, 0) + + for clusterId, cluster := range s.indexerClients { + nodeIds := make([]string, 0) + for nodeId := range cluster { + nodeIds = append(nodeIds, nodeId) + } + + // pick a client at random + nodeId := nodeIds[rand.New(rand.NewSource(time.Now().UnixNano())).Intn(len(nodeIds))] + + indexerClients[clusterId] = s.indexerClients[clusterId][nodeId] + } + + return indexerClients +} + +func (s *GRPCService) GetDocument(ctx context.Context, req *protobuf.GetDocumentRequest) (*protobuf.GetDocumentResponse, error) { + start := time.Now() + defer RecordMetrics(start, "get") + + indexerClients := s.getIndexerClients() + + // cluster id list sorted by cluster id + clusterIds := make([]string, 0) + for clusterId := range indexerClients { + clusterIds = append(clusterIds, clusterId) + sort.Strings(clusterIds) + } + + type respVal struct { + clusterId string + fields map[string]interface{} + err error + } + + // create response channel + respChan := make(chan respVal, len(clusterIds)) + + wg := &sync.WaitGroup{} + for clusterId, client := range indexerClients { + wg.Add(1) + go func(clusterId string, client *grpc.Client, id string, respChan chan respVal) { + // index documents + fields, err := client.GetDocument(id) + wg.Done() + respChan <- respVal{ + clusterId: clusterId, + fields: fields, + err: err, + } + }(clusterId, client, req.Id, respChan) + } + wg.Wait() + + // close response channel + close(respChan) + + // summarize responses + var fields map[string]interface{} + for r := range respChan { + if r.fields != nil { + fields = r.fields + } + if r.err != nil { + s.logger.Printf("[ERR] %s %v", r.clusterId, r.err) + } + } + + resp := &protobuf.GetDocumentResponse{} + + fieldsAny := &any.Any{} + err := protobuf.UnmarshalAny(fields, fieldsAny) + if err != nil { + return resp, err + } + + // response + resp.Fields = fieldsAny + + return resp, nil +} + +func (s *GRPCService) Search(ctx context.Context, req *protobuf.SearchRequest) (*protobuf.SearchResponse, error) { + start := time.Now() + defer RecordMetrics(start, "search") + + resp := &protobuf.SearchResponse{} + + indexerClients := s.getIndexerClients() + + // cluster id list sorted by cluster id + clusterIds := make([]string, 0) + for clusterId := range indexerClients { + clusterIds = append(clusterIds, clusterId) + sort.Strings(clusterIds) + } + + type respVal struct { + clusterId string + searchResult *bleve.SearchResult + err error + } + + // create response channel + respChan := make(chan respVal, len(clusterIds)) + + // create search request + ins, err := protobuf.MarshalAny(req.SearchRequest) + if err != nil { + return resp, err + } + searchRequest := ins.(*bleve.SearchRequest) + + // change to distributed search request + from := searchRequest.From + size := searchRequest.Size + searchRequest.From = 0 + searchRequest.Size = from + size + + wg := &sync.WaitGroup{} + for clusterId, client := range indexerClients { + wg.Add(1) + go func(clusterId string, client *grpc.Client, searchRequest *bleve.SearchRequest, respChan chan respVal) { + s.logger.Printf("[DEBUG] search %s", client.GetAddress()) + searchResult, err := client.Search(searchRequest) + wg.Done() + respChan <- respVal{ + clusterId: clusterId, + searchResult: searchResult, + err: err, + } + }(clusterId, client, searchRequest, respChan) + } + wg.Wait() + + // close response channel + close(respChan) + + // revert to original search request + searchRequest.From = from + searchRequest.Size = size + + // summarize responses + var searchResult *bleve.SearchResult + for r := range respChan { + if r.searchResult != nil { + if searchResult == nil { + searchResult = r.searchResult + } else { + searchResult.Merge(r.searchResult) + } + } + if r.err != nil { + s.logger.Printf("[ERR] %s %v", r.clusterId, r.err) + } + } + + // handle case where no results were successful + if searchResult == nil { + searchResult = &bleve.SearchResult{ + Status: &bleve.SearchStatus{ + Errors: make(map[string]error), + }, + } + } + + // sort all hits with the requested order + if len(searchRequest.Sort) > 0 { + sorter := sortutils.NewMultiSearchHitSorter(searchRequest.Sort, searchResult.Hits) + sort.Sort(sorter) + } + + // now skip over the correct From + if searchRequest.From > 0 && len(searchResult.Hits) > searchRequest.From { + searchResult.Hits = searchResult.Hits[searchRequest.From:] + } else if searchRequest.From > 0 { + searchResult.Hits = search.DocumentMatchCollection{} + } + + // now trim to the correct size + if searchRequest.Size > 0 && len(searchResult.Hits) > searchRequest.Size { + searchResult.Hits = searchResult.Hits[0:searchRequest.Size] + } + + // fix up facets + for name, fr := range searchRequest.Facets { + searchResult.Facets.Fixup(name, fr.Size) + } + + // fix up original request + searchResult.Request = searchRequest + searchDuration := time.Since(start) + searchResult.Took = searchDuration + + searchResultAny := &any.Any{} + err = protobuf.UnmarshalAny(searchResult, searchResultAny) + if err != nil { + return resp, err + } + + // response + resp.SearchResult = searchResultAny + + return resp, nil +} + +func (s *GRPCService) docIdHash(docId string) uint64 { + hash := fnv.New64() + _, err := hash.Write([]byte(docId)) + if err != nil { + return 0 + } + + return hash.Sum64() +} + +func (s *GRPCService) IndexDocument(stream protobuf.Blast_IndexDocumentServer) error { + indexerClients := s.getIndexerClients() + + // cluster id list sorted by cluster id + clusterIds := make([]string, 0) + for clusterId := range indexerClients { + clusterIds = append(clusterIds, clusterId) + sort.Strings(clusterIds) + } + + // initialize document list for each cluster + docSet := make(map[string][]map[string]interface{}, 0) + for _, clusterId := range clusterIds { + docSet[clusterId] = make([]map[string]interface{}, 0) + } + + for { + req, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + return status.Error(codes.Internal, err.Error()) + } + + // fields + ins, err := protobuf.MarshalAny(req.Fields) + if err != nil { + return status.Error(codes.Internal, err.Error()) + } + fields := *ins.(*map[string]interface{}) + + // document + doc := map[string]interface{}{ + "id": req.Id, + "fields": fields, + } + + // distribute documents to each cluster based on document id + docIdHash := s.docIdHash(req.Id) + clusterNum := uint64(len(indexerClients)) + clusterId := clusterIds[int(docIdHash%clusterNum)] + docSet[clusterId] = append(docSet[clusterId], doc) + } + + type respVal struct { + clusterId string + count int + err error + } + + // create response channel + respChan := make(chan respVal, len(clusterIds)) + + wg := &sync.WaitGroup{} + for clusterId, docs := range docSet { + wg.Add(1) + go func(clusterId string, docs []map[string]interface{}, respChan chan respVal) { + // index documents + count, err := indexerClients[clusterId].IndexDocument(docs) + wg.Done() + respChan <- respVal{ + clusterId: clusterId, + count: count, + err: err, + } + }(clusterId, docs, respChan) + } + wg.Wait() + + // close response channel + close(respChan) + + // summarize responses + totalCount := 0 + for r := range respChan { + if r.count >= 0 { + totalCount += r.count + } + if r.err != nil { + s.logger.Printf("[ERR] %s %v", r.clusterId, r.err) + } + } + + // response + resp := &protobuf.IndexDocumentResponse{ + Count: int32(totalCount), + } + + return stream.SendAndClose(resp) +} + +func (s *GRPCService) DeleteDocument(stream protobuf.Blast_DeleteDocumentServer) error { + indexerClients := s.getIndexerClients() + + // cluster id list sorted by cluster id + clusterIds := make([]string, 0) + for clusterId := range indexerClients { + clusterIds = append(clusterIds, clusterId) + sort.Strings(clusterIds) + } + + ids := make([]string, 0) + + for { + req, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + return status.Error(codes.Internal, err.Error()) + } + + ids = append(ids, req.Id) + } + + type respVal struct { + clusterId string + count int + err error + } + + // create response channel + respChan := make(chan respVal, len(clusterIds)) + + wg := &sync.WaitGroup{} + for clusterId, client := range indexerClients { + wg.Add(1) + go func(clusterId string, client *grpc.Client, ids []string, respChan chan respVal) { + // index documents + count, err := client.DeleteDocument(ids) + wg.Done() + respChan <- respVal{ + clusterId: clusterId, + count: count, + err: err, + } + }(clusterId, client, ids, respChan) + } + wg.Wait() + + // close response channel + close(respChan) + + // summarize responses + totalCount := len(ids) + for r := range respChan { + if r.err != nil { + s.logger.Printf("[ERR] %s %v", r.clusterId, r.err) + } + } + + // response + resp := &protobuf.DeleteDocumentResponse{ + Count: int32(totalCount), + } + + return stream.SendAndClose(resp) +} diff --git a/dispatcher/http_handler.go b/dispatcher/http_handler.go new file mode 100644 index 0000000..c626ba2 --- /dev/null +++ b/dispatcher/http_handler.go @@ -0,0 +1,468 @@ +// Copyright (c) 2019 Minoru Osuka +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dispatcher + +import ( + "encoding/json" + "io/ioutil" + "log" + "net/http" + "time" + + "github.com/blevesearch/bleve" + "github.com/gorilla/mux" + "github.com/mosuka/blast/errors" + "github.com/mosuka/blast/grpc" + blasthttp "github.com/mosuka/blast/http" + "github.com/mosuka/blast/version" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +func NewRouter(grpcAddr string, logger *log.Logger) (*blasthttp.Router, error) { + router, err := blasthttp.NewRouter(grpcAddr, logger) + if err != nil { + return nil, err + } + + router.StrictSlash(true) + + router.Handle("/", NewRootHandler(logger)).Methods("GET") + router.Handle("/documents", NewSetDocumentHandler(router.GRPCClient, logger)).Methods("PUT") + router.Handle("/documents", NewDeleteDocumentHandler(router.GRPCClient, logger)).Methods("DELETE") + router.Handle("/documents/{id}", NewGetDocumentHandler(router.GRPCClient, logger)).Methods("GET") + router.Handle("/documents/{id}", NewSetDocumentHandler(router.GRPCClient, logger)).Methods("PUT") + router.Handle("/documents/{id}", NewDeleteDocumentHandler(router.GRPCClient, logger)).Methods("DELETE") + router.Handle("/search", NewSearchHandler(router.GRPCClient, logger)).Methods("POST") + router.Handle("/metrics", promhttp.Handler()).Methods("GET") + + return router, nil +} + +type RootHandler struct { + logger *log.Logger +} + +func NewRootHandler(logger *log.Logger) *RootHandler { + return &RootHandler{ + logger: logger, + } +} + +func (h *RootHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + start := time.Now() + status := http.StatusOK + content := make([]byte, 0) + defer func() { + blasthttp.WriteResponse(w, content, status, h.logger) + blasthttp.RecordMetrics(start, status, w, r, h.logger) + }() + + msgMap := map[string]interface{}{ + "version": version.Version, + "status": status, + } + + content, err := blasthttp.NewJSONMessage(msgMap) + if err != nil { + h.logger.Printf("[ERR] %v", err) + } +} + +type GetHandler struct { + client *grpc.Client + logger *log.Logger +} + +func NewGetDocumentHandler(client *grpc.Client, logger *log.Logger) *GetHandler { + return &GetHandler{ + client: client, + logger: logger, + } +} + +func (h *GetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + start := time.Now() + httpStatus := http.StatusOK + content := make([]byte, 0) + defer func() { + blasthttp.WriteResponse(w, content, httpStatus, h.logger) + blasthttp.RecordMetrics(start, httpStatus, w, r, h.logger) + }() + + vars := mux.Vars(r) + + fields, err := h.client.GetDocument(vars["id"]) + if err != nil { + switch err { + case errors.ErrNotFound: + httpStatus = http.StatusNotFound + default: + httpStatus = http.StatusInternalServerError + } + + msgMap := map[string]interface{}{ + "message": err.Error(), + "status": httpStatus, + } + + content, err = blasthttp.NewJSONMessage(msgMap) + if err != nil { + h.logger.Printf("[ERR] %v", err) + } + + return + } + + // map[string]interface{} -> bytes + content, err = json.MarshalIndent(fields, "", " ") + if err != nil { + httpStatus = http.StatusInternalServerError + + msgMap := map[string]interface{}{ + "message": err.Error(), + "status": httpStatus, + } + + content, err = blasthttp.NewJSONMessage(msgMap) + if err != nil { + h.logger.Printf("[ERR] %v", err) + } + + return + } +} + +type IndexHandler struct { + client *grpc.Client + logger *log.Logger +} + +func NewSetDocumentHandler(client *grpc.Client, logger *log.Logger) *IndexHandler { + return &IndexHandler{ + client: client, + logger: logger, + } +} + +func (h *IndexHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + start := time.Now() + httpStatus := http.StatusOK + content := make([]byte, 0) + defer func() { + blasthttp.WriteResponse(w, content, httpStatus, h.logger) + blasthttp.RecordMetrics(start, httpStatus, w, r, h.logger) + }() + + // create documents + docs := make([]map[string]interface{}, 0) + + vars := mux.Vars(r) + id := vars["id"] + + bodyBytes, err := ioutil.ReadAll(r.Body) + if err != nil { + httpStatus = http.StatusInternalServerError + + msgMap := map[string]interface{}{ + "message": err.Error(), + "status": httpStatus, + } + + content, err = blasthttp.NewJSONMessage(msgMap) + if err != nil { + h.logger.Printf("[ERR] %v", err) + } + + return + } + + if id == "" { + // Indexing documents in bulk + err := json.Unmarshal(bodyBytes, &docs) + if err != nil { + httpStatus = http.StatusBadRequest + + msgMap := map[string]interface{}{ + "message": err.Error(), + "status": httpStatus, + } + + content, err = blasthttp.NewJSONMessage(msgMap) + if err != nil { + h.logger.Printf("[ERR] %v", err) + } + + return + } + } else { + // Indexing a document + var fields map[string]interface{} + err := json.Unmarshal(bodyBytes, &fields) + if err != nil { + httpStatus = http.StatusBadRequest + + msgMap := map[string]interface{}{ + "message": err.Error(), + "status": httpStatus, + } + + content, err = blasthttp.NewJSONMessage(msgMap) + if err != nil { + h.logger.Printf("[ERR] %v", err) + } + + return + } + + doc := map[string]interface{}{ + "id": id, + "fields": fields, + } + + docs = append(docs, doc) + } + + // index documents in bulk + result, err := h.client.IndexDocument(docs) + if err != nil { + httpStatus = http.StatusInternalServerError + + msgMap := map[string]interface{}{ + "message": err.Error(), + "status": httpStatus, + } + + content, err = blasthttp.NewJSONMessage(msgMap) + if err != nil { + h.logger.Printf("[ERR] %v", err) + } + + return + } + + content, err = json.MarshalIndent(result, "", " ") + if err != nil { + httpStatus = http.StatusInternalServerError + + msgMap := map[string]interface{}{ + "message": err.Error(), + "status": httpStatus, + } + + content, err = blasthttp.NewJSONMessage(msgMap) + if err != nil { + h.logger.Printf("[ERR] %v", err) + } + + return + } +} + +type DeleteHandler struct { + client *grpc.Client + logger *log.Logger +} + +func NewDeleteDocumentHandler(client *grpc.Client, logger *log.Logger) *DeleteHandler { + return &DeleteHandler{ + client: client, + logger: logger, + } +} + +func (h *DeleteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + start := time.Now() + httpStatus := http.StatusOK + content := make([]byte, 0) + defer func() { + blasthttp.WriteResponse(w, content, httpStatus, h.logger) + blasthttp.RecordMetrics(start, httpStatus, w, r, h.logger) + }() + + // create documents + ids := make([]string, 0) + + vars := mux.Vars(r) + id := vars["id"] + + bodyBytes, err := ioutil.ReadAll(r.Body) + if err != nil { + httpStatus = http.StatusInternalServerError + + msgMap := map[string]interface{}{ + "message": err.Error(), + "status": httpStatus, + } + + content, err = blasthttp.NewJSONMessage(msgMap) + if err != nil { + h.logger.Printf("[ERR] %v", err) + } + + return + } + + if id == "" { + // Deleting documents in bulk + err := json.Unmarshal(bodyBytes, &ids) + if err != nil { + httpStatus = http.StatusBadRequest + + msgMap := map[string]interface{}{ + "message": err.Error(), + "status": httpStatus, + } + + content, err = blasthttp.NewJSONMessage(msgMap) + if err != nil { + h.logger.Printf("[ERR] %v", err) + } + + return + } + } else { + // Deleting a document + ids = append(ids, id) + } + + // delete documents in bulk + result, err := h.client.DeleteDocument(ids) + if err != nil { + httpStatus = http.StatusInternalServerError + + msgMap := map[string]interface{}{ + "message": err.Error(), + "status": httpStatus, + } + + content, err = blasthttp.NewJSONMessage(msgMap) + if err != nil { + h.logger.Printf("[ERR] %v", err) + } + + return + } + + content, err = json.MarshalIndent(result, "", " ") + if err != nil { + httpStatus = http.StatusInternalServerError + + msgMap := map[string]interface{}{ + "message": err.Error(), + "status": httpStatus, + } + + content, err = blasthttp.NewJSONMessage(msgMap) + if err != nil { + h.logger.Printf("[ERR] %v", err) + } + + return + } +} + +type SearchHandler struct { + client *grpc.Client + logger *log.Logger +} + +func NewSearchHandler(client *grpc.Client, logger *log.Logger) *SearchHandler { + return &SearchHandler{ + client: client, + logger: logger, + } +} + +func (h *SearchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + start := time.Now() + httpStatus := http.StatusOK + content := make([]byte, 0) + defer func() { + blasthttp.WriteResponse(w, content, httpStatus, h.logger) + blasthttp.RecordMetrics(start, httpStatus, w, r, h.logger) + }() + + searchRequestBytes, err := ioutil.ReadAll(r.Body) + if err != nil { + httpStatus = http.StatusInternalServerError + + msgMap := map[string]interface{}{ + "message": err.Error(), + "status": httpStatus, + } + + content, err = blasthttp.NewJSONMessage(msgMap) + if err != nil { + h.logger.Printf("[ERR] %v", err) + } + + return + } + + // []byte -> bleve.SearchRequest + searchRequest := bleve.NewSearchRequest(nil) + if len(searchRequestBytes) > 0 { + err := json.Unmarshal(searchRequestBytes, searchRequest) + if err != nil { + httpStatus = http.StatusBadRequest + + msgMap := map[string]interface{}{ + "message": err.Error(), + "status": httpStatus, + } + + content, err = blasthttp.NewJSONMessage(msgMap) + if err != nil { + h.logger.Printf("[ERR] %v", err) + } + + return + } + } + + searchResult, err := h.client.Search(searchRequest) + if err != nil { + httpStatus = http.StatusInternalServerError + + msgMap := map[string]interface{}{ + "message": err.Error(), + "status": httpStatus, + } + + content, err = blasthttp.NewJSONMessage(msgMap) + if err != nil { + h.logger.Printf("[ERR] %v", err) + } + + return + } + + content, err = json.MarshalIndent(&searchResult, "", " ") + if err != nil { + httpStatus = http.StatusInternalServerError + + msgMap := map[string]interface{}{ + "message": err.Error(), + "status": httpStatus, + } + + content, err = blasthttp.NewJSONMessage(msgMap) + if err != nil { + h.logger.Printf("[ERR] %v", err) + } + + return + } +} diff --git a/dispatcher/metric.go b/dispatcher/metric.go new file mode 100644 index 0000000..adee6e0 --- /dev/null +++ b/dispatcher/metric.go @@ -0,0 +1,61 @@ +// Copyright (c) 2019 Minoru Osuka +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dispatcher + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +var ( + namespace = "blast" + subsystem = "federator" + + DurationSeconds = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "duration_seconds", + Help: "The index operation durations in seconds.", + }, + []string{ + "func", + }, + ) + OperationsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "operations_total", + Help: "The number of index operations.", + }, + []string{ + "func", + }, + ) +) + +func init() { + prometheus.MustRegister(DurationSeconds) + prometheus.MustRegister(OperationsTotal) +} + +func RecordMetrics(start time.Time, funcName string) { + DurationSeconds.With(prometheus.Labels{"func": funcName}).Observe(float64(time.Since(start)) / float64(time.Second)) + OperationsTotal.With(prometheus.Labels{"func": funcName}).Inc() + + return +} diff --git a/dispatcher/server.go b/dispatcher/server.go new file mode 100644 index 0000000..5389b28 --- /dev/null +++ b/dispatcher/server.go @@ -0,0 +1,140 @@ +// Copyright (c) 2019 Minoru Osuka +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dispatcher + +import ( + "log" + + accesslog "github.com/mash/go-accesslog" + "github.com/mosuka/blast/grpc" + "github.com/mosuka/blast/http" +) + +type Server struct { + managerAddr string + clusterId string + + grpcAddr string + httpAddr string + + grpcService *GRPCService + grpcServer *grpc.Server + httpRouter *http.Router + httpServer *http.Server + + logger *log.Logger + httpLogger accesslog.Logger +} + +func NewServer(managerAddr string, grpcAddr string, httpAddr string, logger *log.Logger, httpLogger accesslog.Logger) (*Server, error) { + return &Server{ + managerAddr: managerAddr, + + grpcAddr: grpcAddr, + httpAddr: httpAddr, + + logger: logger, + httpLogger: httpLogger, + }, nil +} + +func (s *Server) Start() { + s.logger.Printf("[INFO] start coordinator") + + var err error + + // create gRPC service + s.grpcService, err = NewGRPCService(s.managerAddr, s.logger) + if err != nil { + s.logger.Printf("[ERR] %v", err) + return + } + + // create gRPC server + s.grpcServer, err = grpc.NewServer(s.grpcAddr, s.grpcService, s.logger) + if err != nil { + s.logger.Printf("[ERR] %v", err) + return + } + + // create HTTP router + s.httpRouter, err = NewRouter(s.grpcAddr, s.logger) + if err != nil { + s.logger.Printf("[ERR] %v", err) + return + } + + // create HTTP server + s.httpServer, err = http.NewServer(s.httpAddr, s.httpRouter, s.logger, s.httpLogger) + if err != nil { + s.logger.Printf("[ERR] %v", err) + return + } + + // start gRPC service + s.logger.Print("[INFO] start gRPC service") + go func() { + err := s.grpcService.Start() + if err != nil { + s.logger.Printf("[ERR] %v", err) + return + } + }() + + // start gRPC server + s.logger.Print("[INFO] start gRPC server") + go func() { + err := s.grpcServer.Start() + if err != nil { + s.logger.Printf("[ERR] %v", err) + return + } + }() + + // start HTTP server + s.logger.Print("[INFO] start HTTP server") + go func() { + _ = s.httpServer.Start() + }() +} + +func (s *Server) Stop() { + // stop HTTP server + s.logger.Printf("[INFO] stop HTTP server") + err := s.httpServer.Stop() + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + + // stop HTTP router + err = s.httpRouter.Close() + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + + // stop gRPC server + s.logger.Printf("[INFO] stop gRPC server") + err = s.grpcServer.Stop() + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + + // stop gRPC service + s.logger.Print("[INFO] stop gRPC service") + err = s.grpcService.Stop() + if err != nil { + s.logger.Printf("[ERR] %v", err) + } +} diff --git a/example/bulk_delete_wiki.json b/example/bulk_delete_wiki.json new file mode 100644 index 0000000..3f17c76 --- /dev/null +++ b/example/bulk_delete_wiki.json @@ -0,0 +1,6 @@ +[ + "arwiki_1", + "bgwiki_1", + "cawiki_1", + "zhwiki_1" +] diff --git a/example/docs_wiki.json b/example/bulk_index_wiki.json similarity index 100% rename from example/docs_wiki.json rename to example/bulk_index_wiki.json diff --git a/example/search_request.json b/example/search_request.json index 06e8994..c189f9f 100644 --- a/example/search_request.json +++ b/example/search_request.json @@ -8,12 +8,14 @@ "*" ], "sort": [ - "-_score" + "-_score", + "_id", + "-timestamp" ], "facets": { - "Contributor count": { + "Type count": { "size": 10, - "field": "contributor" + "field": "_type" }, "Timestamp range": { "size": 10, diff --git a/go.mod b/go.mod index 9b2a191..9e95dfd 100644 --- a/go.mod +++ b/go.mod @@ -25,17 +25,18 @@ require ( github.com/hashicorp/raft v1.0.0 github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea github.com/ikawaha/kagome.ipadic v1.0.1 // indirect + github.com/imdario/mergo v0.3.7 github.com/jmhodges/levigo v1.0.0 // indirect github.com/mash/go-accesslog v0.0.0-20180522074327-610c2be04217 github.com/mosuka/bbadger v0.0.0-20190319122948-67a91aedfe68 github.com/mosuka/logutils v0.1.2 - github.com/mosuka/maputils v0.1.2 github.com/pascaldekloe/goe v0.1.0 // indirect github.com/prometheus/client_golang v0.9.2 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 // indirect github.com/prometheus/common v0.2.0 // indirect github.com/prometheus/procfs v0.0.0-20190322151404-55ae3d9d5573 // indirect github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001 // indirect + github.com/stretchr/objx v0.1.1 github.com/syndtr/goleveldb v1.0.0 github.com/tebeka/snowball v0.0.0-20130405174319-16e884df4e19 // indirect github.com/tecbot/gorocksdb v0.0.0-20181010114359-8752a9433481 // indirect @@ -43,4 +44,5 @@ require ( golang.org/x/net v0.0.0-20190327214358-63eda1eb0650 // indirect google.golang.org/genproto v0.0.0-20190327125643-d831d65fe17d // indirect google.golang.org/grpc v1.19.1 + gopkg.in/yaml.v2 v2.2.2 ) diff --git a/go.sum b/go.sum index cffc11b..535ee7a 100644 --- a/go.sum +++ b/go.sum @@ -113,12 +113,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0j github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mosuka/bbadger v0.0.0-20190319122948-67a91aedfe68 h1:LE+XIZDiXr587to+tCWKYPTrtQOmJzOxzcwhiDQIJbE= github.com/mosuka/bbadger v0.0.0-20190319122948-67a91aedfe68/go.mod h1:qy5KaSXSrNqdWFS/e3wWNFXZPRDnqjX79iRhOveUpfc= -github.com/mosuka/logutils v0.1.1 h1:OUZEGdgsCfAwgtyRCpmORkP+Nl9ldoRawWCHaLB2z9k= -github.com/mosuka/logutils v0.1.1/go.mod h1:CpJK2hcZNUSbmmVP839jPubOrj7/ANH6F3chUtyIpuM= github.com/mosuka/logutils v0.1.2 h1:3mTh6ulzJGDv4Cp6JG4QIIiS/kJ7fLtycdJWv0XqZN4= github.com/mosuka/logutils v0.1.2/go.mod h1:CpJK2hcZNUSbmmVP839jPubOrj7/ANH6F3chUtyIpuM= -github.com/mosuka/maputils v0.1.2 h1:P71cr3V2WI3/K0dQF8CPnDSslaIWxqCT6G9jyrsiR9g= -github.com/mosuka/maputils v0.1.2/go.mod h1:eBVvvSa8IK7aZJdGqYltLQmp7XBnqd9J1/dRBZsdHLg= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -161,6 +157,7 @@ github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:K github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2 h1:JNEGSiWg6D3lcBCMCBqN3ELniXujt+0QNHLhNnO0w3s= github.com/steveyen/gtreap v0.0.0-20150807155958-0abe01ef9be2/go.mod h1:mjqs7N0Q6m5HpR7QfXVBZXZWSqTjQLeTujjA/xUp2uw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= diff --git a/grpc/client.go b/grpc/client.go new file mode 100644 index 0000000..f3d6876 --- /dev/null +++ b/grpc/client.go @@ -0,0 +1,462 @@ +// Copyright (c) 2019 Minoru Osuka +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package grpc + +import ( + "context" + "errors" + "math" + + "github.com/blevesearch/bleve" + "github.com/golang/protobuf/ptypes/any" + "github.com/golang/protobuf/ptypes/empty" + blasterrors "github.com/mosuka/blast/errors" + "github.com/mosuka/blast/protobuf" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type Client struct { + ctx context.Context + cancel context.CancelFunc + conn *grpc.ClientConn + client protobuf.BlastClient +} + +func NewContext() (context.Context, context.CancelFunc) { + baseCtx := context.TODO() + return context.WithCancel(baseCtx) +} + +func NewClient(address string) (*Client, error) { + ctx, cancel := NewContext() + + dialOpts := []grpc.DialOption{ + grpc.WithInsecure(), + grpc.WithDefaultCallOptions( + grpc.MaxCallSendMsgSize(math.MaxInt32), + grpc.MaxCallRecvMsgSize(math.MaxInt32), + ), + } + + conn, err := grpc.DialContext(ctx, address, dialOpts...) + if err != nil { + return nil, err + } + + return &Client{ + ctx: ctx, + cancel: cancel, + conn: conn, + client: protobuf.NewBlastClient(conn), + }, nil +} + +func (c *Client) Cancel() { + c.cancel() +} + +func (c *Client) Close() error { + c.Cancel() + if c.conn != nil { + return c.conn.Close() + } + + return c.ctx.Err() +} + +func (c *Client) GetAddress() string { + return c.conn.Target() +} + +func (c *Client) GetNodeMetadata(id string, opts ...grpc.CallOption) (map[string]interface{}, error) { + req := &protobuf.GetMetadataRequest{ + Id: id, + } + + resp, err := c.client.GetMetadata(c.ctx, req, opts...) + if err != nil { + st, _ := status.FromError(err) + + return nil, errors.New(st.Message()) + } + + ins, err := protobuf.MarshalAny(resp.Metadata) + + var metadata map[string]interface{} + if ins == nil { + return nil, errors.New("nil") + } + metadata = *ins.(*map[string]interface{}) + + return metadata, nil +} + +func (c *Client) GetNodeState(id string, opts ...grpc.CallOption) (string, error) { + req := &protobuf.GetNodeStateRequest{ + Id: id, + } + + resp, err := c.client.GetNodeState(c.ctx, req, opts...) + if err != nil { + st, _ := status.FromError(err) + + return "", errors.New(st.Message()) + } + + return resp.State, nil +} + +func (c *Client) GetNode(id string, opts ...grpc.CallOption) (map[string]interface{}, error) { + req := &protobuf.GetNodeRequest{ + Id: id, + } + + resp, err := c.client.GetNode(c.ctx, req, opts...) + if err != nil { + st, _ := status.FromError(err) + + return nil, errors.New(st.Message()) + } + + ins, err := protobuf.MarshalAny(resp.Metadata) + metadata := *ins.(*map[string]interface{}) + + node := map[string]interface{}{ + "metadata": metadata, + "state": resp.State, + } + + return node, nil +} + +func (c *Client) SetNode(id string, metadata map[string]interface{}, opts ...grpc.CallOption) error { + metadataAny := &any.Any{} + err := protobuf.UnmarshalAny(metadata, metadataAny) + if err != nil { + return err + } + + req := &protobuf.SetNodeRequest{ + Id: id, + Metadata: metadataAny, + } + + _, err = c.client.SetNode(c.ctx, req, opts...) + if err != nil { + return err + } + + return nil +} + +func (c *Client) DeleteNode(id string, opts ...grpc.CallOption) error { + req := &protobuf.DeleteNodeRequest{ + Id: id, + } + + _, err := c.client.DeleteNode(c.ctx, req, opts...) + if err != nil { + return err + } + + return nil +} + +func (c *Client) GetCluster(opts ...grpc.CallOption) (map[string]interface{}, error) { + resp, err := c.client.GetCluster(c.ctx, &empty.Empty{}, opts...) + if err != nil { + st, _ := status.FromError(err) + + return nil, errors.New(st.Message()) + } + + ins, err := protobuf.MarshalAny(resp.Cluster) + cluster := *ins.(*map[string]interface{}) + + return cluster, nil +} + +func (c *Client) WatchCluster(opts ...grpc.CallOption) (protobuf.Blast_WatchClusterClient, error) { + req := &empty.Empty{} + + watchClient, err := c.client.WatchCluster(c.ctx, req, opts...) + if err != nil { + st, _ := status.FromError(err) + return nil, errors.New(st.Message()) + } + + return watchClient, nil +} + +func (c *Client) Snapshot(opts ...grpc.CallOption) error { + _, err := c.client.Snapshot(c.ctx, &empty.Empty{}) + if err != nil { + st, _ := status.FromError(err) + + return errors.New(st.Message()) + } + + return nil +} + +func (c *Client) LivenessProbe(opts ...grpc.CallOption) (string, error) { + resp, err := c.client.LivenessProbe(c.ctx, &empty.Empty{}) + if err != nil { + st, _ := status.FromError(err) + + return protobuf.LivenessProbeResponse_UNKNOWN.String(), errors.New(st.Message()) + } + + return resp.State.String(), nil +} + +func (c *Client) ReadinessProbe(opts ...grpc.CallOption) (string, error) { + resp, err := c.client.ReadinessProbe(c.ctx, &empty.Empty{}) + if err != nil { + st, _ := status.FromError(err) + + return protobuf.ReadinessProbeResponse_UNKNOWN.String(), errors.New(st.Message()) + } + + return resp.State.String(), nil +} + +func (c *Client) GetState(key string, opts ...grpc.CallOption) (interface{}, error) { + req := &protobuf.GetStateRequest{ + Key: key, + } + + resp, err := c.client.GetState(c.ctx, req, opts...) + if err != nil { + st, _ := status.FromError(err) + + switch st.Code() { + case codes.NotFound: + return nil, blasterrors.ErrNotFound + default: + return nil, errors.New(st.Message()) + } + } + + value, err := protobuf.MarshalAny(resp.Value) + + return value, nil +} + +func (c *Client) SetState(key string, value interface{}, opts ...grpc.CallOption) error { + valueAny := &any.Any{} + err := protobuf.UnmarshalAny(value, valueAny) + if err != nil { + return err + } + + req := &protobuf.SetStateRequest{ + Key: key, + Value: valueAny, + } + + _, err = c.client.SetState(c.ctx, req, opts...) + if err != nil { + st, _ := status.FromError(err) + + switch st.Code() { + case codes.NotFound: + return blasterrors.ErrNotFound + default: + return errors.New(st.Message()) + } + } + + return nil +} + +func (c *Client) DeleteState(key string, opts ...grpc.CallOption) error { + req := &protobuf.DeleteStateRequest{ + Key: key, + } + + _, err := c.client.DeleteState(c.ctx, req, opts...) + if err != nil { + st, _ := status.FromError(err) + + switch st.Code() { + case codes.NotFound: + return blasterrors.ErrNotFound + default: + return errors.New(st.Message()) + } + } + + return nil +} + +func (c *Client) WatchState(key string, opts ...grpc.CallOption) (protobuf.Blast_WatchStateClient, error) { + req := &protobuf.WatchStateRequest{ + Key: key, + } + + watchClient, err := c.client.WatchState(c.ctx, req, opts...) + if err != nil { + st, _ := status.FromError(err) + return nil, errors.New(st.Message()) + } + + return watchClient, nil +} + +func (c *Client) GetDocument(id string, opts ...grpc.CallOption) (map[string]interface{}, error) { + req := &protobuf.GetDocumentRequest{ + Id: id, + } + + resp, err := c.client.GetDocument(c.ctx, req, opts...) + if err != nil { + st, _ := status.FromError(err) + + switch st.Code() { + case codes.NotFound: + return nil, blasterrors.ErrNotFound + default: + return nil, errors.New(st.Message()) + } + } + + ins, err := protobuf.MarshalAny(resp.Fields) + fields := *ins.(*map[string]interface{}) + + return fields, nil +} + +func (c *Client) Search(searchRequest *bleve.SearchRequest, opts ...grpc.CallOption) (*bleve.SearchResult, error) { + // bleve.SearchRequest -> Any + searchRequestAny := &any.Any{} + err := protobuf.UnmarshalAny(searchRequest, searchRequestAny) + if err != nil { + return nil, err + } + + req := &protobuf.SearchRequest{ + SearchRequest: searchRequestAny, + } + + resp, err := c.client.Search(c.ctx, req, opts...) + if err != nil { + st, _ := status.FromError(err) + + return nil, errors.New(st.Message()) + } + + // Any -> bleve.SearchResult + searchResultInstance, err := protobuf.MarshalAny(resp.SearchResult) + if err != nil { + st, _ := status.FromError(err) + + return nil, errors.New(st.Message()) + } + if searchResultInstance == nil { + return nil, errors.New("nil") + } + searchResult := searchResultInstance.(*bleve.SearchResult) + + return searchResult, nil +} + +func (c *Client) IndexDocument(docs []map[string]interface{}, opts ...grpc.CallOption) (int, error) { + stream, err := c.client.IndexDocument(c.ctx, opts...) + if err != nil { + st, _ := status.FromError(err) + + return -1, errors.New(st.Message()) + } + + for _, doc := range docs { + id := doc["id"].(string) + fields := doc["fields"].(map[string]interface{}) + + fieldsAny := &any.Any{} + err := protobuf.UnmarshalAny(&fields, fieldsAny) + if err != nil { + return -1, err + } + + req := &protobuf.IndexDocumentRequest{ + Id: id, + Fields: fieldsAny, + } + + err = stream.Send(req) + if err != nil { + return -1, err + } + } + + resp, err := stream.CloseAndRecv() + if err != nil { + return -1, err + } + + return int(resp.Count), nil +} + +func (c *Client) DeleteDocument(ids []string, opts ...grpc.CallOption) (int, error) { + stream, err := c.client.DeleteDocument(c.ctx, opts...) + if err != nil { + st, _ := status.FromError(err) + + return -1, errors.New(st.Message()) + } + + for _, id := range ids { + req := &protobuf.DeleteDocumentRequest{ + Id: id, + } + + err := stream.Send(req) + if err != nil { + return -1, err + } + } + + resp, err := stream.CloseAndRecv() + if err != nil { + return -1, err + } + + return int(resp.Count), nil +} + +func (c *Client) GetIndexConfig(opts ...grpc.CallOption) (*protobuf.GetIndexConfigResponse, error) { + conf, err := c.client.GetIndexConfig(c.ctx, &empty.Empty{}, opts...) + if err != nil { + st, _ := status.FromError(err) + + return nil, errors.New(st.Message()) + } + + return conf, nil +} + +func (c *Client) GetIndexStats(opts ...grpc.CallOption) (*protobuf.GetIndexStatsResponse, error) { + stats, err := c.client.GetIndexStats(c.ctx, &empty.Empty{}, opts...) + if err != nil { + st, _ := status.FromError(err) + + return nil, errors.New(st.Message()) + } + + return stats, nil +} diff --git a/indexer/grpc_server.go b/grpc/server.go similarity index 68% rename from indexer/grpc_server.go rename to grpc/server.go index 0f9fa3a..9dbee30 100644 --- a/indexer/grpc_server.go +++ b/grpc/server.go @@ -12,41 +12,44 @@ // See the License for the specific language governing permissions and // limitations under the License. -package indexer +package grpc import ( "log" "net" - "github.com/mosuka/blast/protobuf/index" + "github.com/mosuka/blast/protobuf" "google.golang.org/grpc" ) -type GRPCServer struct { +type Server struct { + service protobuf.BlastServer server *grpc.Server listener net.Listener logger *log.Logger } -func NewGRPCServer(grpcAddr string, service *GRPCService, logger *log.Logger) (*GRPCServer, error) { +func NewServer(grpcAddr string, service protobuf.BlastServer, logger *log.Logger) (*Server, error) { server := grpc.NewServer() - index.RegisterIndexServer(server, service) + protobuf.RegisterBlastServer(server, service) listener, err := net.Listen("tcp", grpcAddr) if err != nil { return nil, err } - return &GRPCServer{ + return &Server{ + service: service, server: server, listener: listener, logger: logger, }, nil } -func (s *GRPCServer) Start() error { +func (s *Server) Start() error { + s.logger.Print("[INFO] start server") err := s.server.Serve(s.listener) if err != nil { return err @@ -55,8 +58,9 @@ func (s *GRPCServer) Start() error { return nil } -func (s *GRPCServer) Stop() error { - s.server.GracefulStop() +func (s *Server) Stop() error { + s.logger.Print("[INFO] stop server") + s.server.Stop() // TODO: graceful stop return nil } diff --git a/grpc/service.go b/grpc/service.go new file mode 100644 index 0000000..2f9c770 --- /dev/null +++ b/grpc/service.go @@ -0,0 +1,122 @@ +// Copyright (c) 2019 Minoru Osuka +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package grpc + +import ( + "context" + + "github.com/golang/protobuf/ptypes/empty" + "github.com/mosuka/blast/protobuf" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type Service struct{} + +func (s *Service) Start() error { + return nil +} + +func (s *Service) Stop() error { + return nil +} + +func (s *Service) LivenessProbe(ctx context.Context, req *empty.Empty) (*protobuf.LivenessProbeResponse, error) { + resp := &protobuf.LivenessProbeResponse{ + State: protobuf.LivenessProbeResponse_ALIVE, + } + + return resp, nil +} + +func (s *Service) ReadinessProbe(ctx context.Context, req *empty.Empty) (*protobuf.ReadinessProbeResponse, error) { + resp := &protobuf.ReadinessProbeResponse{ + State: protobuf.ReadinessProbeResponse_READY, + } + + return resp, nil +} + +func (s *Service) GetMetadata(ctx context.Context, req *protobuf.GetMetadataRequest) (*protobuf.GetMetadataResponse, error) { + return &protobuf.GetMetadataResponse{}, status.Error(codes.Unavailable, "not implement") +} + +func (s *Service) GetNodeState(ctx context.Context, req *protobuf.GetNodeStateRequest) (*protobuf.GetNodeStateResponse, error) { + return &protobuf.GetNodeStateResponse{}, status.Error(codes.Unavailable, "not implement") +} + +func (s *Service) GetNode(ctx context.Context, req *protobuf.GetNodeRequest) (*protobuf.GetNodeResponse, error) { + return &protobuf.GetNodeResponse{}, status.Error(codes.Unavailable, "not implement") +} + +func (s *Service) SetNode(ctx context.Context, req *protobuf.SetNodeRequest) (*empty.Empty, error) { + return &empty.Empty{}, status.Error(codes.Unavailable, "not implement") +} + +func (s *Service) DeleteNode(ctx context.Context, req *protobuf.DeleteNodeRequest) (*empty.Empty, error) { + return &empty.Empty{}, status.Error(codes.Unavailable, "not implement") +} + +func (s *Service) GetCluster(ctx context.Context, req *empty.Empty) (*protobuf.GetClusterResponse, error) { + return &protobuf.GetClusterResponse{}, status.Error(codes.Unavailable, "not implement") +} + +func (s *Service) WatchCluster(req *empty.Empty, server protobuf.Blast_WatchClusterServer) error { + return status.Error(codes.Unavailable, "not implement") +} + +func (s *Service) Snapshot(ctx context.Context, req *empty.Empty) (*empty.Empty, error) { + return &empty.Empty{}, status.Error(codes.Unavailable, "not implement") +} + +func (s *Service) GetState(ctx context.Context, req *protobuf.GetStateRequest) (*protobuf.GetStateResponse, error) { + return &protobuf.GetStateResponse{}, status.Error(codes.Unavailable, "not implement") +} + +func (s *Service) SetState(ctx context.Context, req *protobuf.SetStateRequest) (*empty.Empty, error) { + return &empty.Empty{}, status.Error(codes.Unavailable, "not implement") +} + +func (s *Service) DeleteState(ctx context.Context, req *protobuf.DeleteStateRequest) (*empty.Empty, error) { + return &empty.Empty{}, status.Error(codes.Unavailable, "not implement") +} + +func (s *Service) WatchState(req *protobuf.WatchStateRequest, server protobuf.Blast_WatchStateServer) error { + return status.Error(codes.Unavailable, "not implement") +} + +func (s *Service) GetDocument(ctx context.Context, req *protobuf.GetDocumentRequest) (*protobuf.GetDocumentResponse, error) { + return &protobuf.GetDocumentResponse{}, status.Error(codes.Unavailable, "not implement") +} + +func (s *Service) Search(ctx context.Context, req *protobuf.SearchRequest) (*protobuf.SearchResponse, error) { + return &protobuf.SearchResponse{}, status.Error(codes.Unavailable, "not implement") +} + +func (s *Service) IndexDocument(stream protobuf.Blast_IndexDocumentServer) error { + return status.Error(codes.Unavailable, "not implement") +} + +func (s *Service) DeleteDocument(stream protobuf.Blast_DeleteDocumentServer) error { + return status.Error(codes.Unavailable, "not implement") +} + +func (s *Service) GetIndexConfig(ctx context.Context, req *empty.Empty) (*protobuf.GetIndexConfigResponse, error) { + return &protobuf.GetIndexConfigResponse{}, status.Error(codes.Unavailable, "not implement") +} + +func (s *Service) GetIndexStats(ctx context.Context, req *empty.Empty) (*protobuf.GetIndexStatsResponse, error) { + return &protobuf.GetIndexStatsResponse{}, status.Error(codes.Unavailable, "not implement") +} diff --git a/http/router.go b/http/router.go new file mode 100644 index 0000000..b2ea7fb --- /dev/null +++ b/http/router.go @@ -0,0 +1,52 @@ +// Copyright (c) 2019 Minoru Osuka +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "log" + + "github.com/gorilla/mux" + "github.com/mosuka/blast/grpc" +) + +type Router struct { + mux.Router + + GRPCClient *grpc.Client + logger *log.Logger +} + +func NewRouter(grpcAddr string, logger *log.Logger) (*Router, error) { + grpcClient, err := grpc.NewClient(grpcAddr) + if err != nil { + return nil, err + } + + return &Router{ + GRPCClient: grpcClient, + logger: logger, + }, nil +} + +func (r *Router) Close() error { + r.GRPCClient.Cancel() + + err := r.GRPCClient.Close() + if err != nil { + return err + } + + return nil +} diff --git a/manager/http_server.go b/http/server.go similarity index 57% rename from manager/http_server.go rename to http/server.go index c66134e..4e531c2 100644 --- a/manager/http_server.go +++ b/http/server.go @@ -12,53 +12,39 @@ // See the License for the specific language governing permissions and // limitations under the License. -package manager +package http import ( "log" "net" "net/http" - "github.com/gorilla/mux" accesslog "github.com/mash/go-accesslog" - "github.com/prometheus/client_golang/prometheus/promhttp" ) -type HTTPServer struct { +type Server struct { listener net.Listener - router *mux.Router - - grpcClient *GRPCClient + router *Router logger *log.Logger httpLogger accesslog.Logger } -func NewHTTPServer(httpAddr string, grpcClient *GRPCClient, logger *log.Logger, httpLogger accesslog.Logger) (*HTTPServer, error) { +func NewServer(httpAddr string, router *Router, logger *log.Logger, httpLogger accesslog.Logger) (*Server, error) { listener, err := net.Listen("tcp", httpAddr) if err != nil { return nil, err } - router := mux.NewRouter() - router.StrictSlash(true) - - router.Handle("/", NewRootHandler(logger)).Methods("GET") - router.Handle("/configs/{path:.*}", NewPutHandler(grpcClient, logger)).Methods("PUT") - router.Handle("/configs/{path:.*}", NewGetHandler(grpcClient, logger)).Methods("GET") - router.Handle("/configs/{path:.*}", NewDeleteHandler(grpcClient, logger)).Methods("DELETE") - router.Handle("/metrics", promhttp.Handler()).Methods("GET") - - return &HTTPServer{ + return &Server{ listener: listener, router: router, - grpcClient: grpcClient, logger: logger, httpLogger: httpLogger, }, nil } -func (s *HTTPServer) Start() error { +func (s *Server) Start() error { err := http.Serve( s.listener, accesslog.NewLoggingHandler( @@ -73,7 +59,7 @@ func (s *HTTPServer) Start() error { return nil } -func (s *HTTPServer) Stop() error { +func (s *Server) Stop() error { err := s.listener.Close() if err != nil { return err diff --git a/indexer/grpc_client.go b/indexer/grpc_client.go deleted file mode 100644 index 1112073..0000000 --- a/indexer/grpc_client.go +++ /dev/null @@ -1,241 +0,0 @@ -// Copyright (c) 2019 Minoru Osuka -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package indexer - -import ( - "context" - "errors" - "log" - "math" - - "github.com/blevesearch/bleve" - "github.com/golang/protobuf/ptypes/any" - "github.com/golang/protobuf/ptypes/empty" - blasterrors "github.com/mosuka/blast/errors" - "github.com/mosuka/blast/protobuf" - "github.com/mosuka/blast/protobuf/index" - "github.com/mosuka/blast/protobuf/raft" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -type GRPCClient struct { - ctx context.Context - cancel context.CancelFunc - conn *grpc.ClientConn - client index.IndexClient - - logger *log.Logger -} - -func NewGRPCClient(grpcAddr string) (*GRPCClient, error) { - baseCtx := context.TODO() - ctx, cancel := context.WithCancel(baseCtx) - - dialOpts := []grpc.DialOption{ - grpc.WithInsecure(), - grpc.WithDefaultCallOptions( - grpc.MaxCallSendMsgSize(math.MaxInt32), - grpc.MaxCallRecvMsgSize(math.MaxInt32), - ), - } - - conn, err := grpc.DialContext(ctx, grpcAddr, dialOpts...) - if err != nil { - cancel() - return nil, err - } - - return &GRPCClient{ - ctx: ctx, - cancel: cancel, - conn: conn, - client: index.NewIndexClient(conn), - }, nil -} - -func (c *GRPCClient) Close() error { - c.cancel() - if c.conn != nil { - return c.conn.Close() - } - - return c.ctx.Err() -} - -func (c *GRPCClient) Join(node *raft.Node, opts ...grpc.CallOption) error { - _, err := c.client.Join(c.ctx, node, opts...) - if err != nil { - st, _ := status.FromError(err) - - return errors.New(st.Message()) - } - - return nil -} - -func (c *GRPCClient) Leave(node *raft.Node, opts ...grpc.CallOption) error { - _, err := c.client.Leave(c.ctx, node, opts...) - if err != nil { - st, _ := status.FromError(err) - - return errors.New(st.Message()) - } - - return nil -} - -func (c *GRPCClient) GetNode(opts ...grpc.CallOption) (*raft.Node, error) { - node, err := c.client.GetNode(c.ctx, &empty.Empty{}, opts...) - if err != nil { - st, _ := status.FromError(err) - - return nil, errors.New(st.Message()) - } - - return node, nil -} - -func (c *GRPCClient) GetCluster(opts ...grpc.CallOption) (*raft.Cluster, error) { - cluster, err := c.client.GetCluster(c.ctx, &empty.Empty{}, opts...) - if err != nil { - st, _ := status.FromError(err) - - return nil, errors.New(st.Message()) - } - - return cluster, nil -} - -func (c *GRPCClient) Snapshot(opts ...grpc.CallOption) error { - _, err := c.client.Snapshot(c.ctx, &empty.Empty{}) - if err != nil { - st, _ := status.FromError(err) - - return errors.New(st.Message()) - } - - return nil -} - -func (c *GRPCClient) Get(doc *index.Document, opts ...grpc.CallOption) (*index.Document, error) { - retDoc, err := c.client.Get(c.ctx, doc, opts...) - if err != nil { - st, _ := status.FromError(err) - - switch st.Code() { - case codes.NotFound: - return nil, blasterrors.ErrNotFound - default: - return nil, errors.New(st.Message()) - } - } - - return retDoc, nil -} - -func (c *GRPCClient) Search(searchRequest *bleve.SearchRequest, opts ...grpc.CallOption) (*bleve.SearchResult, error) { - // bleve.SearchRequest -> Any - searchRequestAny := &any.Any{} - err := protobuf.UnmarshalAny(searchRequest, searchRequestAny) - if err != nil { - return nil, err - } - - req := &index.SearchRequest{ - SearchRequest: searchRequestAny, - } - - resp, err := c.client.Search(c.ctx, req, opts...) - if err != nil { - st, _ := status.FromError(err) - - return nil, errors.New(st.Message()) - } - - // Any -> bleve.SearchResult - searchResultInstance, err := protobuf.MarshalAny(resp.SearchResult) - if err != nil { - st, _ := status.FromError(err) - - return nil, errors.New(st.Message()) - } - if searchResultInstance == nil { - return nil, errors.New("nil") - } - searchResult := searchResultInstance.(*bleve.SearchResult) - - return searchResult, nil -} - -func (c *GRPCClient) Index(docs []*index.Document, opts ...grpc.CallOption) (*index.UpdateResult, error) { - stream, err := c.client.Index(c.ctx, opts...) - if err != nil { - st, _ := status.FromError(err) - - return nil, errors.New(st.Message()) - } - - for _, doc := range docs { - err := stream.Send(doc) - if err != nil { - c.logger.Printf("[WARN]: %v", err) - break - } - } - - rep, err := stream.CloseAndRecv() - if err != nil { - return nil, err - } - - return rep, nil -} - -func (c *GRPCClient) Delete(docs []*index.Document, opts ...grpc.CallOption) (*index.UpdateResult, error) { - stream, err := c.client.Delete(c.ctx, opts...) - if err != nil { - st, _ := status.FromError(err) - - return nil, errors.New(st.Message()) - } - - for _, doc := range docs { - err := stream.Send(doc) - if err != nil { - c.logger.Printf("[WARN]: %v", err) - break - } - } - - rep, err := stream.CloseAndRecv() - if err != nil { - return nil, err - } - - return rep, nil -} - -func (c *GRPCClient) GetIndexStats(opts ...grpc.CallOption) (*index.Stats, error) { - stats, err := c.client.GetStats(c.ctx, &empty.Empty{}, opts...) - if err != nil { - st, _ := status.FromError(err) - - return nil, errors.New(st.Message()) - } - - return stats, nil -} diff --git a/indexer/grpc_service.go b/indexer/grpc_service.go index e59dc06..94263f5 100644 --- a/indexer/grpc_service.go +++ b/indexer/grpc_service.go @@ -16,90 +16,704 @@ package indexer import ( "context" + "errors" + "fmt" "io" "log" + "reflect" + "sync" "time" "github.com/blevesearch/bleve" "github.com/golang/protobuf/ptypes/any" "github.com/golang/protobuf/ptypes/empty" - "github.com/mosuka/blast/errors" + "github.com/hashicorp/raft" + blasterrors "github.com/mosuka/blast/errors" + "github.com/mosuka/blast/grpc" "github.com/mosuka/blast/protobuf" - "github.com/mosuka/blast/protobuf/index" - "github.com/mosuka/blast/protobuf/raft" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) type GRPCService struct { - raftServer *RaftServer + *grpc.Service + + managerAddr string + clusterId string - logger *log.Logger + raftServer *RaftServer + logger *log.Logger + + watchClusterStopCh chan struct{} + watchClusterDoneCh chan struct{} + peers map[string]interface{} + peerClients map[string]*grpc.Client + cluster map[string]interface{} + clusterChans map[chan protobuf.GetClusterResponse]struct{} + clusterMutex sync.RWMutex + + managers map[string]interface{} + managerClients map[string]*grpc.Client + watchManagersStopCh chan struct{} + watchManagersDoneCh chan struct{} } -func NewGRPCService(store *RaftServer, logger *log.Logger) (*GRPCService, error) { +func NewGRPCService(managerAddr string, clusterId string, raftServer *RaftServer, logger *log.Logger) (*GRPCService, error) { return &GRPCService{ - raftServer: store, + managerAddr: managerAddr, + clusterId: clusterId, + + raftServer: raftServer, logger: logger, + + peers: make(map[string]interface{}, 0), + peerClients: make(map[string]*grpc.Client, 0), + cluster: make(map[string]interface{}, 0), + clusterChans: make(map[chan protobuf.GetClusterResponse]struct{}), + + managers: make(map[string]interface{}, 0), + managerClients: make(map[string]*grpc.Client, 0), }, nil } -func (s *GRPCService) Join(ctx context.Context, req *raft.Node) (*empty.Empty, error) { - s.logger.Printf("[INFO] join %v", req) +func (s *GRPCService) Start() error { + s.logger.Print("[INFO] start watching cluster") + go s.startUpdateCluster(500 * time.Millisecond) - resp := &empty.Empty{} + if s.managerAddr != "" { + s.logger.Print("[INFO] start watching managers") + go s.startUpdateManagers(500 * time.Millisecond) + } + + return nil +} + +func (s *GRPCService) Stop() error { + s.logger.Print("[INFO] stop watching cluster") + s.stopUpdateCluster() + + if s.managerAddr != "" { + s.logger.Print("[INFO] stop watching managers") + s.stopUpdateManagers() + } + + return nil +} + +func (s *GRPCService) getManagerClient() (*grpc.Client, error) { + var client *grpc.Client - err := s.raftServer.Join(req) + for id, node := range s.managers { + state := node.(map[string]interface{})["state"].(string) + if state != raft.Shutdown.String() { + + if _, exist := s.managerClients[id]; exist { + client = s.managerClients[id] + break + } else { + s.logger.Printf("[DEBUG] %v does not exist", id) + } + } + } + + if client == nil { + return nil, errors.New("client does not exist") + } + + return client, nil +} + +func (s *GRPCService) getInitialManagers(managerAddr string) (map[string]interface{}, error) { + client, err := grpc.NewClient(s.managerAddr) + defer func() { + err := client.Close() + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + return + }() if err != nil { - return resp, status.Error(codes.Internal, err.Error()) + s.logger.Printf("[ERR] %v", err) + return nil, err } - return resp, nil + managers, err := client.GetCluster() + if err != nil { + s.logger.Printf("[ERR] %v", err) + return nil, err + } + + return managers, nil } -func (s *GRPCService) Leave(ctx context.Context, req *raft.Node) (*empty.Empty, error) { - s.logger.Printf("[INFO] leave %v", req) +func (s *GRPCService) startUpdateManagers(checkInterval time.Duration) { + s.logger.Printf("[INFO] start watching a cluster") - resp := &empty.Empty{} + s.watchManagersStopCh = make(chan struct{}) + s.watchManagersDoneCh = make(chan struct{}) + + defer func() { + close(s.watchManagersDoneCh) + }() + + var err error + + // get initial managers + s.managers, err = s.getInitialManagers(s.managerAddr) + if err != nil { + s.logger.Printf("[ERR] %v", err) + return + } + s.logger.Printf("[DEBUG] %v", s.managers) + + // create clients for managers + for nodeId, node := range s.managers { + metadata := node.(map[string]interface{})["metadata"].(map[string]interface{}) + + s.logger.Printf("[DEBUG] create client for %s", metadata["grpc_addr"].(string)) + + client, err := grpc.NewClient(metadata["grpc_addr"].(string)) + if err != nil { + s.logger.Printf("[ERR] %v", err) + continue + } + s.managerClients[nodeId] = client + } + + for { + select { + case <-s.watchManagersStopCh: + s.logger.Print("[DEBUG] receive request that stop watching a cluster") + return + default: + // get active client for manager + client, err := s.getManagerClient() + if err != nil { + s.logger.Printf("[ERR] %v", err) + continue + } + + // create stream + stream, err := client.WatchCluster() + if err != nil { + st, _ := status.FromError(err) + switch st.Code() { + case codes.Canceled: + s.logger.Printf("[DEBUG] %v", err) + default: + s.logger.Printf("[ERR] %v", err) + } + continue + } + + // wait for receive cluster updates from stream + s.logger.Print("[DEBUG] wait for receive cluster updates from stream") + resp, err := stream.Recv() + if err == io.EOF { + continue + } + if err != nil { + st, _ := status.FromError(err) + switch st.Code() { + case codes.Canceled: + s.logger.Printf("[DEBUG] %v", err) + default: + s.logger.Printf("[ERR] %v", err) + } + continue + } + + // get current manager cluster + cluster, err := protobuf.MarshalAny(resp.Cluster) + if err != nil { + s.logger.Printf("[ERR] %v", err) + continue + } + if cluster == nil { + s.logger.Print("[ERR] nil") + continue + } + managers := *cluster.(*map[string]interface{}) + + // compare previous manager with current manager + if !reflect.DeepEqual(s.managers, managers) { + s.logger.Printf("[INFO] %v", managers) + + // close the client for left manager node + for id := range s.managers { + if _, managerExists := managers[id]; !managerExists { + if _, clientExists := s.managerClients[id]; clientExists { + client := s.managerClients[id] + + s.logger.Printf("[DEBUG] close client for %s", client.GetAddress()) + err = client.Close() + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + + delete(s.managerClients, id) + } + } + } + + // keep current manager cluster + s.managers = managers + } + } + } +} + +func (s *GRPCService) stopUpdateManagers() { + // close clients + s.logger.Printf("[INFO] close manager clients") + for _, client := range s.managerClients { + s.logger.Printf("[DEBUG] close manager client for %s", client.GetAddress()) + err := client.Close() + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + } + + // stop watching managers + if s.watchManagersStopCh != nil { + s.logger.Printf("[INFO] stop watching managers") + close(s.watchManagersStopCh) + } + + // wait for stop watching managers has done + s.logger.Printf("[INFO] wait for stop watching managers has done") + <-s.watchManagersDoneCh +} + +func (s *GRPCService) getLeaderClient() (*grpc.Client, error) { + var client *grpc.Client + + for id, node := range s.cluster { + state := node.(map[string]interface{})["state"].(string) + if state != raft.Shutdown.String() { - err := s.raftServer.Leave(req) + if _, exist := s.peerClients[id]; exist { + client = s.peerClients[id] + break + } else { + s.logger.Printf("[DEBUG] %v does not exist", id) + } + } + } + + if client == nil { + return nil, errors.New("client does not exist") + } + + return client, nil +} + +func (s *GRPCService) startUpdateCluster(checkInterval time.Duration) { + s.watchClusterStopCh = make(chan struct{}) + s.watchClusterDoneCh = make(chan struct{}) + + s.logger.Printf("[INFO] start watching a cluster") + + defer func() { + close(s.watchClusterDoneCh) + }() + + ticker := time.NewTicker(checkInterval) + defer ticker.Stop() + + for { + select { + case <-s.watchClusterStopCh: + s.logger.Print("[DEBUG] receive request that stop watching a cluster") + return + case <-ticker.C: + // get servers + servers, err := s.raftServer.GetServers() + if err != nil { + s.logger.Printf("[ERR] %v", err) + return + } + + // create peer node list with out self node + peers := make(map[string]interface{}, 0) + for id, metadata := range servers { + if id != s.raftServer.id { + peers[id] = metadata + } + } + + // create and close clients for manager + if !reflect.DeepEqual(s.peers, peers) { + // create clients + for id, metadata := range peers { + grpcAddr := metadata.(map[string]interface{})["grpc_addr"].(string) + + if _, clientExists := s.peerClients[id]; clientExists { + client := s.peerClients[id] + if client.GetAddress() != grpcAddr { + s.logger.Printf("[DEBUG] close client for %s", client.GetAddress()) + err = client.Close() + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + + s.logger.Printf("[DEBUG] create client for %s", grpcAddr) + newClient, err := grpc.NewClient(grpcAddr) + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + + if client != nil { + s.logger.Printf("[DEBUG] create client for %s", newClient.GetAddress()) + s.peerClients[id] = newClient + } + } + } else { + s.logger.Printf("[DEBUG] create client for %s", grpcAddr) + newClient, err := grpc.NewClient(grpcAddr) + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + + s.peerClients[id] = newClient + } + } + + // close nonexistent clients + for id := range s.peers { + if _, peerExists := peers[id]; !peerExists { + if _, clientExists := s.peerClients[id]; clientExists { + client := s.peerClients[id] + + s.logger.Printf("[DEBUG] close client for %s", client.GetAddress()) + err = client.Close() + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + + delete(s.peerClients, id) + } + } + } + + // keep current peer nodes + s.peers = peers + } + + // get cluster + ctx, _ := grpc.NewContext() + resp, err := s.GetCluster(ctx, &empty.Empty{}) + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + ins, err := protobuf.MarshalAny(resp.Cluster) + cluster := *ins.(*map[string]interface{}) + + if !reflect.DeepEqual(s.cluster, cluster) { + // notify cluster state + for c := range s.clusterChans { + c <- *resp + } + + // update cluster to manager + if s.raftServer.IsLeader() && s.managerAddr != "" { + // get active client for manager + client, err := s.getManagerClient() + if err != nil { + s.logger.Printf("[ERR] %v", err) + continue + } + s.logger.Printf("[DEBUG] update cluster state: %s %v", s.clusterId, cluster) + err = client.SetState(fmt.Sprintf("/cluster_config/clusters/%s/nodes", s.clusterId), cluster) + if err != nil { + continue + } + } + + // keep current cluster + s.cluster = cluster + } + default: + time.Sleep(100 * time.Millisecond) + } + } +} + +func (s *GRPCService) stopUpdateCluster() { + // close clients + s.logger.Printf("[INFO] close peer clients") + for _, client := range s.peerClients { + s.logger.Printf("[DEBUG] close peer client for %s", client.GetAddress()) + err := client.Close() + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + } + + // stop watching peers + if s.watchClusterStopCh != nil { + s.logger.Printf("[INFO] stop watching peers") + close(s.watchClusterStopCh) + } + + // wait for stop watching peers has done + s.logger.Printf("[INFO] wait for stop watching peers has done") + <-s.watchClusterDoneCh +} + +func (s *GRPCService) getMetadata() (map[string]interface{}, error) { + metadata, err := s.raftServer.GetMetadata(s.raftServer.id) + if err != nil { + return nil, err + } + + return metadata, nil +} + +func (s *GRPCService) GetMetadata(ctx context.Context, req *protobuf.GetMetadataRequest) (*protobuf.GetMetadataResponse, error) { + resp := &protobuf.GetMetadataResponse{} + + var metadata map[string]interface{} + var err error + + if req.Id == "" || req.Id == s.raftServer.id { + metadata, err = s.getMetadata() + if err != nil { + s.logger.Printf("[ERR] %v: %v", err, s.raftServer.id) + } + } else { + if client, exist := s.peerClients[req.Id]; exist { + metadata, err = client.GetNodeMetadata(req.Id) + if err != nil { + s.logger.Printf("[ERR] %v: %v", err, req.Id) + } + } + } + + metadataAny := &any.Any{} + err = protobuf.UnmarshalAny(metadata, metadataAny) if err != nil { return resp, status.Error(codes.Internal, err.Error()) } + resp.Metadata = metadataAny + return resp, nil } -func (s *GRPCService) GetNode(ctx context.Context, req *empty.Empty) (*raft.Node, error) { - s.logger.Printf("[INFO] get node %v", req) +func (s *GRPCService) getNodeState() string { + return s.raftServer.State() +} - resp := &raft.Node{} +func (s *GRPCService) GetNodeState(ctx context.Context, req *protobuf.GetNodeStateRequest) (*protobuf.GetNodeStateResponse, error) { + resp := &protobuf.GetNodeStateResponse{} + state := "" var err error - resp, err = s.raftServer.GetNode() + if req.Id == s.raftServer.id { + state = s.getNodeState() + } else { + if client, exist := s.peerClients[req.Id]; exist { + state, err = client.GetNodeState(req.Id) + if err != nil { + return resp, status.Error(codes.Internal, err.Error()) + } + } + } + + resp.State = state + + return resp, nil +} + +func (s *GRPCService) getNode() (map[string]interface{}, error) { + metadata, err := s.getMetadata() + if err != nil { + return nil, err + } + + state := s.getNodeState() + + node := map[string]interface{}{ + "metadata": metadata, + "state": state, + } + + return node, nil +} + +func (s *GRPCService) GetNode(ctx context.Context, req *protobuf.GetNodeRequest) (*protobuf.GetNodeResponse, error) { + resp := &protobuf.GetNodeResponse{} + + var metadata map[string]interface{} + var state string + var err error + + if req.Id == "" || req.Id == s.raftServer.id { + node, err := s.getNode() + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + + metadata = node["metadata"].(map[string]interface{}) + + state = node["state"].(string) + } else { + if client, exist := s.peerClients[req.Id]; exist { + metadata, err = client.GetNodeMetadata(req.Id) + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + + state, err = client.GetNodeState(req.Id) + if err != nil { + s.logger.Printf("[ERR] %v", err) + state = raft.Shutdown.String() + } + } + } + + metadataAny := &any.Any{} + err = protobuf.UnmarshalAny(metadata, metadataAny) if err != nil { return resp, status.Error(codes.Internal, err.Error()) } + resp.Metadata = metadataAny + resp.State = state + return resp, nil } -func (s *GRPCService) GetCluster(ctx context.Context, req *empty.Empty) (*raft.Cluster, error) { - s.logger.Printf("[INFO] get cluster %v", req) +func (s *GRPCService) SetNode(ctx context.Context, req *protobuf.SetNodeRequest) (*empty.Empty, error) { + s.logger.Printf("[INFO] %v", req) - resp := &raft.Cluster{} + resp := &empty.Empty{} - var err error + ins, err := protobuf.MarshalAny(req.Metadata) + if err != nil { + return resp, status.Error(codes.Internal, err.Error()) + } - resp, err = s.raftServer.GetCluster() + metadata := *ins.(*map[string]interface{}) + + if s.raftServer.IsLeader() { + err = s.raftServer.SetMetadata(req.Id, metadata) + if err != nil { + return resp, status.Error(codes.Internal, err.Error()) + } + } else { + // forward to leader + client, err := s.getLeaderClient() + if err != nil { + return resp, status.Error(codes.Internal, err.Error()) + } + err = client.SetNode(req.Id, metadata) + if err != nil { + return resp, status.Error(codes.Internal, err.Error()) + } + } + + return resp, nil +} + +func (s *GRPCService) DeleteNode(ctx context.Context, req *protobuf.DeleteNodeRequest) (*empty.Empty, error) { + s.logger.Printf("[INFO] leave %v", req) + + resp := &empty.Empty{} + + if s.raftServer.IsLeader() { + err := s.raftServer.DeleteMetadata(req.Id) + if err != nil { + return resp, status.Error(codes.Internal, err.Error()) + } + } else { + // forward to leader + client, err := s.getLeaderClient() + if err != nil { + return resp, status.Error(codes.Internal, err.Error()) + } + err = client.DeleteNode(req.Id) + if err != nil { + return resp, status.Error(codes.Internal, err.Error()) + } + } + + return resp, nil +} + +func (s *GRPCService) GetCluster(ctx context.Context, req *empty.Empty) (*protobuf.GetClusterResponse, error) { + resp := &protobuf.GetClusterResponse{} + + servers, err := s.raftServer.GetServers() + if err != nil { + return resp, status.Error(codes.Internal, err.Error()) + } + + cluster := map[string]interface{}{} + + for id := range servers { + node := map[string]interface{}{} + if id == s.raftServer.id { + //node, err = s.getNode(id) + node, err = s.getNode() + if err != nil { + return resp, err + } + } else { + r, err := s.GetNode(ctx, &protobuf.GetNodeRequest{Id: id}) + if err != nil { + return resp, err + } + + ins, err := protobuf.MarshalAny(r.Metadata) + metadata := *ins.(*map[string]interface{}) + + node = map[string]interface{}{ + "metadata": metadata, + "state": r.State, + } + } + + cluster[id] = node + } + + clusterAny := &any.Any{} + err = protobuf.UnmarshalAny(cluster, clusterAny) if err != nil { return resp, status.Error(codes.Internal, err.Error()) } + resp.Cluster = clusterAny + return resp, nil } +func (s *GRPCService) WatchCluster(req *empty.Empty, server protobuf.Blast_WatchClusterServer) error { + chans := make(chan protobuf.GetClusterResponse) + + s.clusterMutex.Lock() + s.clusterChans[chans] = struct{}{} + s.clusterMutex.Unlock() + + defer func() { + s.clusterMutex.Lock() + delete(s.clusterChans, chans) + s.clusterMutex.Unlock() + close(chans) + }() + + for resp := range chans { + err := server.Send(&resp) + if err != nil { + return status.Error(codes.Internal, err.Error()) + } + } + + return nil +} + func (s *GRPCService) Snapshot(ctx context.Context, req *empty.Empty) (*empty.Empty, error) { s.logger.Printf("[INFO] %v", req) @@ -113,36 +727,42 @@ func (s *GRPCService) Snapshot(ctx context.Context, req *empty.Empty) (*empty.Em return resp, nil } -func (s *GRPCService) Get(ctx context.Context, req *index.Document) (*index.Document, error) { +func (s *GRPCService) GetDocument(ctx context.Context, req *protobuf.GetDocumentRequest) (*protobuf.GetDocumentResponse, error) { start := time.Now() defer RecordMetrics(start, "get") s.logger.Printf("[INFO] get %v", req) - resp := &index.Document{} - - var err error + resp := &protobuf.GetDocumentResponse{} - resp, err = s.raftServer.Get(req) + fields, err := s.raftServer.GetDocument(req.Id) if err != nil { switch err { - case errors.ErrNotFound: + case blasterrors.ErrNotFound: return resp, status.Error(codes.NotFound, err.Error()) default: return resp, status.Error(codes.Internal, err.Error()) } } + fieldsAny := &any.Any{} + err = protobuf.UnmarshalAny(fields, fieldsAny) + if err != nil { + return resp, status.Error(codes.Internal, err.Error()) + } + + resp.Fields = fieldsAny + return resp, nil } -func (s *GRPCService) Search(ctx context.Context, req *index.SearchRequest) (*index.SearchResponse, error) { +func (s *GRPCService) Search(ctx context.Context, req *protobuf.SearchRequest) (*protobuf.SearchResponse, error) { start := time.Now() defer RecordMetrics(start, "search") s.logger.Printf("[INFO] search %v", req) - resp := &index.SearchResponse{} + resp := &protobuf.SearchResponse{} // Any -> bleve.SearchRequest searchRequest, err := protobuf.MarshalAny(req.SearchRequest) @@ -167,11 +787,11 @@ func (s *GRPCService) Search(ctx context.Context, req *index.SearchRequest) (*in return resp, nil } -func (s *GRPCService) Index(stream index.Index_IndexServer) error { - docs := make([]*index.Document, 0) +func (s *GRPCService) IndexDocument(stream protobuf.Blast_IndexDocumentServer) error { + docs := make([]map[string]interface{}, 0) for { - doc, err := stream.Recv() + req, err := stream.Recv() if err == io.EOF { break } @@ -179,23 +799,54 @@ func (s *GRPCService) Index(stream index.Index_IndexServer) error { return status.Error(codes.Internal, err.Error()) } + // fields + ins, err := protobuf.MarshalAny(req.Fields) + if err != nil { + return status.Error(codes.Internal, err.Error()) + } + fields := *ins.(*map[string]interface{}) + + // document + doc := map[string]interface{}{ + "id": req.Id, + "fields": fields, + } + docs = append(docs, doc) } // index - result, err := s.raftServer.Index(docs) - if err != nil { - return status.Error(codes.Internal, err.Error()) + count := -1 + var err error + if s.raftServer.IsLeader() { + count, err = s.raftServer.IndexDocument(docs) + if err != nil { + return status.Error(codes.Internal, err.Error()) + } + } else { + // forward to leader + client, err := s.getLeaderClient() + if err != nil { + return status.Error(codes.Internal, err.Error()) + } + count, err = client.IndexDocument(docs) + if err != nil { + return status.Error(codes.Internal, err.Error()) + } } - return stream.SendAndClose(result) + return stream.SendAndClose( + &protobuf.IndexDocumentResponse{ + Count: int32(count), + }, + ) } -func (s *GRPCService) Delete(stream index.Index_DeleteServer) error { - docs := make([]*index.Document, 0) +func (s *GRPCService) DeleteDocument(stream protobuf.Blast_DeleteDocumentServer) error { + ids := make([]string, 0) for { - doc, err := stream.Recv() + req, err := stream.Recv() if err == io.EOF { break } @@ -203,32 +854,84 @@ func (s *GRPCService) Delete(stream index.Index_DeleteServer) error { return status.Error(codes.Internal, err.Error()) } - docs = append(docs, doc) + ids = append(ids, req.Id) } // delete - result, err := s.raftServer.Delete(docs) + count := -1 + var err error + if s.raftServer.IsLeader() { + count, err = s.raftServer.DeleteDocument(ids) + if err != nil { + return status.Error(codes.Internal, err.Error()) + } + } else { + // forward to leader + client, err := s.getLeaderClient() + if err != nil { + return status.Error(codes.Internal, err.Error()) + } + count, err = client.DeleteDocument(ids) + if err != nil { + return status.Error(codes.Internal, err.Error()) + } + } + + return stream.SendAndClose( + &protobuf.DeleteDocumentResponse{ + Count: int32(count), + }, + ) +} + +func (s *GRPCService) GetIndexConfig(ctx context.Context, req *empty.Empty) (*protobuf.GetIndexConfigResponse, error) { + start := time.Now() + defer RecordMetrics(start, "indexconfig") + + resp := &protobuf.GetIndexConfigResponse{} + + s.logger.Printf("[INFO] stats %v", req) + + var err error + + indexConfig, err := s.raftServer.GetIndexConfig() + if err != nil { + return resp, status.Error(codes.Internal, err.Error()) + } + + indexConfigAny := &any.Any{} + err = protobuf.UnmarshalAny(indexConfig, indexConfigAny) if err != nil { - return status.Error(codes.Internal, err.Error()) + return resp, status.Error(codes.Internal, err.Error()) } - return stream.SendAndClose(result) + resp.IndexConfig = indexConfigAny + + return resp, nil } -func (s *GRPCService) GetStats(ctx context.Context, req *empty.Empty) (*index.Stats, error) { +func (s *GRPCService) GetIndexStats(ctx context.Context, req *empty.Empty) (*protobuf.GetIndexStatsResponse, error) { start := time.Now() - defer RecordMetrics(start, "stats") + defer RecordMetrics(start, "indexstats") - resp := &index.Stats{} + resp := &protobuf.GetIndexStatsResponse{} s.logger.Printf("[INFO] stats %v", req) var err error - resp, err = s.raftServer.Stats() + indexStats, err := s.raftServer.GetIndexStats() if err != nil { return resp, status.Error(codes.Internal, err.Error()) } + indexStatsAny := &any.Any{} + err = protobuf.UnmarshalAny(indexStats, indexStatsAny) + if err != nil { + return resp, status.Error(codes.Internal, err.Error()) + } + + resp.IndexStats = indexStatsAny + return resp, nil } diff --git a/indexer/http_handler.go b/indexer/http_handler.go index c3e0dee..8cdc8cd 100644 --- a/indexer/http_handler.go +++ b/indexer/http_handler.go @@ -22,16 +22,34 @@ import ( "time" "github.com/blevesearch/bleve" - "github.com/golang/protobuf/ptypes/any" "github.com/gorilla/mux" "github.com/mosuka/blast/errors" + "github.com/mosuka/blast/grpc" blasthttp "github.com/mosuka/blast/http" - "github.com/mosuka/blast/protobuf" - "github.com/mosuka/blast/protobuf/index" - pbindex "github.com/mosuka/blast/protobuf/index" "github.com/mosuka/blast/version" + "github.com/prometheus/client_golang/prometheus/promhttp" ) +func NewRouter(grpcAddr string, logger *log.Logger) (*blasthttp.Router, error) { + router, err := blasthttp.NewRouter(grpcAddr, logger) + if err != nil { + return nil, err + } + + router.StrictSlash(true) + + router.Handle("/", NewRootHandler(logger)).Methods("GET") + router.Handle("/documents", NewSetDocumentHandler(router.GRPCClient, logger)).Methods("PUT") + router.Handle("/documents", NewDeleteDocumentHandler(router.GRPCClient, logger)).Methods("DELETE") + router.Handle("/documents/{id}", NewGetDocumentHandler(router.GRPCClient, logger)).Methods("GET") + router.Handle("/documents/{id}", NewSetDocumentHandler(router.GRPCClient, logger)).Methods("PUT") + router.Handle("/documents/{id}", NewDeleteDocumentHandler(router.GRPCClient, logger)).Methods("DELETE") + router.Handle("/search", NewSearchHandler(router.GRPCClient, logger)).Methods("POST") + router.Handle("/metrics", promhttp.Handler()).Methods("GET") + + return router, nil +} + type RootHandler struct { logger *log.Logger } @@ -63,11 +81,11 @@ func (h *RootHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } type GetHandler struct { - client *GRPCClient + client *grpc.Client logger *log.Logger } -func NewGetHandler(client *GRPCClient, logger *log.Logger) *GetHandler { +func NewGetDocumentHandler(client *grpc.Client, logger *log.Logger) *GetHandler { return &GetHandler{ client: client, logger: logger, @@ -85,11 +103,9 @@ func (h *GetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - doc := &index.Document{ - Id: vars["id"], - } + id := vars["id"] - doc, err := h.client.Get(doc) + fields, err := h.client.GetDocument(id) if err != nil { switch err { case errors.ErrNotFound: @@ -111,43 +127,8 @@ func (h *GetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - // Any -> map[string]interface{} - var fieldsMap *map[string]interface{} - fieldsInstance, err := protobuf.MarshalAny(doc.Fields) - if err != nil { - httpStatus = http.StatusInternalServerError - - msgMap := map[string]interface{}{ - "message": err.Error(), - "status": httpStatus, - } - - content, err = blasthttp.NewJSONMessage(msgMap) - if err != nil { - h.logger.Printf("[ERR] %v", err) - } - - return - } - if fieldsInstance == nil { - httpStatus = http.StatusInternalServerError - - msgMap := map[string]interface{}{ - "message": err.Error(), - "status": httpStatus, - } - - content, err = blasthttp.NewJSONMessage(msgMap) - if err != nil { - h.logger.Printf("[ERR] %v", err) - } - - return - } - fieldsMap = fieldsInstance.(*map[string]interface{}) - // map[string]interface{} -> bytes - content, err = json.MarshalIndent(fieldsMap, "", " ") + content, err = json.MarshalIndent(fields, "", " ") if err != nil { httpStatus = http.StatusInternalServerError @@ -166,11 +147,11 @@ func (h *GetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } type IndexHandler struct { - client *GRPCClient + client *grpc.Client logger *log.Logger } -func NewIndexHandler(client *GRPCClient, logger *log.Logger) *IndexHandler { +func NewSetDocumentHandler(client *grpc.Client, logger *log.Logger) *IndexHandler { return &IndexHandler{ client: client, logger: logger, @@ -187,7 +168,7 @@ func (h *IndexHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { }() // create documents - docs := make([]*index.Document, 0) + docs := make([]map[string]interface{}, 0) vars := mux.Vars(r) id := vars["id"] @@ -211,8 +192,7 @@ func (h *IndexHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if id == "" { // Indexing documents in bulk - var docMaps []map[string]interface{} - err := json.Unmarshal(bodyBytes, &docMaps) + err := json.Unmarshal(bodyBytes, &docs) if err != nil { httpStatus = http.StatusBadRequest @@ -228,55 +208,10 @@ func (h *IndexHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - - for _, docMap := range docMaps { - fieldsAny := &any.Any{} - err = protobuf.UnmarshalAny(docMap["fields"], fieldsAny) - if err != nil { - httpStatus = http.StatusBadRequest - - msgMap := map[string]interface{}{ - "message": err.Error(), - "status": httpStatus, - } - - content, err = blasthttp.NewJSONMessage(msgMap) - if err != nil { - h.logger.Printf("[ERR] %v", err) - } - - return - } - - doc := &pbindex.Document{ - Id: docMap["id"].(string), - Fields: fieldsAny, - } - - docs = append(docs, doc) - } } else { // Indexing a document - var fieldsMap map[string]interface{} - err := json.Unmarshal(bodyBytes, &fieldsMap) - if err != nil { - httpStatus = http.StatusBadRequest - - msgMap := map[string]interface{}{ - "message": err.Error(), - "status": httpStatus, - } - - content, err = blasthttp.NewJSONMessage(msgMap) - if err != nil { - h.logger.Printf("[ERR] %v", err) - } - - return - } - - fieldsAny := &any.Any{} - err = protobuf.UnmarshalAny(fieldsMap, fieldsAny) + var fields map[string]interface{} + err := json.Unmarshal(bodyBytes, &fields) if err != nil { httpStatus = http.StatusBadRequest @@ -293,16 +228,16 @@ func (h *IndexHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - doc := &pbindex.Document{ - Id: id, - Fields: fieldsAny, + doc := map[string]interface{}{ + "id": id, + "fields": fields, } docs = append(docs, doc) } // index documents in bulk - result, err := h.client.Index(docs) + count, err := h.client.IndexDocument(docs) if err != nil { httpStatus = http.StatusInternalServerError @@ -319,7 +254,7 @@ func (h *IndexHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - content, err = json.MarshalIndent(result, "", " ") + content, err = json.MarshalIndent(count, "", " ") if err != nil { httpStatus = http.StatusInternalServerError @@ -338,11 +273,11 @@ func (h *IndexHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } type DeleteHandler struct { - client *GRPCClient + client *grpc.Client logger *log.Logger } -func NewDeleteHandler(client *GRPCClient, logger *log.Logger) *DeleteHandler { +func NewDeleteDocumentHandler(client *grpc.Client, logger *log.Logger) *DeleteHandler { return &DeleteHandler{ client: client, logger: logger, @@ -359,7 +294,7 @@ func (h *DeleteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { }() // create documents - docs := make([]*index.Document, 0) + ids := make([]string, 0) vars := mux.Vars(r) id := vars["id"] @@ -383,8 +318,7 @@ func (h *DeleteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if id == "" { // Deleting documents in bulk - var docMaps []map[string]interface{} - err := json.Unmarshal(bodyBytes, &docMaps) + err := json.Unmarshal(bodyBytes, &ids) if err != nil { httpStatus = http.StatusBadRequest @@ -400,25 +334,13 @@ func (h *DeleteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - - for _, docMap := range docMaps { - doc := &pbindex.Document{ - Id: docMap["id"].(string), - } - - docs = append(docs, doc) - } } else { // Deleting a document - doc := &pbindex.Document{ - Id: id, - } - - docs = append(docs, doc) + ids = append(ids, id) } // delete documents in bulk - result, err := h.client.Delete(docs) + result, err := h.client.DeleteDocument(ids) if err != nil { httpStatus = http.StatusInternalServerError @@ -454,11 +376,11 @@ func (h *DeleteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } type SearchHandler struct { - client *GRPCClient + client *grpc.Client logger *log.Logger } -func NewSearchHandler(client *GRPCClient, logger *log.Logger) *SearchHandler { +func NewSearchHandler(client *grpc.Client, logger *log.Logger) *SearchHandler { return &SearchHandler{ client: client, logger: logger, diff --git a/indexer/http_server.go b/indexer/http_server.go deleted file mode 100644 index 0941730..0000000 --- a/indexer/http_server.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (c) 2019 Minoru Osuka -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package indexer - -import ( - "log" - "net" - "net/http" - - "github.com/gorilla/mux" - accesslog "github.com/mash/go-accesslog" - "github.com/prometheus/client_golang/prometheus/promhttp" -) - -type HTTPServer struct { - listener net.Listener - router *mux.Router - - grpcClient *GRPCClient - - logger *log.Logger - httpLogger accesslog.Logger -} - -func NewHTTPServer(httpAddr string, grpcClient *GRPCClient, logger *log.Logger, httpLogger accesslog.Logger) (*HTTPServer, error) { - listener, err := net.Listen("tcp", httpAddr) - if err != nil { - return nil, err - } - - router := mux.NewRouter() - router.StrictSlash(true) - - router.Handle("/", NewRootHandler(logger)).Methods("GET") - router.Handle("/documents", NewIndexHandler(grpcClient, logger)).Methods("PUT") - router.Handle("/documents", NewDeleteHandler(grpcClient, logger)).Methods("DELETE") - router.Handle("/documents/{id}", NewGetHandler(grpcClient, logger)).Methods("GET") - router.Handle("/documents/{id}", NewIndexHandler(grpcClient, logger)).Methods("PUT") - router.Handle("/documents/{id}", NewDeleteHandler(grpcClient, logger)).Methods("DELETE") - router.Handle("/search", NewSearchHandler(grpcClient, logger)).Methods("POST") - router.Handle("/metrics", promhttp.Handler()).Methods("GET") - - return &HTTPServer{ - listener: listener, - router: router, - grpcClient: grpcClient, - logger: logger, - httpLogger: httpLogger, - }, nil -} - -func (s *HTTPServer) Start() error { - err := http.Serve( - s.listener, - accesslog.NewLoggingHandler( - s.router, - s.httpLogger, - ), - ) - if err != nil { - return err - } - - return nil -} - -func (s *HTTPServer) Stop() error { - err := s.listener.Close() - if err != nil { - return err - } - - return nil -} diff --git a/indexer/index.go b/indexer/index.go index 2887569..247bb6b 100644 --- a/indexer/index.go +++ b/indexer/index.go @@ -21,26 +21,36 @@ import ( "time" "github.com/blevesearch/bleve" - "github.com/blevesearch/bleve/mapping" "github.com/golang/protobuf/ptypes/any" blasterrors "github.com/mosuka/blast/errors" "github.com/mosuka/blast/protobuf" - pbindex "github.com/mosuka/blast/protobuf/index" ) type Index struct { - index bleve.Index + index bleve.Index + + indexConfig map[string]interface{} + logger *log.Logger } -func NewIndex(dir string, indexMapping *mapping.IndexMappingImpl, indexStorageType string, logger *log.Logger) (*Index, error) { +func NewIndex(dir string, indexConfig map[string]interface{}, logger *log.Logger) (*Index, error) { bleve.SetLog(logger) var index bleve.Index _, err := os.Stat(dir) if os.IsNotExist(err) { // create new index - index, err = bleve.NewUsing(dir, indexMapping, bleve.Config.DefaultIndexType, indexStorageType, nil) + indexMappingSrc, err := json.Marshal(indexConfig["index_mapping"]) + if err != nil { + return nil, err + } + indexMapping := bleve.NewIndexMapping() + err = json.Unmarshal(indexMappingSrc, indexMapping) + if err != nil { + return nil, err + } + index, err = bleve.NewUsing(dir, indexMapping, indexConfig["index_type"].(string), indexConfig["index_storage_type"].(string), nil) if err != nil { return nil, err } @@ -56,13 +66,14 @@ func NewIndex(dir string, indexMapping *mapping.IndexMappingImpl, indexStorageTy } return &Index{ - index: index, - logger: logger, + index: index, + indexConfig: indexConfig, + logger: logger, }, nil } -func (b *Index) Close() error { - err := b.index.Close() +func (i *Index) Close() error { + err := i.index.Close() if err != nil { return err } @@ -70,13 +81,13 @@ func (b *Index) Close() error { return nil } -func (b *Index) Get(id string) (map[string]interface{}, error) { +func (i *Index) Get(id string) (map[string]interface{}, error) { start := time.Now() defer func() { - b.logger.Printf("[DEBUG] get %s %f", id, float64(time.Since(start))/float64(time.Second)) + i.logger.Printf("[DEBUG] get %s %f", id, float64(time.Since(start))/float64(time.Second)) }() - fieldsBytes, err := b.index.GetInternal([]byte(id)) + fieldsBytes, err := i.index.GetInternal([]byte(id)) if err != nil { return nil, err } @@ -94,14 +105,14 @@ func (b *Index) Get(id string) (map[string]interface{}, error) { return fieldsMap, nil } -func (b *Index) Search(request *bleve.SearchRequest) (*bleve.SearchResult, error) { +func (i *Index) Search(request *bleve.SearchRequest) (*bleve.SearchResult, error) { start := time.Now() defer func() { rb, _ := json.Marshal(request) - b.logger.Printf("[DEBUG] search %s %f", rb, float64(time.Since(start))/float64(time.Second)) + i.logger.Printf("[DEBUG] search %s %f", rb, float64(time.Since(start))/float64(time.Second)) }() - result, err := b.index.Search(request) + result, err := i.index.Search(request) if err != nil { return nil, err } @@ -109,19 +120,19 @@ func (b *Index) Search(request *bleve.SearchRequest) (*bleve.SearchResult, error return result, nil } -func (b *Index) Index(id string, fields map[string]interface{}) error { +func (i *Index) Index(id string, fields map[string]interface{}) error { start := time.Now() defer func() { - b.logger.Printf("[DEBUG] index %s %v %f", id, fields, float64(time.Since(start))/float64(time.Second)) + i.logger.Printf("[DEBUG] index %s %v %f", id, fields, float64(time.Since(start))/float64(time.Second)) }() // index - b.logger.Printf("[DEBUG] index %s, %v", id, fields) - err := b.index.Index(id, fields) + i.logger.Printf("[DEBUG] index %s, %v", id, fields) + err := i.index.Index(id, fields) if err != nil { return err } - b.logger.Printf("[DEBUG] indexed %s, %v", id, fields) + i.logger.Printf("[DEBUG] indexed %s, %v", id, fields) // map[string]interface{} -> bytes fieldsBytes, err := json.Marshal(fields) @@ -130,7 +141,7 @@ func (b *Index) Index(id string, fields map[string]interface{}) error { } // set original document - err = b.index.SetInternal([]byte(id), fieldsBytes) + err = i.index.SetInternal([]byte(id), fieldsBytes) if err != nil { return err } @@ -138,19 +149,19 @@ func (b *Index) Index(id string, fields map[string]interface{}) error { return nil } -func (b *Index) Delete(id string) error { +func (i *Index) Delete(id string) error { start := time.Now() defer func() { - b.logger.Printf("[DEBUG] delete %s %f", id, float64(time.Since(start))/float64(time.Second)) + i.logger.Printf("[DEBUG] delete %s %f", id, float64(time.Since(start))/float64(time.Second)) }() - err := b.index.Delete(id) + err := i.index.Delete(id) if err != nil { return err } // delete original document - err = b.index.SetInternal([]byte(id), nil) + err = i.index.SetInternal([]byte(id), nil) if err != nil { return err } @@ -158,30 +169,39 @@ func (b *Index) Delete(id string) error { return nil } -func (b *Index) Stats() (map[string]interface{}, error) { +func (i *Index) Config() (map[string]interface{}, error) { + start := time.Now() + defer func() { + i.logger.Printf("[DEBUG] stats %f", float64(time.Since(start))/float64(time.Second)) + }() + + return i.indexConfig, nil +} + +func (i *Index) Stats() (map[string]interface{}, error) { start := time.Now() defer func() { - b.logger.Printf("[DEBUG] stats %f", float64(time.Since(start))/float64(time.Second)) + i.logger.Printf("[DEBUG] stats %f", float64(time.Since(start))/float64(time.Second)) }() - stats := b.index.StatsMap() + stats := i.index.StatsMap() return stats, nil } -func (b *Index) SnapshotItems() <-chan *pbindex.Document { - ch := make(chan *pbindex.Document, 1024) +func (i *Index) SnapshotItems() <-chan *protobuf.Document { + ch := make(chan *protobuf.Document, 1024) go func() { - i, _, err := b.index.Advanced() + idx, _, err := i.index.Advanced() if err != nil { - b.logger.Printf("[ERR] %v", err) + i.logger.Printf("[ERR] %v", err) return } - r, err := i.Reader() + r, err := idx.Reader() if err != nil { - b.logger.Printf("[ERR] %v", err) + i.logger.Printf("[ERR] %v", err) return } @@ -191,34 +211,34 @@ func (b *Index) SnapshotItems() <-chan *pbindex.Document { for { id, err := dr.Next() if id == nil { - b.logger.Print("[DEBUG] finished to read all document ids") + i.logger.Print("[DEBUG] finished to read all document ids") break } else if err != nil { - b.logger.Printf("[WARN] %v", err) + i.logger.Printf("[WARN] %v", err) continue } // get original document - fieldsBytes, err := b.index.GetInternal(id) + fieldsBytes, err := i.index.GetInternal(id) // bytes -> map[string]interface{} var fieldsMap map[string]interface{} err = json.Unmarshal([]byte(fieldsBytes), &fieldsMap) if err != nil { - b.logger.Printf("[ERR] %v", err) + i.logger.Printf("[ERR] %v", err) break } - b.logger.Printf("[DEBUG] %v", fieldsMap) + i.logger.Printf("[DEBUG] %v", fieldsMap) // map[string]interface{} -> Any fieldsAny := &any.Any{} err = protobuf.UnmarshalAny(fieldsMap, fieldsAny) if err != nil { - b.logger.Printf("[ERR] %v", err) + i.logger.Printf("[ERR] %v", err) break } - doc := &pbindex.Document{ + doc := &protobuf.Document{ Id: string(id), Fields: fieldsAny, } @@ -228,10 +248,10 @@ func (b *Index) SnapshotItems() <-chan *pbindex.Document { docCount = docCount + 1 } - b.logger.Print("[DEBUG] finished to write all documents to channel") + i.logger.Print("[DEBUG] finished to write all documents to channel") ch <- nil - b.logger.Printf("[INFO] snapshot total %d documents", docCount) + i.logger.Printf("[INFO] snapshot total %d documents", docCount) return }() diff --git a/indexer/metric.go b/indexer/metric.go index 6dd484f..a85d552 100644 --- a/indexer/metric.go +++ b/indexer/metric.go @@ -22,7 +22,7 @@ import ( var ( namespace = "blast" - subsystem = "index" + subsystem = "indexer" DurationSeconds = prometheus.NewHistogramVec( prometheus.HistogramOpts{ diff --git a/indexer/raft_command.go b/indexer/raft_command.go new file mode 100644 index 0000000..3cab8f0 --- /dev/null +++ b/indexer/raft_command.go @@ -0,0 +1,43 @@ +// Copyright (c) 2019 Minoru Osuka +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package indexer + +import "encoding/json" + +type command int + +const ( + unknown command = iota + setNode + deleteNode + indexDocument + deleteDocument +) + +type message struct { + Command command `json:"command,omitempty"` + Data json.RawMessage `json:"data,omitempty"` +} + +func newMessage(cmd command, data interface{}) (*message, error) { + b, err := json.Marshal(data) + if err != nil { + return nil, err + } + return &message{ + Command: cmd, + Data: b, + }, nil +} diff --git a/indexer/raft_fsm.go b/indexer/raft_fsm.go index e63c78b..123cc33 100644 --- a/indexer/raft_fsm.go +++ b/indexer/raft_fsm.go @@ -15,43 +15,55 @@ package indexer import ( + "encoding/json" "errors" "io" "io/ioutil" "log" + "sync" "github.com/blevesearch/bleve" - "github.com/blevesearch/bleve/mapping" "github.com/golang/protobuf/proto" "github.com/hashicorp/raft" - blasterrors "github.com/mosuka/blast/errors" + "github.com/mosuka/blast/maputils" "github.com/mosuka/blast/protobuf" - pbindex "github.com/mosuka/blast/protobuf/index" - blastraft "github.com/mosuka/blast/protobuf/raft" ) type RaftFSM struct { + metadata maputils.Map + metadataMutex sync.RWMutex + + path string + index *Index - metadata map[string]*blastraft.Node + indexConfig map[string]interface{} logger *log.Logger } -func NewRaftFSM(path string, indexMapping *mapping.IndexMappingImpl, indexStorageType string, logger *log.Logger) (*RaftFSM, error) { - index, err := NewIndex(path, indexMapping, indexStorageType, logger) +func NewRaftFSM(path string, indexConfig map[string]interface{}, logger *log.Logger) (*RaftFSM, error) { + return &RaftFSM{ + path: path, + indexConfig: indexConfig, + logger: logger, + }, nil +} + +func (f *RaftFSM) Start() error { + var err error + + f.metadata = maputils.Map{} + + f.index, err = NewIndex(f.path, f.indexConfig, f.logger) if err != nil { - return nil, err + return err } - return &RaftFSM{ - metadata: make(map[string]*blastraft.Node, 0), - index: index, - logger: logger, - }, nil + return nil } -func (f *RaftFSM) Close() error { +func (f *RaftFSM) Stop() error { err := f.index.Close() if err != nil { return err @@ -60,154 +72,131 @@ func (f *RaftFSM) Close() error { return nil } -func (f *RaftFSM) Get(id string) (map[string]interface{}, error) { - fields, err := f.index.Get(id) +func (f *RaftFSM) GetMetadata(id string) (map[string]interface{}, error) { + f.metadataMutex.RLock() + defer f.metadataMutex.RUnlock() + + value, err := f.metadata.Get(id) if err != nil { return nil, err } - return fields, nil + return value.(maputils.Map).ToMap(), nil } -func (f *RaftFSM) applyIndex(id string, fields map[string]interface{}) interface{} { - f.logger.Printf("[DEBUG] index %s, %v", id, fields) +func (f *RaftFSM) applySetMetadata(id string, value map[string]interface{}) interface{} { + f.metadataMutex.RLock() + defer f.metadataMutex.RUnlock() - err := f.index.Index(id, fields) + err := f.metadata.Merge(id, value) if err != nil { - f.logger.Printf("[ERR] %v", err) return err } return nil } -func (f *RaftFSM) applyDelete(id string) interface{} { - err := f.index.Delete(id) +func (f *RaftFSM) applyDeleteMetadata(id string) interface{} { + f.metadataMutex.RLock() + defer f.metadataMutex.RUnlock() + + err := f.metadata.Delete(id) if err != nil { - f.logger.Printf("[ERR] %v", err) return err } return nil } -func (f *RaftFSM) Search(request *bleve.SearchRequest) (*bleve.SearchResult, error) { - result, err := f.index.Search(request) +func (f *RaftFSM) GetDocument(id string) (map[string]interface{}, error) { + fields, err := f.index.Get(id) if err != nil { return nil, err } - return result, nil + return fields, nil } -func (f *RaftFSM) GetMetadata(nodeId string) (*blastraft.Node, error) { - node, exists := f.metadata[nodeId] - if !exists { - return nil, blasterrors.ErrNotFound - } - if node == nil { - return nil, errors.New("nil") +func (f *RaftFSM) applyIndexDocument(id string, fields map[string]interface{}) interface{} { + f.logger.Printf("[DEBUG] index %s, %v", id, fields) + + err := f.index.Index(id, fields) + if err != nil { + f.logger.Printf("[ERR] %v", err) + return err } - value := node - return value, nil + return nil } -func (f *RaftFSM) applySetMetadata(nodeId string, node *blastraft.Node) interface{} { - f.metadata[nodeId] = node +func (f *RaftFSM) applyDeleteDocument(id string) interface{} { + err := f.index.Delete(id) + if err != nil { + f.logger.Printf("[ERR] %v", err) + return err + } return nil } -func (f *RaftFSM) applyDeleteMetadata(nodeId string) interface{} { - _, exists := f.metadata[nodeId] - if exists { - delete(f.metadata, nodeId) +func (f *RaftFSM) Search(request *bleve.SearchRequest) (*bleve.SearchResult, error) { + result, err := f.index.Search(request) + if err != nil { + return nil, err } - return nil + return result, nil } func (f *RaftFSM) Apply(l *raft.Log) interface{} { - var c pbindex.IndexCommand - err := proto.Unmarshal(l.Data, &c) + var msg message + err := json.Unmarshal(l.Data, &msg) if err != nil { return err } - f.logger.Printf("[DEBUG] Apply %v", c) - - switch c.Type { - case pbindex.IndexCommand_SET_METADATA: - // Any -> Node - nodeInstance, err := protobuf.MarshalAny(c.Data) - if err != nil { - return err - } - if nodeInstance == nil { - return errors.New("nil") - } - metadata := nodeInstance.(*blastraft.Node) + f.logger.Printf("[DEBUG] Apply %v", msg) - return f.applySetMetadata(metadata.Id, metadata) - case pbindex.IndexCommand_DELETE_METADATA: - // Any -> Node - metadataInstance, err := protobuf.MarshalAny(c.Data) + switch msg.Command { + case setNode: + var data map[string]interface{} + err := json.Unmarshal(msg.Data, &data) if err != nil { return err } - if metadataInstance == nil { - return errors.New("nil") - } - metadata := *metadataInstance.(*blastraft.Node) - - return f.applyDeleteMetadata(metadata.Id) - case pbindex.IndexCommand_INDEX_DOCUMENT: - // Any -> Document - docInstance, err := protobuf.MarshalAny(c.Data) + return f.applySetMetadata(data["id"].(string), data["metadata"].(map[string]interface{})) + case deleteNode: + var data map[string]interface{} + err := json.Unmarshal(msg.Data, &data) if err != nil { return err } - if docInstance == nil { - return errors.New("nil") - } - doc := *docInstance.(*pbindex.Document) - - // Any -> map[string]interface{} - fieldsInstance, err := protobuf.MarshalAny(doc.Fields) + return f.applyDeleteMetadata(data["id"].(string)) + case indexDocument: + var data map[string]interface{} + err := json.Unmarshal(msg.Data, &data) if err != nil { return err } - if fieldsInstance == nil { - return errors.New("nil") - } - fields := *fieldsInstance.(*map[string]interface{}) - - return f.applyIndex(doc.Id, fields) - case pbindex.IndexCommand_DELETE_DOCUMENT: - // Any -> Document - docInstance, err := protobuf.MarshalAny(c.Data) + return f.applyIndexDocument(data["id"].(string), data["fields"].(map[string]interface{})) + case deleteDocument: + var data string + err := json.Unmarshal(msg.Data, &data) if err != nil { return err } - if docInstance == nil { - return errors.New("nil") - } - doc := *docInstance.(*pbindex.Document) - - return f.applyDelete(doc.Id) + return f.applyDeleteDocument(data) default: return errors.New("command type not support") } } -func (f *RaftFSM) Stats() (map[string]interface{}, error) { - stats, err := f.index.Stats() - if err != nil { - return nil, err - } +func (f *RaftFSM) GetIndexConfig() (map[string]interface{}, error) { + return f.index.Config() +} - return stats, nil +func (f *RaftFSM) GetIndexStats() (map[string]interface{}, error) { + return f.index.Stats() } func (f *RaftFSM) Snapshot() (raft.FSMSnapshot, error) { @@ -235,7 +224,7 @@ func (f *RaftFSM) Restore(rc io.ReadCloser) error { buff := proto.NewBuffer(data) for { - doc := &pbindex.Document{} + doc := &protobuf.Document{} err = buff.DecodeMessage(doc) if err == io.ErrUnexpectedEOF { break @@ -245,7 +234,6 @@ func (f *RaftFSM) Restore(rc io.ReadCloser) error { return err } - // Any -> map[string]interface{} fields, err := protobuf.MarshalAny(doc.Fields) if err != nil { return err @@ -296,14 +284,12 @@ func (f *IndexFSMSnapshot) Persist(sink raft.SnapshotSink) error { } docCount = docCount + 1 - - buff := proto.NewBuffer([]byte{}) - err := buff.EncodeMessage(doc) + docBytes, err := json.Marshal(doc) if err != nil { return err } - _, err = sink.Write(buff.Bytes()) + _, err = sink.Write(docBytes) if err != nil { return err } diff --git a/indexer/raft_server.go b/indexer/raft_server.go index d23c8ae..d88cf52 100644 --- a/indexer/raft_server.go +++ b/indexer/raft_server.go @@ -15,73 +15,84 @@ package indexer import ( + "encoding/json" "log" "net" "path/filepath" "time" "github.com/blevesearch/bleve" - "github.com/blevesearch/bleve/mapping" - "github.com/golang/protobuf/proto" - "github.com/golang/protobuf/ptypes/any" "github.com/hashicorp/raft" raftboltdb "github.com/hashicorp/raft-boltdb" _ "github.com/mosuka/blast/config" "github.com/mosuka/blast/errors" - "github.com/mosuka/blast/protobuf" - "github.com/mosuka/blast/protobuf/index" - blastraft "github.com/mosuka/blast/protobuf/raft" ) type RaftServer struct { - Node *blastraft.Node + id string + metadata map[string]interface{} + bootstrap bool raft *raft.Raft fsm *RaftFSM + indexConfig map[string]interface{} + logger *log.Logger } -func NewRaftServer(node *blastraft.Node, bootstrap bool, indexMapping *mapping.IndexMappingImpl, indexStorageType string, logger *log.Logger) (*RaftServer, error) { - fsm, err := NewRaftFSM(filepath.Join(node.DataDir, "index"), indexMapping, indexStorageType, logger) - if err != nil { - return nil, err - } - +func NewRaftServer(id string, metadata map[string]interface{}, bootstrap bool, indexConfig map[string]interface{}, logger *log.Logger) (*RaftServer, error) { return &RaftServer{ - Node: node, + id: id, + metadata: metadata, + bootstrap: bootstrap, - fsm: fsm, - logger: logger, + + indexConfig: indexConfig, + logger: logger, }, nil } func (s *RaftServer) Start() error { + var err error + + s.logger.Print("[INFO] create finite state machine") + s.fsm, err = NewRaftFSM(filepath.Join(s.metadata["data_dir"].(string), "index"), s.indexConfig, s.logger) + if err != nil { + return err + } + + s.logger.Print("[INFO] start finite state machine") + err = s.fsm.Start() + if err != nil { + return err + } + config := raft.DefaultConfig() - config.LocalID = raft.ServerID(s.Node.Id) + config.LocalID = raft.ServerID(s.id) config.SnapshotThreshold = 1024 config.Logger = s.logger - addr, err := net.ResolveTCPAddr("tcp", s.Node.BindAddr) + addr, err := net.ResolveTCPAddr("tcp", s.metadata["bind_addr"].(string)) if err != nil { return err } // create transport - transport, err := raft.NewTCPTransportWithLogger(s.Node.BindAddr, addr, 3, 10*time.Second, s.logger) + transport, err := raft.NewTCPTransportWithLogger(s.metadata["bind_addr"].(string), addr, 3, 10*time.Second, s.logger) if err != nil { return err } // create snapshot store - snapshotStore, err := raft.NewFileSnapshotStoreWithLogger(s.Node.DataDir, 2, s.logger) + snapshotStore, err := raft.NewFileSnapshotStoreWithLogger(s.metadata["data_dir"].(string), 2, s.logger) if err != nil { return err } // create raft log store - raftLogStore, err := raftboltdb.NewBoltStore(filepath.Join(s.Node.DataDir, "raft.db")) + raftLogStore, err := raftboltdb.NewBoltStore(filepath.Join(s.metadata["data_dir"].(string), "raft.db")) if err != nil { return err } @@ -110,12 +121,13 @@ func (s *RaftServer) Start() error { s.logger.Printf("[WARN] %v", err) } else { s.logger.Printf("[ERR] %v", err) - return nil + return err } } // set metadata - err = s.setMetadata(s.Node.Id, s.Node) + s.logger.Print("[INFO] register itself in a cluster") + err = s.setMetadata(s.id, s.metadata) if err != nil { s.logger.Printf("[ERR] %v", err) return nil @@ -126,34 +138,20 @@ func (s *RaftServer) Start() error { } func (s *RaftServer) Stop() error { - err := s.fsm.Close() + s.logger.Print("[INFO] shutdown Raft") + f := s.raft.Shutdown() + err := f.Error() if err != nil { return err } - return nil -} - -func (s *RaftServer) WaitForDetectLeader(timeout time.Duration) error { - ticker := time.NewTicker(1000 * time.Millisecond) - defer ticker.Stop() - timer := time.NewTimer(timeout) - defer timer.Stop() - - for { - select { - case <-ticker.C: - leaderAddr := s.raft.Leader() - if leaderAddr != "" { - s.logger.Printf("[INFO] detected %v as a leader", leaderAddr) - return nil - } else { - s.logger.Printf("[WARN] %v", errors.ErrNotFoundLeader) - } - case <-timer.C: - return errors.ErrTimeout - } + s.logger.Print("[INFO] stop finite state machine") + err = s.fsm.Stop() + if err != nil { + return err } + + return nil } func (s *RaftServer) LeaderAddress(timeout time.Duration) (raft.ServerAddress, error) { @@ -176,13 +174,13 @@ func (s *RaftServer) LeaderAddress(timeout time.Duration) (raft.ServerAddress, e } func (s *RaftServer) LeaderID(timeout time.Duration) (raft.ServerID, error) { - cf := s.raft.GetConfiguration() - err := cf.Error() + leaderAddr, err := s.LeaderAddress(timeout) if err != nil { return "", err } - leaderAddr, err := s.LeaderAddress(timeout) + cf := s.raft.GetConfiguration() + err = cf.Error() if err != nil { return "", err } @@ -196,34 +194,54 @@ func (s *RaftServer) LeaderID(timeout time.Duration) (raft.ServerID, error) { return "", errors.ErrNotFoundLeader } -func (s *RaftServer) getMetadata(nodeId string) (*blastraft.Node, error) { - node, err := s.fsm.GetMetadata(nodeId) +func (s *RaftServer) Stats() map[string]string { + return s.raft.Stats() +} + +func (s *RaftServer) State() string { + return s.raft.State().String() +} + +func (s *RaftServer) IsLeader() bool { + return s.raft.State() == raft.Leader +} + +func (s *RaftServer) WaitForDetectLeader(timeout time.Duration) error { + _, err := s.LeaderAddress(timeout) if err != nil { - return nil, err + return err } - return node, nil + return nil } -func (s *RaftServer) setMetadata(nodeId string, node *blastraft.Node) error { - // Node -> Any - nodeAny := &any.Any{} - err := protobuf.UnmarshalAny(node, nodeAny) +func (s *RaftServer) getMetadata(id string) (map[string]interface{}, error) { + metadata, err := s.fsm.GetMetadata(id) if err != nil { - return err + return nil, err } - c := &index.IndexCommand{ - Type: index.IndexCommand_SET_METADATA, - Data: nodeAny, + return metadata, nil +} + +func (s *RaftServer) setMetadata(id string, metadata map[string]interface{}) error { + msg, err := newMessage( + setNode, + map[string]interface{}{ + "id": id, + "metadata": metadata, + }, + ) + if err != nil { + return err } - msg, err := proto.Marshal(c) + msgBytes, err := json.Marshal(msg) if err != nil { return err } - f := s.raft.Apply(msg, 10*time.Second) + f := s.raft.Apply(msgBytes, 10*time.Second) err = f.Error() if err != nil { return err @@ -232,29 +250,23 @@ func (s *RaftServer) setMetadata(nodeId string, node *blastraft.Node) error { return nil } -func (s *RaftServer) deleteMetadata(nodeId string) error { - node := &blastraft.Node{ - Id: nodeId, - } - - // Node -> Any - nodeAny := &any.Any{} - err := protobuf.UnmarshalAny(node, nodeAny) +func (s *RaftServer) deleteMetadata(id string) error { + msg, err := newMessage( + deleteNode, + map[string]interface{}{ + "id": id, + }, + ) if err != nil { return err } - c := &index.IndexCommand{ - Type: index.IndexCommand_DELETE_METADATA, - Data: nodeAny, - } - - msg, err := proto.Marshal(c) + msgBytes, err := json.Marshal(msg) if err != nil { return err } - f := s.raft.Apply(msg, 10*time.Second) + f := s.raft.Apply(msgBytes, 10*time.Second) err = f.Error() if err != nil { return err @@ -263,39 +275,30 @@ func (s *RaftServer) deleteMetadata(nodeId string) error { return nil } -func (s *RaftServer) Join(node *blastraft.Node) error { - if s.raft.State() != raft.Leader { - // forward to leader node - leaderId, err := s.LeaderID(60 * time.Second) - if err != nil { - return err - } - - node, err := s.getMetadata(string(leaderId)) - if err != nil { - s.logger.Printf("[ERR] %v", err) - return nil - } +func (s *RaftServer) GetMetadata(id string) (map[string]interface{}, error) { + cf := s.raft.GetConfiguration() + err := cf.Error() + if err != nil { + return nil, err + } - client, err := NewGRPCClient(string(node.GrpcAddr)) - defer func() { - err := client.Close() + var metadata map[string]interface{} + for _, server := range cf.Configuration().Servers { + if server.ID == raft.ServerID(id) { + metadata, err = s.getMetadata(id) if err != nil { - s.logger.Printf("[ERR] %v", err) + return nil, err } - }() - if err != nil { - s.logger.Printf("[ERR] %v", err) - return nil + break } + } - err = client.Join(node) - if err != nil { - s.logger.Printf("[ERR] %v", err) - return nil - } + return metadata, nil +} - return nil +func (s *RaftServer) SetMetadata(id string, metadata map[string]interface{}) error { + if !s.IsLeader() { + return raft.ErrNotLeader } cf := s.raft.GetConfiguration() @@ -305,62 +308,32 @@ func (s *RaftServer) Join(node *blastraft.Node) error { } for _, server := range cf.Configuration().Servers { - if server.ID == raft.ServerID(node.Id) { - s.logger.Printf("[INFO] node %s already joined the cluster", node.Id) + if server.ID == raft.ServerID(id) { + s.logger.Printf("[INFO] node %v already joined the cluster", id) return nil } } - f := s.raft.AddVoter(raft.ServerID(node.Id), raft.ServerAddress(node.BindAddr), 0, 0) + f := s.raft.AddVoter(raft.ServerID(id), raft.ServerAddress(metadata["bind_addr"].(string)), 0, 0) err = f.Error() if err != nil { return err } // set metadata - err = s.setMetadata(node.Id, node) + err = s.setMetadata(id, metadata) if err != nil { s.logger.Printf("[ERR] %v", err) return nil } - s.logger.Printf("[INFO] node %s at %s joined successfully", node.Id, node.BindAddr) + s.logger.Printf("[INFO] node %v joined successfully", id) return nil } -func (s *RaftServer) Leave(node *blastraft.Node) error { - if s.raft.State() != raft.Leader { - // forward to leader node - leaderId, err := s.LeaderID(60 * time.Second) - if err != nil { - return err - } - - node, err := s.getMetadata(string(leaderId)) - if err != nil { - s.logger.Printf("[ERR] %v", err) - return nil - } - - client, err := NewGRPCClient(string(node.GrpcAddr)) - defer func() { - err := client.Close() - if err != nil { - s.logger.Printf("[ERR] %v", err) - } - }() - if err != nil { - s.logger.Printf("[ERR] %v", err) - return nil - } - - err = client.Leave(node) - if err != nil { - s.logger.Printf("[ERR] %v", err) - return nil - } - - return nil +func (s *RaftServer) DeleteMetadata(id string) error { + if !s.IsLeader() { + return raft.ErrNotLeader } cf := s.raft.GetConfiguration() @@ -370,97 +343,48 @@ func (s *RaftServer) Leave(node *blastraft.Node) error { } for _, server := range cf.Configuration().Servers { - if server.ID == raft.ServerID(node.Id) { + if server.ID == raft.ServerID(id) { f := s.raft.RemoveServer(server.ID, 0, 0) err = f.Error() if err != nil { return err } - s.logger.Printf("[INFO] node %s leaved successfully", node.Id) + s.logger.Printf("[INFO] node %v leaved successfully", id) return nil } } // delete metadata - err = s.deleteMetadata(node.Id) + err = s.deleteMetadata(id) if err != nil { s.logger.Printf("[ERR] %v", err) return nil } - s.logger.Printf("[INFO] node %s does not exists in the cluster", node.Id) + s.logger.Printf("[INFO] node %v does not exists in the cluster", id) return nil } -func (s *RaftServer) GetNode() (*blastraft.Node, error) { +func (s *RaftServer) GetServers() (map[string]interface{}, error) { cf := s.raft.GetConfiguration() err := cf.Error() if err != nil { return nil, err } - leaderAddr, err := s.LeaderAddress(60 * time.Second) - if err != nil { - return nil, err - } - - node := &blastraft.Node{} + servers := map[string]interface{}{} for _, server := range cf.Configuration().Servers { - if server.ID == raft.ServerID(s.Node.Id) { - node.Id = string(server.ID) - node.BindAddr = string(server.Address) - node.Leader = server.Address == leaderAddr - - nodeInfo, err := s.getMetadata(node.Id) - if err != nil { - s.logger.Printf("[WARN] %v", err) - break - } - node.GrpcAddr = nodeInfo.GrpcAddr - node.HttpAddr = nodeInfo.HttpAddr - node.DataDir = nodeInfo.DataDir - break - } - } - - return node, nil -} - -func (s *RaftServer) GetCluster() (*blastraft.Cluster, error) { - cf := s.raft.GetConfiguration() - err := cf.Error() - if err != nil { - return nil, err - } - - leaderAddr, err := s.LeaderAddress(60 * time.Second) - if err != nil { - return nil, err - } - - nodes := make([]*blastraft.Node, 0) - for _, server := range cf.Configuration().Servers { - node := &blastraft.Node{} - node.Id = string(server.ID) - node.BindAddr = string(server.Address) - node.Leader = server.Address == leaderAddr - - nodeInfo, err := s.getMetadata(node.Id) + metadata, err := s.GetMetadata(string(server.ID)) if err != nil { - s.logger.Printf("[WARN] %v", err) + s.logger.Printf("[DEBUG] %v", err) continue } - node.GrpcAddr = nodeInfo.GrpcAddr - node.HttpAddr = nodeInfo.HttpAddr - node.DataDir = nodeInfo.DataDir - nodes = append(nodes, node) + servers[string(server.ID)] = metadata } - return &blastraft.Cluster{ - Nodes: nodes, - }, nil + return servers, nil } func (s *RaftServer) Snapshot() error { @@ -473,25 +397,13 @@ func (s *RaftServer) Snapshot() error { return nil } -func (s *RaftServer) Get(doc *index.Document) (*index.Document, error) { - fieldsMap, err := s.fsm.Get(doc.Id) - if err != nil { - return nil, err - } - - // map[string]interface{} -> Any - fieldsAny := &any.Any{} - err = protobuf.UnmarshalAny(fieldsMap, fieldsAny) +func (s *RaftServer) GetDocument(id string) (map[string]interface{}, error) { + fields, err := s.fsm.GetDocument(id) if err != nil { return nil, err } - retDoc := &index.Document{ - Id: doc.Id, - Fields: fieldsAny, - } - - return retDoc, nil + return fields, nil } func (s *RaftServer) Search(request *bleve.SearchRequest) (*bleve.SearchResult, error) { @@ -503,160 +415,84 @@ func (s *RaftServer) Search(request *bleve.SearchRequest) (*bleve.SearchResult, return result, nil } -func (s *RaftServer) Index(docs []*index.Document) (*index.UpdateResult, error) { +func (s *RaftServer) IndexDocument(docs []map[string]interface{}) (int, error) { if s.raft.State() != raft.Leader { - // forward to leader node - leaderId, err := s.LeaderID(60 * time.Second) - if err != nil { - return nil, err - } - - node, err := s.getMetadata(string(leaderId)) - if err != nil { - s.logger.Printf("[ERR] %v", err) - return nil, err - } - - client, err := NewGRPCClient(string(node.GrpcAddr)) - defer func() { - err := client.Close() - if err != nil { - s.logger.Printf("[ERR] %v", err) - } - }() - if err != nil { - s.logger.Printf("[ERR] %v", err) - return nil, err - } - - result, err := client.Index(docs) - if err != nil { - s.logger.Printf("[ERR] %v", err) - return nil, err - } - s.logger.Printf("[DEBUG] %v", result) - - return result, nil + return -1, raft.ErrNotLeader } - count := int32(0) + count := 0 for _, doc := range docs { - // Document -> Any - docAny := &any.Any{} - err := protobuf.UnmarshalAny(doc, docAny) + msg, err := newMessage( + indexDocument, + doc, + ) if err != nil { - return nil, err - } - - c := &index.IndexCommand{ - Type: index.IndexCommand_INDEX_DOCUMENT, - Data: docAny, + return -1, err } - msg, err := proto.Marshal(c) + msgBytes, err := json.Marshal(msg) if err != nil { - return nil, err + return -1, err } - f := s.raft.Apply(msg, 10*time.Second) + f := s.raft.Apply(msgBytes, 10*time.Second) err = f.Error() if err != nil { - return nil, err + return -1, err } count++ } - return &index.UpdateResult{ - Count: count, - }, nil + return count, nil } -func (s *RaftServer) Delete(docs []*index.Document) (*index.UpdateResult, error) { +func (s *RaftServer) DeleteDocument(ids []string) (int, error) { if s.raft.State() != raft.Leader { - // forward to leader node - leaderId, err := s.LeaderID(60 * time.Second) - if err != nil { - return nil, err - } - - node, err := s.getMetadata(string(leaderId)) - if err != nil { - s.logger.Printf("[ERR] %v", err) - return nil, err - } - - client, err := NewGRPCClient(string(node.GrpcAddr)) - defer func() { - err := client.Close() - if err != nil { - s.logger.Printf("[ERR] %v", err) - } - }() - if err != nil { - s.logger.Printf("[ERR] %v", err) - return nil, err - } - - result, err := client.Delete(docs) - if err != nil { - s.logger.Printf("[ERR] %v", err) - return nil, err - } - s.logger.Printf("[DEBUG] %v", result) - - return result, nil + return -1, raft.ErrNotLeader } - count := int32(0) - for _, doc := range docs { - // Document -> Any - docAny := &any.Any{} - err := protobuf.UnmarshalAny(doc, docAny) + count := 0 + for _, id := range ids { + msg, err := newMessage( + deleteDocument, + id, + ) if err != nil { - return nil, err - } - - c := &index.IndexCommand{ - Type: index.IndexCommand_DELETE_DOCUMENT, - Data: docAny, + return -1, err } - msg, err := proto.Marshal(c) + msgBytes, err := json.Marshal(msg) if err != nil { - return nil, err + return -1, err } - f := s.raft.Apply(msg, 10*time.Second) + f := s.raft.Apply(msgBytes, 10*time.Second) err = f.Error() if err != nil { - return nil, err + return -1, err } count++ } - return &index.UpdateResult{ - Count: count, - }, nil + return count, nil } -func (s *RaftServer) Stats() (*index.Stats, error) { - statsMap, err := s.fsm.Stats() +func (s *RaftServer) GetIndexConfig() (map[string]interface{}, error) { + indexConfig, err := s.fsm.GetIndexConfig() if err != nil { return nil, err } - // map[string]interface{} -> Any - statsAny := &any.Any{} - err = protobuf.UnmarshalAny(statsMap, statsAny) + return indexConfig, nil +} + +func (s *RaftServer) GetIndexStats() (map[string]interface{}, error) { + indexStats, err := s.fsm.GetIndexStats() if err != nil { return nil, err } - indexStats := &index.Stats{ - Stats: statsAny, - } - return indexStats, nil } diff --git a/indexer/server.go b/indexer/server.go index 442a4a7..f8b5af3 100644 --- a/indexer/server.go +++ b/indexer/server.go @@ -15,130 +15,219 @@ package indexer import ( - "encoding/json" - "io/ioutil" + "fmt" "log" - "os" - "github.com/blevesearch/bleve/mapping" accesslog "github.com/mash/go-accesslog" - "github.com/mosuka/blast/protobuf/raft" + "github.com/mosuka/blast/errors" + "github.com/mosuka/blast/grpc" + "github.com/mosuka/blast/http" + "github.com/mosuka/blast/protobuf" ) type Server struct { - node *raft.Node - bootstrap bool - joinAddr string + managerAddr string + clusterId string - raftServer *RaftServer + id string + metadata map[string]interface{} - grpcService *GRPCService - grpcServer *GRPCServer - grpcClient *GRPCClient + peerAddr string + + indexConfig map[string]interface{} - httpServer *HTTPServer + raftServer *RaftServer + grpcService *GRPCService + grpcServer *grpc.Server + httpRouter *http.Router + httpServer *http.Server logger *log.Logger httpLogger accesslog.Logger } -func NewServer(nodeId string, bindAddr string, grpcAddr string, httpAddr string, dataDir string, joinAddr string, indexMappingPath string, indexStorageType string, logger *log.Logger, httpLogger accesslog.Logger) (*Server, error) { - var err error +func NewServer(managerAddr string, clusterId string, id string, metadata map[string]interface{}, peerAddr string, indexConfig map[string]interface{}, logger *log.Logger, httpLogger accesslog.Logger) (*Server, error) { + return &Server{ + managerAddr: managerAddr, + clusterId: clusterId, + + id: id, + metadata: metadata, + + peerAddr: peerAddr, + + indexConfig: indexConfig, - server := &Server{ - bootstrap: joinAddr == "", - joinAddr: joinAddr, logger: logger, httpLogger: httpLogger, - } + }, nil +} - // set default index mapping - indexMapping := mapping.NewIndexMapping() +func (s *Server) Start() { + // get peer from manager + if s.managerAddr != "" { + s.logger.Printf("[INFO] connect to master %s", s.managerAddr) - // check index mapping file - if indexMappingPath != "" { - _, err = os.Stat(indexMappingPath) - if err == nil { - // read index mapping file - indexMappingFile, err := os.Open(indexMappingPath) + mc, err := grpc.NewClient(s.managerAddr) + defer func() { + s.logger.Printf("[DEBUG] close client for %v", mc.GetAddress()) + err = mc.Close() if err != nil { - return nil, err + s.logger.Printf("[ERR] %v", err) + return } - defer func() { - err = indexMappingFile.Close() - if err != nil { - server.logger.Printf("[ERR] %v", err) + }() + if err != nil { + s.logger.Printf("[ERR] %v", err) + return + } + + s.logger.Printf("[INFO] get nodes in cluster: %s", s.clusterId) + clusterIntr, err := mc.GetState(fmt.Sprintf("cluster_config/clusters/%s/nodes", s.clusterId)) + if err == errors.ErrNotFound { + // cluster does not found + s.logger.Printf("[INFO] cluster does not found: %s", s.clusterId) + } else if err != nil { + s.logger.Printf("[ERR] %v", err) + return + } else { + if clusterIntr == nil { + s.logger.Print("[INFO] value is nil") + } else { + cluster := *clusterIntr.(*map[string]interface{}) + for nodeId, nodeIntr := range cluster { + // skip if it is own node id + if nodeId == s.id { + continue + } + + // get the peer node address + metadata := nodeIntr.(map[string]interface{})["metadata"].(map[string]interface{}) + s.peerAddr = metadata["grpc_addr"].(string) + + s.logger.Printf("[INFO] peer node detected: %s", s.peerAddr) + + break } - }() + } + } + } + + // bootstrap node? + bootstrap := s.peerAddr == "" + s.logger.Printf("[INFO] bootstrap: %v", bootstrap) - b, err := ioutil.ReadAll(indexMappingFile) + // get index config from manager or peer + if s.managerAddr != "" { + mc, err := grpc.NewClient(s.managerAddr) + defer func() { + s.logger.Printf("[DEBUG] close client for %v", mc.GetAddress()) + err = mc.Close() if err != nil { - return nil, err + s.logger.Printf("[ERR] %v", err) + return } + }() + if err != nil { + s.logger.Printf("[ERR] %v", err) + return + } + + value, err := mc.GetState("index_config") + if err != nil { + s.logger.Printf("[ERR] %v", err) + return + } - err = json.Unmarshal(b, indexMapping) + if value != nil { + s.indexConfig = *value.(*map[string]interface{}) + } + } else if s.peerAddr != "" { + pc, err := grpc.NewClient(s.peerAddr) + defer func() { + err = pc.Close() if err != nil { - return nil, err + s.logger.Printf("[ERR] %v", err) + return } - } else if os.IsNotExist(err) { - return nil, err + }() + if err != nil { + s.logger.Printf("[ERR] %v", err) + return + } + + resp, err := pc.GetIndexConfig() + if err != nil { + s.logger.Printf("[ERR] %v", err) + return + } + + ins, err := protobuf.MarshalAny(resp.IndexConfig) + if err != nil { + s.logger.Printf("[ERR] %v", err) + return } - } - // create node information - server.node = &raft.Node{ - Id: nodeId, - BindAddr: bindAddr, - GrpcAddr: grpcAddr, - HttpAddr: httpAddr, - DataDir: dataDir, + s.indexConfig = *ins.(*map[string]interface{}) } + var err error + // create raft server - server.raftServer, err = NewRaftServer(server.node, server.bootstrap, indexMapping, indexStorageType, server.logger) + s.raftServer, err = NewRaftServer(s.id, s.metadata, bootstrap, s.indexConfig, s.logger) if err != nil { - return nil, err + s.logger.Printf("[ERR] %v", err) + return } // create gRPC service - server.grpcService, err = NewGRPCService(server.raftServer, server.logger) + s.grpcService, err = NewGRPCService(s.managerAddr, s.clusterId, s.raftServer, s.logger) if err != nil { - return nil, err + s.logger.Printf("[ERR] %v", err) + return } // create gRPC server - server.grpcServer, err = NewGRPCServer(grpcAddr, server.grpcService, server.logger) + s.grpcServer, err = grpc.NewServer(s.metadata["grpc_addr"].(string), s.grpcService, s.logger) if err != nil { - return nil, err + s.logger.Printf("[ERR] %v", err) + return } - // create gRPC client for HTTP server - server.grpcClient, err = NewGRPCClient(grpcAddr) + // create HTTP router + s.httpRouter, err = NewRouter(s.metadata["grpc_addr"].(string), s.logger) if err != nil { - return nil, err + s.logger.Printf("[ERR] %v", err) + return } // create HTTP server - server.httpServer, err = NewHTTPServer(httpAddr, server.grpcClient, server.logger, server.httpLogger) + s.httpServer, err = http.NewServer(s.metadata["http_addr"].(string), s.httpRouter, s.logger, s.httpLogger) if err != nil { - return nil, err + s.logger.Printf("[ERR] %v", err) + return } - return server, nil -} - -func (s *Server) Start() { // start Raft server + s.logger.Print("[INFO] start Raft server") + err = s.raftServer.Start() + if err != nil { + s.logger.Printf("[ERR] %v", err) + return + } + + // start gRPC service + s.logger.Print("[INFO] start gRPC service") go func() { - err := s.raftServer.Start() + err := s.grpcService.Start() if err != nil { s.logger.Printf("[ERR] %v", err) return } }() - s.logger.Print("[INFO] Raft server started") // start gRPC server + s.logger.Print("[INFO] start gRPC server") go func() { err := s.grpcServer.Start() if err != nil { @@ -146,21 +235,16 @@ func (s *Server) Start() { return } }() - s.logger.Print("[INFO] gRPC server started") // start HTTP server + s.logger.Print("[INFO] start HTTP server") go func() { - err := s.httpServer.Start() - if err != nil { - s.logger.Printf("[ERR] %v", err) - return - } + _ = s.httpServer.Start() }() - s.logger.Print("[INFO] HTTP server started") - if !s.bootstrap { - // create gRPC client - client, err := NewGRPCClient(s.joinAddr) + // join to the existing cluster + if !bootstrap { + client, err := grpc.NewClient(s.peerAddr) defer func() { err := client.Close() if err != nil { @@ -172,8 +256,7 @@ func (s *Server) Start() { return } - // join to the existing cluster - err = client.Join(s.node) + err = client.SetNode(s.id, s.metadata) if err != nil { s.logger.Printf("[ERR] %v", err) return @@ -183,24 +266,34 @@ func (s *Server) Start() { func (s *Server) Stop() { // stop HTTP server + s.logger.Printf("[INFO] stop HTTP server") err := s.httpServer.Stop() if err != nil { s.logger.Printf("[ERR] %v", err) } - // close gRPC client - err = s.grpcClient.Close() + // stop HTTP router + err = s.httpRouter.Close() if err != nil { s.logger.Printf("[ERR] %v", err) } // stop gRPC server + s.logger.Printf("[INFO] stop gRPC server") err = s.grpcServer.Stop() if err != nil { s.logger.Printf("[ERR] %v", err) } + // stop gRPC service + s.logger.Print("[INFO] stop gRPC service") + err = s.grpcService.Stop() + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + // stop Raft server + s.logger.Printf("[INFO] stop Raft server") err = s.raftServer.Stop() if err != nil { s.logger.Printf("[ERR] %v", err) diff --git a/manager/grpc_client.go b/manager/grpc_client.go deleted file mode 100644 index d72adec..0000000 --- a/manager/grpc_client.go +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright (c) 2019 Minoru Osuka -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package manager - -import ( - "context" - "errors" - "log" - "math" - - "github.com/golang/protobuf/ptypes/empty" - blasterrors "github.com/mosuka/blast/errors" - "github.com/mosuka/blast/protobuf/management" - "github.com/mosuka/blast/protobuf/raft" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" -) - -type GRPCClient struct { - ctx context.Context - cancel context.CancelFunc - conn *grpc.ClientConn - client management.ManagementClient - - logger *log.Logger -} - -func NewGRPCClient(address string) (*GRPCClient, error) { - baseCtx := context.TODO() - ctx, cancel := context.WithCancel(baseCtx) - - dialOpts := []grpc.DialOption{ - grpc.WithInsecure(), - grpc.WithDefaultCallOptions( - grpc.MaxCallSendMsgSize(math.MaxInt32), - grpc.MaxCallRecvMsgSize(math.MaxInt32), - ), - } - - conn, err := grpc.DialContext(ctx, address, dialOpts...) - if err != nil { - cancel() - return nil, err - } - - return &GRPCClient{ - ctx: ctx, - cancel: cancel, - conn: conn, - client: management.NewManagementClient(conn), - }, nil -} - -func (c *GRPCClient) Close() error { - c.cancel() - if c.conn != nil { - return c.conn.Close() - } - - return c.ctx.Err() -} - -func (c *GRPCClient) Join(req *raft.Node, opts ...grpc.CallOption) error { - _, err := c.client.Join(c.ctx, req, opts...) - if err != nil { - return err - } - - return nil -} - -func (c *GRPCClient) Leave(req *raft.Node, opts ...grpc.CallOption) error { - _, err := c.client.Leave(c.ctx, req, opts...) - if err != nil { - return err - } - - return nil -} - -func (c *GRPCClient) GetNode(opts ...grpc.CallOption) (*raft.Node, error) { - node, err := c.client.GetNode(c.ctx, &empty.Empty{}, opts...) - if err != nil { - st, _ := status.FromError(err) - - return nil, errors.New(st.Message()) - } - - return node, nil -} - -func (c *GRPCClient) GetCluster(opts ...grpc.CallOption) (*raft.Cluster, error) { - cluster, err := c.client.GetCluster(c.ctx, &empty.Empty{}, opts...) - if err != nil { - st, _ := status.FromError(err) - - return nil, errors.New(st.Message()) - } - - return cluster, nil -} - -func (c *GRPCClient) Snapshot(opts ...grpc.CallOption) error { - _, err := c.client.Snapshot(c.ctx, &empty.Empty{}) - if err != nil { - st, _ := status.FromError(err) - - return errors.New(st.Message()) - } - - return nil -} - -func (c *GRPCClient) Get(req *management.KeyValuePair, opts ...grpc.CallOption) (*management.KeyValuePair, error) { - resp, err := c.client.Get(c.ctx, req, opts...) - if err != nil { - st, _ := status.FromError(err) - - switch st.Code() { - case codes.NotFound: - return nil, blasterrors.ErrNotFound - default: - return nil, errors.New(st.Message()) - } - } - - return resp, nil -} - -func (c *GRPCClient) Set(req *management.KeyValuePair, opts ...grpc.CallOption) error { - _, err := c.client.Set(c.ctx, req, opts...) - if err != nil { - st, _ := status.FromError(err) - - switch st.Code() { - case codes.NotFound: - return blasterrors.ErrNotFound - default: - return errors.New(st.Message()) - } - } - - return nil -} - -func (c *GRPCClient) Delete(req *management.KeyValuePair, opts ...grpc.CallOption) error { - _, err := c.client.Delete(c.ctx, req, opts...) - if err != nil { - st, _ := status.FromError(err) - - switch st.Code() { - case codes.NotFound: - return blasterrors.ErrNotFound - default: - return errors.New(st.Message()) - } - } - - return nil -} diff --git a/manager/grpc_server.go b/manager/grpc_server.go deleted file mode 100644 index 39b36c4..0000000 --- a/manager/grpc_server.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) 2019 Minoru Osuka -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package manager - -import ( - "log" - "net" - - "github.com/mosuka/blast/protobuf/management" - "google.golang.org/grpc" -) - -type GRPCServer struct { - server *grpc.Server - listener net.Listener - - logger *log.Logger -} - -func NewGRPCServer(grpcAddr string, service *GRPCService, logger *log.Logger) (*GRPCServer, error) { - server := grpc.NewServer() - - management.RegisterManagementServer(server, service) - - listener, err := net.Listen("tcp", grpcAddr) - if err != nil { - return nil, err - } - - return &GRPCServer{ - server: server, - listener: listener, - logger: logger, - }, nil -} - -func (s *GRPCServer) Start() error { - err := s.server.Serve(s.listener) - if err != nil { - return err - } - - return nil -} - -func (s *GRPCServer) Stop() error { - s.server.GracefulStop() - - return nil -} diff --git a/manager/grpc_service.go b/manager/grpc_service.go index 7eda6d8..5726b96 100644 --- a/manager/grpc_service.go +++ b/manager/grpc_service.go @@ -16,160 +16,671 @@ package manager import ( "context" + "errors" "log" + "reflect" + "strings" + "sync" "time" + "github.com/golang/protobuf/ptypes/any" "github.com/golang/protobuf/ptypes/empty" - "github.com/mosuka/blast/protobuf/management" - "github.com/mosuka/blast/protobuf/raft" - "github.com/syndtr/goleveldb/leveldb/errors" + "github.com/hashicorp/raft" + blasterrors "github.com/mosuka/blast/errors" + "github.com/mosuka/blast/grpc" + "github.com/mosuka/blast/protobuf" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) type GRPCService struct { - raftServer *RaftServer + *grpc.Service - logger *log.Logger + raftServer *RaftServer + logger *log.Logger + + watchClusterStopCh chan struct{} + watchClusterDoneCh chan struct{} + peers map[string]interface{} + peerClients map[string]*grpc.Client + cluster map[string]interface{} + clusterChans map[chan protobuf.GetClusterResponse]struct{} + clusterMutex sync.RWMutex + + stateChans map[chan protobuf.WatchStateResponse]struct{} + stateMutex sync.RWMutex } func NewGRPCService(raftServer *RaftServer, logger *log.Logger) (*GRPCService, error) { return &GRPCService{ raftServer: raftServer, logger: logger, + + peers: make(map[string]interface{}, 0), + peerClients: make(map[string]*grpc.Client, 0), + cluster: make(map[string]interface{}, 0), + clusterChans: make(map[chan protobuf.GetClusterResponse]struct{}), + + stateChans: make(map[chan protobuf.WatchStateResponse]struct{}), }, nil } -func (s *GRPCService) Join(ctx context.Context, req *raft.Node) (*empty.Empty, error) { - s.logger.Printf("[INFO] %v", req) +func (s *GRPCService) Start() error { + s.logger.Print("[INFO] start update cluster") + go s.startUpdateCluster(500 * time.Millisecond) - resp := &empty.Empty{} + return nil +} + +func (s *GRPCService) Stop() error { + s.logger.Print("[INFO] stop update cluster") + s.stopUpdateCluster() + + return nil +} + +func (s *GRPCService) getLeaderClient() (*grpc.Client, error) { + var client *grpc.Client + + for id, node := range s.cluster { + state := node.(map[string]interface{})["state"].(string) + if state != raft.Shutdown.String() { + + if _, exist := s.peerClients[id]; exist { + client = s.peerClients[id] + break + } else { + s.logger.Printf("[DEBUG] %v does not exist", id) + } + } + } + + if client == nil { + return nil, errors.New("client does not exist") + } + + return client, nil +} + +func (s *GRPCService) startUpdateCluster(checkInterval time.Duration) { + s.watchClusterStopCh = make(chan struct{}) + s.watchClusterDoneCh = make(chan struct{}) + + s.logger.Printf("[INFO] start watching a cluster") + + defer func() { + close(s.watchClusterDoneCh) + }() + + ticker := time.NewTicker(checkInterval) + defer ticker.Stop() + + for { + select { + case <-s.watchClusterStopCh: + s.logger.Print("[DEBUG] receive request that stop watching a cluster") + return + case <-ticker.C: + // get servers + servers, err := s.raftServer.GetServers() + if err != nil { + s.logger.Printf("[ERR] %v", err) + return + } + + // create peer node list with out self node + peers := make(map[string]interface{}, 0) + for id, metadata := range servers { + if id != s.raftServer.id { + peers[id] = metadata + } + } + + if !reflect.DeepEqual(s.peers, peers) { + // open clients + for id, metadata := range peers { + grpcAddr := metadata.(map[string]interface{})["grpc_addr"].(string) + + if _, clientExists := s.peerClients[id]; clientExists { + client := s.peerClients[id] + if client.GetAddress() != grpcAddr { + s.logger.Printf("[DEBUG] close client for %s", client.GetAddress()) + err = client.Close() + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + + s.logger.Printf("[DEBUG] create client for %s", grpcAddr) + newClient, err := grpc.NewClient(grpcAddr) + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + + if client != nil { + s.logger.Printf("[DEBUG] create client for %s", newClient.GetAddress()) + s.peerClients[id] = newClient + } + } + } else { + s.logger.Printf("[DEBUG] create client for %s", grpcAddr) + newClient, err := grpc.NewClient(grpcAddr) + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + + s.peerClients[id] = newClient + } + } + + // close nonexistent clients + for id := range s.peers { + if _, peerExists := peers[id]; !peerExists { + if _, clientExists := s.peerClients[id]; clientExists { + client := s.peerClients[id] + + s.logger.Printf("[DEBUG] close client for %s", client.GetAddress()) + err = client.Close() + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + + delete(s.peerClients, id) + } + } + } + + // keep current peer nodes + s.peers = peers + } + + // get cluster + ctx, _ := grpc.NewContext() + resp, err := s.GetCluster(ctx, &empty.Empty{}) + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + ins, err := protobuf.MarshalAny(resp.Cluster) + cluster := *ins.(*map[string]interface{}) + + // notify current cluster + if !reflect.DeepEqual(s.cluster, cluster) { + for c := range s.clusterChans { + c <- *resp + } + + // keep current cluster + s.cluster = cluster + } + default: + time.Sleep(100 * time.Millisecond) + } + } +} + +func (s *GRPCService) stopUpdateCluster() { + // close clients + s.logger.Printf("[INFO] close peer clients") + for _, client := range s.peerClients { + s.logger.Printf("[DEBUG] close peer client for %s", client.GetAddress()) + err := client.Close() + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + } - err := s.raftServer.Join(req) + // stop watching a cluster + if s.watchClusterStopCh != nil { + s.logger.Printf("[INFO] stop watching a cluster") + close(s.watchClusterStopCh) + } + + // wait for stop watching a cluster has done + s.logger.Printf("[INFO] wait for stop watching a cluster has done") + <-s.watchClusterDoneCh +} + +func (s *GRPCService) getMetadata() (map[string]interface{}, error) { + metadata, err := s.raftServer.GetMetadata(s.raftServer.id) if err != nil { - return resp, status.Error(codes.Internal, err.Error()) + return nil, err } - return resp, nil + return metadata, nil } -func (s *GRPCService) Leave(ctx context.Context, req *raft.Node) (*empty.Empty, error) { - s.logger.Printf("[INFO] leave %v", req) +func (s *GRPCService) GetMetadata(ctx context.Context, req *protobuf.GetMetadataRequest) (*protobuf.GetMetadataResponse, error) { + resp := &protobuf.GetMetadataResponse{} - resp := &empty.Empty{} + var metadata map[string]interface{} + var err error + + if req.Id == "" || req.Id == s.raftServer.id { + metadata, err = s.getMetadata() + if err != nil { + s.logger.Printf("[ERR] %v: %v", err, s.raftServer.id) + } + } else { + if client, exist := s.peerClients[req.Id]; exist { + metadata, err = client.GetNodeMetadata(req.Id) + if err != nil { + s.logger.Printf("[ERR] %v: %v", err, req.Id) + } + } + } - err := s.raftServer.Leave(req) + metadataAny := &any.Any{} + err = protobuf.UnmarshalAny(metadata, metadataAny) if err != nil { return resp, status.Error(codes.Internal, err.Error()) } + resp.Metadata = metadataAny + return resp, nil } -func (s *GRPCService) GetNode(ctx context.Context, req *empty.Empty) (*raft.Node, error) { - s.logger.Printf("[INFO] get node %v", req) +func (s *GRPCService) getNodeState() string { + return s.raftServer.State() +} - resp := &raft.Node{} +func (s *GRPCService) GetNodeState(ctx context.Context, req *protobuf.GetNodeStateRequest) (*protobuf.GetNodeStateResponse, error) { + resp := &protobuf.GetNodeStateResponse{} + state := "" var err error - resp, err = s.raftServer.GetNode() - if err != nil { - return resp, status.Error(codes.Internal, err.Error()) + if req.Id == "" || req.Id == s.raftServer.id { + state = s.getNodeState() + } else { + if client, exist := s.peerClients[req.Id]; exist { + state, err = client.GetNodeState(req.Id) + if err != nil { + return resp, status.Error(codes.Internal, err.Error()) + } + } } + resp.State = state + return resp, nil } -func (s *GRPCService) GetCluster(ctx context.Context, req *empty.Empty) (*raft.Cluster, error) { - s.logger.Printf("[INFO] get cluster %v", req) +func (s *GRPCService) getNode() (map[string]interface{}, error) { + metadata, err := s.getMetadata() + if err != nil { + return nil, err + } + + state := s.getNodeState() + + node := map[string]interface{}{ + "metadata": metadata, + "state": state, + } + + return node, nil +} - resp := &raft.Cluster{} +func (s *GRPCService) GetNode(ctx context.Context, req *protobuf.GetNodeRequest) (*protobuf.GetNodeResponse, error) { + resp := &protobuf.GetNodeResponse{} + var metadata map[string]interface{} + var state string var err error - resp, err = s.raftServer.GetCluster() + if req.Id == "" || req.Id == s.raftServer.id { + node, err := s.getNode() + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + + metadata = node["metadata"].(map[string]interface{}) + + state = node["state"].(string) + } else { + if client, exist := s.peerClients[req.Id]; exist { + metadata, err = client.GetNodeMetadata(req.Id) + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + + state, err = client.GetNodeState(req.Id) + if err != nil { + s.logger.Printf("[ERR] %v", err) + state = raft.Shutdown.String() + } + } + } + + metadataAny := &any.Any{} + err = protobuf.UnmarshalAny(metadata, metadataAny) if err != nil { return resp, status.Error(codes.Internal, err.Error()) } + resp.Metadata = metadataAny + resp.State = state + return resp, nil } -func (s *GRPCService) Snapshot(ctx context.Context, req *empty.Empty) (*empty.Empty, error) { +func (s *GRPCService) SetNode(ctx context.Context, req *protobuf.SetNodeRequest) (*empty.Empty, error) { s.logger.Printf("[INFO] %v", req) resp := &empty.Empty{} - err := s.raftServer.Snapshot() + ins, err := protobuf.MarshalAny(req.Metadata) + if err != nil { + return resp, status.Error(codes.Internal, err.Error()) + } + + metadata := *ins.(*map[string]interface{}) + + if s.raftServer.IsLeader() { + err = s.raftServer.SetMetadata(req.Id, metadata) + if err != nil { + return resp, status.Error(codes.Internal, err.Error()) + } + } else { + // forward to leader + client, err := s.getLeaderClient() + if err != nil { + return resp, status.Error(codes.Internal, err.Error()) + } + err = client.SetNode(req.Id, metadata) + if err != nil { + return resp, status.Error(codes.Internal, err.Error()) + } + } + + return resp, nil +} + +func (s *GRPCService) DeleteNode(ctx context.Context, req *protobuf.DeleteNodeRequest) (*empty.Empty, error) { + s.logger.Printf("[INFO] leave %v", req) + + resp := &empty.Empty{} + + if s.raftServer.IsLeader() { + err := s.raftServer.DeleteMetadata(req.Id) + if err != nil { + return resp, status.Error(codes.Internal, err.Error()) + } + } else { + // forward to leader + client, err := s.getLeaderClient() + if err != nil { + return resp, status.Error(codes.Internal, err.Error()) + } + err = client.DeleteNode(req.Id) + if err != nil { + return resp, status.Error(codes.Internal, err.Error()) + } + } + + return resp, nil +} + +func (s *GRPCService) GetCluster(ctx context.Context, req *empty.Empty) (*protobuf.GetClusterResponse, error) { + resp := &protobuf.GetClusterResponse{} + + servers, err := s.raftServer.GetServers() if err != nil { return resp, status.Error(codes.Internal, err.Error()) } + cluster := map[string]interface{}{} + + for id := range servers { + node := map[string]interface{}{} + if id == s.raftServer.id { + node, err = s.getNode() + if err != nil { + return resp, err + } + } else { + r, err := s.GetNode(ctx, &protobuf.GetNodeRequest{Id: id}) + if err != nil { + return resp, err + } + + ins, err := protobuf.MarshalAny(r.Metadata) + metadata := *ins.(*map[string]interface{}) + + node = map[string]interface{}{ + "metadata": metadata, + "state": r.State, + } + } + + cluster[id] = node + } + + clusterAny := &any.Any{} + err = protobuf.UnmarshalAny(cluster, clusterAny) + if err != nil { + return resp, status.Error(codes.Internal, err.Error()) + } + + resp.Cluster = clusterAny + return resp, nil } -func (s *GRPCService) Get(ctx context.Context, req *management.KeyValuePair) (*management.KeyValuePair, error) { +func (s *GRPCService) WatchCluster(req *empty.Empty, server protobuf.Blast_WatchClusterServer) error { + chans := make(chan protobuf.GetClusterResponse) + + s.clusterMutex.Lock() + s.clusterChans[chans] = struct{}{} + s.clusterMutex.Unlock() + + defer func() { + s.clusterMutex.Lock() + delete(s.clusterChans, chans) + s.clusterMutex.Unlock() + close(chans) + }() + + for resp := range chans { + err := server.Send(&resp) + if err != nil { + return status.Error(codes.Internal, err.Error()) + } + } + + return nil +} + +func (s *GRPCService) Snapshot(ctx context.Context, req *empty.Empty) (*empty.Empty, error) { start := time.Now() - defer RecordMetrics(start, "get") + s.stateMutex.Lock() + defer func() { + s.stateMutex.Unlock() + RecordMetrics(start, "snapshot") + }() - s.logger.Printf("[INFO] get %v", req) + resp := &empty.Empty{} - resp := &management.KeyValuePair{} + err := s.raftServer.Snapshot() + if err != nil { + return resp, status.Error(codes.Internal, err.Error()) + } - var err error + return resp, nil +} + +func (s *GRPCService) GetState(ctx context.Context, req *protobuf.GetStateRequest) (*protobuf.GetStateResponse, error) { + start := time.Now() + s.stateMutex.RLock() + defer func() { + s.stateMutex.RUnlock() + RecordMetrics(start, "get") + }() + + resp := &protobuf.GetStateResponse{} - resp, err = s.raftServer.Get(req) + value, err := s.raftServer.GetState(req.Key) if err != nil { switch err { - case errors.ErrNotFound: + case blasterrors.ErrNotFound: return resp, status.Error(codes.NotFound, err.Error()) default: return resp, status.Error(codes.Internal, err.Error()) } } + valueAny := &any.Any{} + err = protobuf.UnmarshalAny(value, valueAny) + if err != nil { + return resp, status.Error(codes.Internal, err.Error()) + } + + resp.Value = valueAny + return resp, nil } -func (s *GRPCService) Set(ctx context.Context, req *management.KeyValuePair) (*empty.Empty, error) { +func (s *GRPCService) SetState(ctx context.Context, req *protobuf.SetStateRequest) (*empty.Empty, error) { start := time.Now() - defer RecordMetrics(start, "set") - - s.logger.Printf("[INFO] set %v", req) + s.stateMutex.Lock() + defer func() { + s.stateMutex.Unlock() + RecordMetrics(start, "set") + }() resp := &empty.Empty{} - err := s.raftServer.Set(req) + value, err := protobuf.MarshalAny(req.Value) if err != nil { - switch err { - case errors.ErrNotFound: - return resp, status.Error(codes.NotFound, err.Error()) - default: + return resp, status.Error(codes.Internal, err.Error()) + } + + if s.raftServer.IsLeader() { + err = s.raftServer.SetState(req.Key, value) + if err != nil { + switch err { + case blasterrors.ErrNotFound: + return resp, status.Error(codes.NotFound, err.Error()) + default: + return resp, status.Error(codes.Internal, err.Error()) + } + } + } else { + // forward to leader + client, err := s.getLeaderClient() + if err != nil { return resp, status.Error(codes.Internal, err.Error()) } + err = client.SetState(req.Key, value) + if err != nil { + return resp, status.Error(codes.Internal, err.Error()) + } + } + + // notify + for c := range s.stateChans { + c <- protobuf.WatchStateResponse{ + Command: protobuf.WatchStateResponse_SET, + Key: req.Key, + Value: req.Value, + } } return resp, nil } -func (s *GRPCService) Delete(ctx context.Context, req *management.KeyValuePair) (*empty.Empty, error) { +func (s *GRPCService) DeleteState(ctx context.Context, req *protobuf.DeleteStateRequest) (*empty.Empty, error) { start := time.Now() - defer RecordMetrics(start, "set") + s.stateMutex.Lock() + defer func() { + s.stateMutex.Unlock() + RecordMetrics(start, "delete") + }() s.logger.Printf("[INFO] set %v", req) resp := &empty.Empty{} - err := s.raftServer.Delete(req) - if err != nil { - switch err { - case errors.ErrNotFound: - return resp, status.Error(codes.NotFound, err.Error()) - default: + if s.raftServer.IsLeader() { + err := s.raftServer.DeleteState(req.Key) + if err != nil { + switch err { + case blasterrors.ErrNotFound: + return resp, status.Error(codes.NotFound, err.Error()) + default: + return resp, status.Error(codes.Internal, err.Error()) + } + } + } else { + // forward to leader + client, err := s.getLeaderClient() + if err != nil { return resp, status.Error(codes.Internal, err.Error()) } + err = client.DeleteState(req.Key) + if err != nil { + return resp, status.Error(codes.Internal, err.Error()) + } + } + + // notify + for c := range s.stateChans { + c <- protobuf.WatchStateResponse{ + Command: protobuf.WatchStateResponse_DELETE, + Key: req.Key, + } } return resp, nil } + +func (s *GRPCService) WatchState(req *protobuf.WatchStateRequest, server protobuf.Blast_WatchStateServer) error { + chans := make(chan protobuf.WatchStateResponse) + + s.stateMutex.Lock() + s.stateChans[chans] = struct{}{} + s.stateMutex.Unlock() + + defer func() { + s.stateMutex.Lock() + delete(s.stateChans, chans) + s.stateMutex.Unlock() + close(chans) + }() + + for resp := range chans { + if !strings.HasPrefix(resp.Key, req.Key) { + continue + } + err := server.Send(&resp) + if err != nil { + return status.Error(codes.Internal, err.Error()) + } + } + + return nil +} + +//func (s *GRPCService) GetDocument(ctx context.Context, req *protobuf.GetDocumentRequest) (*protobuf.GetDocumentResponse, error) { +// return &protobuf.GetDocumentResponse{}, status.Error(codes.Unavailable, "not implement") +//} + +//func (s *GRPCService) Search(ctx context.Context, req *protobuf.SearchRequest) (*protobuf.SearchResponse, error) { +// return &protobuf.SearchResponse{}, status.Error(codes.Unavailable, "not implement") +//} + +//func (s *GRPCService) IndexDocument(stream protobuf.Blast_IndexDocumentServer) error { +// return status.Error(codes.Unavailable, "not implement") +//} + +//func (s *GRPCService) DeleteDocument(stream protobuf.Blast_DeleteDocumentServer) error { +// return status.Error(codes.Unavailable, "not implement") +//} + +//func (s *GRPCService) GetIndexConfig(ctx context.Context, req *empty.Empty) (*protobuf.GetIndexConfigResponse, error) { +// return &protobuf.GetIndexConfigResponse{}, status.Error(codes.Unavailable, "not implement") +//} + +//func (s *GRPCService) GetIndexStats(ctx context.Context, req *empty.Empty) (*protobuf.GetIndexStatsResponse, error) { +// return &protobuf.GetIndexStatsResponse{}, status.Error(codes.Unavailable, "not implement") +//} diff --git a/manager/http_handler.go b/manager/http_router.go similarity index 69% rename from manager/http_handler.go rename to manager/http_router.go index 1b54ae8..ff41e7f 100644 --- a/manager/http_handler.go +++ b/manager/http_router.go @@ -16,21 +16,39 @@ package manager import ( "encoding/json" - "errors" "io/ioutil" "log" "net/http" "time" - "github.com/golang/protobuf/ptypes/any" "github.com/gorilla/mux" blasterrors "github.com/mosuka/blast/errors" + "github.com/mosuka/blast/grpc" blasthttp "github.com/mosuka/blast/http" - "github.com/mosuka/blast/protobuf" - "github.com/mosuka/blast/protobuf/management" "github.com/mosuka/blast/version" + "github.com/prometheus/client_golang/prometheus/promhttp" ) +func NewRouter(grpcAddr string, logger *log.Logger) (*blasthttp.Router, error) { + router, err := blasthttp.NewRouter(grpcAddr, logger) + if err != nil { + return nil, err + } + + router.StrictSlash(true) + + router.Handle("/", NewRootHandler(logger)).Methods("GET") + router.Handle("/configs", NewPutHandler(router.GRPCClient, logger)).Methods("PUT") + router.Handle("/configs", NewGetHandler(router.GRPCClient, logger)).Methods("GET") + router.Handle("/configs", NewDeleteHandler(router.GRPCClient, logger)).Methods("DELETE") + router.Handle("/configs/{path:.*}", NewPutHandler(router.GRPCClient, logger)).Methods("PUT") + router.Handle("/configs/{path:.*}", NewGetHandler(router.GRPCClient, logger)).Methods("GET") + router.Handle("/configs/{path:.*}", NewDeleteHandler(router.GRPCClient, logger)).Methods("DELETE") + router.Handle("/metrics", promhttp.Handler()).Methods("GET") + + return router, nil +} + type RootHandler struct { logger *log.Logger } @@ -62,11 +80,11 @@ func (h *RootHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } type GetHandler struct { - client *GRPCClient + client *grpc.Client logger *log.Logger } -func NewGetHandler(client *GRPCClient, logger *log.Logger) *GetHandler { +func NewGetHandler(client *grpc.Client, logger *log.Logger) *GetHandler { return &GetHandler{ client: client, logger: logger, @@ -84,13 +102,9 @@ func (h *GetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - key := "/" + vars["path"] + key := vars["path"] - kvp := &management.KeyValuePair{ - Key: key, - } - - retKVP, err := h.client.Get(kvp) + value, err := h.client.GetState(key) if err != nil { switch err { case blasterrors.ErrNotFound: @@ -112,42 +126,8 @@ func (h *GetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - // Any -> map[string]interface{} - valueInstance, err := protobuf.MarshalAny(retKVP.Value) - if err != nil { - httpStatus = http.StatusInternalServerError - - msgMap := map[string]interface{}{ - "message": err.Error(), - "status": httpStatus, - } - - content, err = blasthttp.NewJSONMessage(msgMap) - if err != nil { - h.logger.Printf("[ERR] %v", err) - } - - return - } - if valueInstance == nil { - httpStatus = http.StatusInternalServerError - - msgMap := map[string]interface{}{ - "message": errors.New("nil"), - "status": httpStatus, - } - - content, err = blasthttp.NewJSONMessage(msgMap) - if err != nil { - h.logger.Printf("[ERR] %v", err) - } - - return - } - valueMap := valueInstance.(*map[string]interface{}) - - // map[string]interface -> []byte - content, err = json.MarshalIndent(valueMap, "", " ") + // interface{} -> []byte + content, err = json.MarshalIndent(value, "", " ") if err != nil { httpStatus = http.StatusInternalServerError @@ -166,11 +146,11 @@ func (h *GetHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } type PutHandler struct { - client *GRPCClient + client *grpc.Client logger *log.Logger } -func NewPutHandler(client *GRPCClient, logger *log.Logger) *PutHandler { +func NewPutHandler(client *grpc.Client, logger *log.Logger) *PutHandler { return &PutHandler{ client: client, logger: logger, @@ -188,7 +168,7 @@ func (h *PutHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - key := "/" + vars["path"] + key := vars["path"] bodyBytes, err := ioutil.ReadAll(r.Body) if err != nil { @@ -208,8 +188,8 @@ func (h *PutHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } // string -> map[string]interface{} - var valueMap map[string]interface{} - err = json.Unmarshal(bodyBytes, &valueMap) + var value interface{} + err = json.Unmarshal(bodyBytes, &value) if err != nil { httpStatus = http.StatusBadRequest @@ -226,31 +206,7 @@ func (h *PutHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - // map[string]interface{} -> Any - valueAny := &any.Any{} - err = protobuf.UnmarshalAny(valueMap, valueAny) - if err != nil { - httpStatus = http.StatusInternalServerError - - msgMap := map[string]interface{}{ - "message": err.Error(), - "status": httpStatus, - } - - content, err = blasthttp.NewJSONMessage(msgMap) - if err != nil { - h.logger.Printf("[ERR] %v", err) - } - - return - } - - kvp := &management.KeyValuePair{ - Key: key, - Value: valueAny, - } - - err = h.client.Set(kvp) + err = h.client.SetState(key, value) if err != nil { httpStatus = http.StatusInternalServerError @@ -269,11 +225,11 @@ func (h *PutHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } type DeleteHandler struct { - client *GRPCClient + client *grpc.Client logger *log.Logger } -func NewDeleteHandler(client *GRPCClient, logger *log.Logger) *DeleteHandler { +func NewDeleteHandler(client *grpc.Client, logger *log.Logger) *DeleteHandler { return &DeleteHandler{ client: client, logger: logger, @@ -291,13 +247,9 @@ func (h *DeleteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - key := "/" + vars["path"] - - kvp := &management.KeyValuePair{ - Key: key, - } + key := vars["path"] - err := h.client.Delete(kvp) + err := h.client.DeleteState(key) if err != nil { httpStatus = http.StatusInternalServerError diff --git a/manager/metric.go b/manager/metric.go index 9d7ca52..17a326e 100644 --- a/manager/metric.go +++ b/manager/metric.go @@ -22,7 +22,7 @@ import ( var ( namespace = "blast" - subsystem = "federation" + subsystem = "manager" DurationSeconds = prometheus.NewHistogramVec( prometheus.HistogramOpts{ diff --git a/protobuf/raft/raft.proto b/manager/raft_command.go similarity index 56% rename from protobuf/raft/raft.proto rename to manager/raft_command.go index 0d7ba75..97fa3df 100644 --- a/protobuf/raft/raft.proto +++ b/manager/raft_command.go @@ -12,22 +12,32 @@ // See the License for the specific language governing permissions and // limitations under the License. -syntax = "proto3"; +package manager -package raft; +import "encoding/json" -option go_package = "github.com/mosuka/blast/protobuf/raft"; +type command int -message Node { - string id = 1; - string bind_addr = 2; - string grpc_addr = 3; - string http_addr = 4; - bool leader = 5; - string data_dir = 6; +const ( + unknown command = iota + setNode + deleteNode + setKeyValue + deleteKeyValue +) + +type message struct { + Command command `json:"command,omitempty"` + Data json.RawMessage `json:"data,omitempty"` } -message Cluster { - string id = 1; - repeated Node nodes = 2; +func newMessage(cmd command, data interface{}) (*message, error) { + b, err := json.Marshal(data) + if err != nil { + return nil, err + } + return &message{ + Command: cmd, + Data: b, + }, nil } diff --git a/manager/raft_fsm.go b/manager/raft_fsm.go index 7ff1c91..4110348 100644 --- a/manager/raft_fsm.go +++ b/manager/raft_fsm.go @@ -21,99 +21,139 @@ import ( "io/ioutil" "log" - "github.com/golang/protobuf/proto" "github.com/hashicorp/raft" blasterrors "github.com/mosuka/blast/errors" - "github.com/mosuka/blast/protobuf" - "github.com/mosuka/blast/protobuf/management" - pbraft "github.com/mosuka/blast/protobuf/raft" - "github.com/mosuka/maputils" + "github.com/mosuka/blast/maputils" ) type RaftFSM struct { - metadata map[string]*pbraft.Node + metadata maputils.Map - federation map[string]interface{} + path string + + data maputils.Map logger *log.Logger } func NewRaftFSM(path string, logger *log.Logger) (*RaftFSM, error) { return &RaftFSM{ - metadata: make(map[string]*pbraft.Node, 0), - federation: make(map[string]interface{}, 0), - logger: logger, + path: path, + logger: logger, }, nil } -func (f *RaftFSM) Close() error { +func (f *RaftFSM) Start() error { + f.logger.Print("[INFO] initialize data") + f.metadata = maputils.Map{} + f.data = maputils.Map{} + return nil } -func (f *RaftFSM) GetMetadata(nodeId string) (*pbraft.Node, error) { - node, exists := f.metadata[nodeId] - if !exists { - return nil, blasterrors.ErrNotFound - } - if node == nil { - return nil, errors.New("nil") +func (f *RaftFSM) Stop() error { + return nil +} + +func (f *RaftFSM) GetMetadata(id string) (map[string]interface{}, error) { + value, err := f.metadata.Get(id) + if err != nil { + return nil, err } - value := node - return value, nil + return value.(maputils.Map).ToMap(), nil } -func (f *RaftFSM) applySetMetadata(nodeId string, node *pbraft.Node) interface{} { - f.metadata[nodeId] = node +func (f *RaftFSM) applySetMetadata(id string, value map[string]interface{}) interface{} { + err := f.metadata.Merge(id, value) + if err != nil { + return err + } return nil } -func (f *RaftFSM) applyDeleteMetadata(nodeId string) interface{} { - _, exists := f.metadata[nodeId] - if exists { - delete(f.metadata, nodeId) +func (f *RaftFSM) applyDeleteMetadata(id string) interface{} { + err := f.metadata.Delete(id) + if err != nil { + return err } return nil } func (f *RaftFSM) Get(key string) (interface{}, error) { - nm, err := maputils.NewNestedMap(f.federation) + value, err := f.data.Get(key) if err != nil { - return nil, err + switch err { + case maputils.ErrNotFound: + return nil, blasterrors.ErrNotFound + default: + return nil, err + } } - value, err := nm.Get(key) - if err == maputils.ErrNotFound { - return nil, blasterrors.ErrNotFound + var ret interface{} + switch value.(type) { + case maputils.Map: + ret = value.(maputils.Map).ToMap() + default: + ret = value } - return value, nil + return ret, nil } -func (f *RaftFSM) applySet(key string, value interface{}) interface{} { - nm, err := maputils.NewNestedMap(f.federation) - if err != nil { - return err - } - - err = nm.Set(key, value) - if err != nil { - return err +func (f *RaftFSM) applySet(key string, value interface{}, merge bool) interface{} { + if merge { + err := f.data.Merge(key, value) + if err != nil { + return err + } + } else { + err := f.data.Set(key, value) + if err != nil { + return err + } } return nil } -func (f *RaftFSM) applyDelete(key string) interface{} { - nm, err := maputils.NewNestedMap(f.federation) - if err != nil { - return err +func (f *RaftFSM) delete(keys []string, data interface{}) (interface{}, error) { + var err error + + if len(keys) >= 1 { + key := keys[0] + + switch data.(type) { + case map[string]interface{}: + if _, exist := data.(map[string]interface{})[key]; exist { + if len(keys) > 1 { + data.(map[string]interface{})[key], err = f.delete(keys[1:], data.(map[string]interface{})[key]) + if err != nil { + return nil, err + } + } else { + mm := data.(map[string]interface{}) + delete(mm, key) + data = mm + } + } else { + return nil, blasterrors.ErrNotFound + } + } } - err = nm.Delete(key) + return data, nil +} + +func (f *RaftFSM) applyDelete(key string) interface{} { + err := f.data.Delete(key) if err != nil { + if err == maputils.ErrNotFound { + return blasterrors.ErrNotFound + } return err } @@ -121,79 +161,58 @@ func (f *RaftFSM) applyDelete(key string) interface{} { } func (f *RaftFSM) Apply(l *raft.Log) interface{} { - var c management.ManagementCommand - err := proto.Unmarshal(l.Data, &c) + var msg message + err := json.Unmarshal(l.Data, &msg) if err != nil { return err } - f.logger.Printf("[DEBUG] Apply %v", c) - - switch c.Type { - case management.ManagementCommand_SET_METADATA: - // Any -> raft.Node - nodeInstance, err := protobuf.MarshalAny(c.Data) + switch msg.Command { + case setNode: + var data map[string]interface{} + err := json.Unmarshal(msg.Data, &data) if err != nil { return err } - if nodeInstance == nil { - return errors.New("nil") - } - node := nodeInstance.(*pbraft.Node) - - return f.applySetMetadata(node.Id, node) - case management.ManagementCommand_DELETE_METADATA: - // Any -> raft.Node - nodeInstance, err := protobuf.MarshalAny(c.Data) + return f.applySetMetadata(data["id"].(string), data["metadata"].(map[string]interface{})) + case deleteNode: + var data map[string]interface{} + err := json.Unmarshal(msg.Data, &data) if err != nil { return err } - if nodeInstance == nil { - return errors.New("nil") - } - node := *nodeInstance.(*pbraft.Node) - - return f.applyDeleteMetadata(node.Id) - case management.ManagementCommand_PUT_KEY_VALUE_PAIR: - // Any -> federation.Node - kvpInstance, err := protobuf.MarshalAny(c.Data) + return f.applyDeleteMetadata(data["id"].(string)) + case setKeyValue: + var data map[string]interface{} + err := json.Unmarshal(msg.Data, &data) if err != nil { return err } - if kvpInstance == nil { - return errors.New("nil") - } - kvp := kvpInstance.(*management.KeyValuePair) - - // Any -> interface{} - value, err := protobuf.MarshalAny(kvp.Value) - return f.applySet(kvp.Key, value) - case management.ManagementCommand_DELETE_KEY_VALUE_PAIR: - // Any -> federation.Node - kvpInstance, err := protobuf.MarshalAny(c.Data) + return f.applySet(data["key"].(string), data["value"], true) + case deleteKeyValue: + var data map[string]interface{} + err := json.Unmarshal(msg.Data, &data) if err != nil { return err } - if kvpInstance == nil { - return errors.New("nil") - } - kvp := *kvpInstance.(*management.KeyValuePair) - return f.applyDelete(kvp.Key) + return f.applyDelete(data["key"].(string)) default: return errors.New("command type not support") } } func (f *RaftFSM) Snapshot() (raft.FSMSnapshot, error) { - return &KVSFSMSnapshot{ - federation: f.federation, - logger: f.logger, + return &RaftFSMSnapshot{ + data: f.data, + logger: f.logger, }, nil } func (f *RaftFSM) Restore(rc io.ReadCloser) error { + f.logger.Print("[INFO] restore data") + defer func() { err := rc.Close() if err != nil { @@ -207,26 +226,22 @@ func (f *RaftFSM) Restore(rc io.ReadCloser) error { return err } - err = json.Unmarshal(data, &f.federation) + err = json.Unmarshal(data, &f.data) if err != nil { f.logger.Printf("[ERR] %v", err) return err } - f.logger.Printf("[INFO] federation was restored: %v", f.federation) - return nil } -// --------------------- - -type KVSFSMSnapshot struct { - federation map[string]interface{} - logger *log.Logger +type RaftFSMSnapshot struct { + data maputils.Map + logger *log.Logger } -func (f *KVSFSMSnapshot) Persist(sink raft.SnapshotSink) error { - f.logger.Printf("[INFO] start data persistence") +func (f *RaftFSMSnapshot) Persist(sink raft.SnapshotSink) error { + f.logger.Printf("[INFO] persist data") defer func() { err := sink.Close() @@ -235,7 +250,7 @@ func (f *KVSFSMSnapshot) Persist(sink raft.SnapshotSink) error { } }() - buff, err := json.Marshal(f.federation) + buff, err := json.Marshal(f.data) if err != nil { return err } @@ -245,11 +260,9 @@ func (f *KVSFSMSnapshot) Persist(sink raft.SnapshotSink) error { return err } - f.logger.Printf("[INFO] federation was persisted: %v", f.federation) - return nil } -func (f *KVSFSMSnapshot) Release() { +func (f *RaftFSMSnapshot) Release() { f.logger.Printf("[INFO] release") } diff --git a/manager/raft_fsm_test.go b/manager/raft_fsm_test.go new file mode 100644 index 0000000..f6ee819 --- /dev/null +++ b/manager/raft_fsm_test.go @@ -0,0 +1,476 @@ +// Copyright (c) 2019 Minoru Osuka +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package manager + +import ( + "io/ioutil" + "log" + "os" + "reflect" + "testing" +) + +func TestRaftFSM_GetNode(t *testing.T) { + tmp, err := ioutil.TempDir("", "") + if err != nil { + t.Errorf("%v", err) + } + defer func() { + err := os.RemoveAll(tmp) + if err != nil { + t.Errorf("%v", err) + } + }() + + logger := log.New(os.Stderr, "", 0) + + fsm, err := NewRaftFSM(tmp, logger) + if err != nil { + t.Errorf("%v", err) + } + err = fsm.Start() + defer func() { + err := fsm.Stop() + if err != nil { + t.Errorf("%v", err) + } + }() + if err != nil { + t.Errorf("%v", err) + } + + fsm.applySetMetadata("node1", map[string]interface{}{ + "bind_addr": ":16060", + "grpc_addr": ":17070", + "http_addr": ":18080", + }) + fsm.applySetMetadata("node2", map[string]interface{}{ + "bind_addr": ":16061", + "grpc_addr": ":17071", + "http_addr": ":18081", + }) + fsm.applySetMetadata("node3", map[string]interface{}{ + "bind_addr": ":16062", + "grpc_addr": ":17072", + "http_addr": ":18082", + }) + + val1, err := fsm.GetMetadata("node2") + if err != nil { + t.Errorf("%v", err) + } + + exp1 := map[string]interface{}{ + "bind_addr": ":16061", + "grpc_addr": ":17071", + "http_addr": ":18081", + } + act1 := val1 + if !reflect.DeepEqual(exp1, act1) { + t.Errorf("expected content to see %v, saw %v", exp1, act1) + } + +} + +func TestRaftFSM_SetNode(t *testing.T) { + tmp, err := ioutil.TempDir("", "") + if err != nil { + t.Errorf("%v", err) + } + defer func() { + err := os.RemoveAll(tmp) + if err != nil { + t.Errorf("%v", err) + } + }() + + logger := log.New(os.Stderr, "", 0) + + fsm, err := NewRaftFSM(tmp, logger) + if err != nil { + t.Errorf("%v", err) + } + err = fsm.Start() + defer func() { + err := fsm.Stop() + if err != nil { + t.Errorf("%v", err) + } + }() + if err != nil { + t.Errorf("%v", err) + } + + fsm.applySetMetadata("node1", map[string]interface{}{ + "bind_addr": ":16060", + "grpc_addr": ":17070", + "http_addr": ":18080", + }) + fsm.applySetMetadata("node2", map[string]interface{}{ + "bind_addr": ":16061", + "grpc_addr": ":17071", + "http_addr": ":18081", + }) + fsm.applySetMetadata("node3", map[string]interface{}{ + "bind_addr": ":16062", + "grpc_addr": ":17072", + "http_addr": ":18082", + }) + + val1, err := fsm.GetMetadata("node2") + if err != nil { + t.Errorf("%v", err) + } + exp1 := map[string]interface{}{ + "bind_addr": ":16061", + "grpc_addr": ":17071", + "http_addr": ":18081", + } + act1 := val1 + if !reflect.DeepEqual(exp1, act1) { + t.Errorf("expected content to see %v, saw %v", exp1, act1) + } + + fsm.applySetMetadata("node2", map[string]interface{}{ + "bind_addr": ":16061", + "grpc_addr": ":17071", + "http_addr": ":18081", + "leader": true, + }) + + val2, err := fsm.GetMetadata("node2") + if err != nil { + t.Errorf("%v", err) + } + exp2 := map[string]interface{}{ + "bind_addr": ":16061", + "grpc_addr": ":17071", + "http_addr": ":18081", + "leader": true, + } + act2 := val2 + if !reflect.DeepEqual(exp2, act2) { + t.Errorf("expected content to see %v, saw %v", exp2, act2) + } +} + +func TestRaftFSM_DeleteNode(t *testing.T) { + tmp, err := ioutil.TempDir("", "") + if err != nil { + t.Errorf("%v", err) + } + defer func() { + err := os.RemoveAll(tmp) + if err != nil { + t.Errorf("%v", err) + } + }() + + logger := log.New(os.Stderr, "", 0) + + fsm, err := NewRaftFSM(tmp, logger) + if err != nil { + t.Errorf("%v", err) + } + err = fsm.Start() + defer func() { + err := fsm.Stop() + if err != nil { + t.Errorf("%v", err) + } + }() + if err != nil { + t.Errorf("%v", err) + } + + fsm.applySetMetadata("node1", map[string]interface{}{ + "bind_addr": ":16060", + "grpc_addr": ":17070", + "http_addr": ":18080", + }) + fsm.applySetMetadata("node2", map[string]interface{}{ + "bind_addr": ":16061", + "grpc_addr": ":17071", + "http_addr": ":18081", + }) + fsm.applySetMetadata("node3", map[string]interface{}{ + "bind_addr": ":16062", + "grpc_addr": ":17072", + "http_addr": ":18082", + }) + + val1, err := fsm.GetMetadata("node2") + if err != nil { + t.Errorf("%v", err) + } + exp1 := map[string]interface{}{ + "bind_addr": ":16061", + "grpc_addr": ":17071", + "http_addr": ":18081", + } + act1 := val1 + if !reflect.DeepEqual(exp1, act1) { + t.Errorf("expected content to see %v, saw %v", exp1, act1) + } + + fsm.applyDeleteMetadata("node2") + + val2, err := fsm.GetMetadata("node2") + if err == nil { + t.Errorf("expected error: %v", err) + } + + act1 = val2 + if reflect.DeepEqual(nil, act1) { + t.Errorf("expected content to see nil, saw %v", act1) + } +} + +func TestRaftFSM_Get(t *testing.T) { + tmp, err := ioutil.TempDir("", "") + if err != nil { + t.Errorf("%v", err) + } + defer func() { + err := os.RemoveAll(tmp) + if err != nil { + t.Errorf("%v", err) + } + }() + + logger := log.New(os.Stderr, "", 0) + + fsm, err := NewRaftFSM(tmp, logger) + if err != nil { + t.Errorf("%v", err) + } + err = fsm.Start() + defer func() { + err := fsm.Stop() + if err != nil { + t.Errorf("%v", err) + } + }() + if err != nil { + t.Errorf("%v", err) + } + + fsm.applySet("/", map[string]interface{}{"a": 1}, false) + + value, err := fsm.Get("/a") + if err != nil { + t.Errorf("%v", err) + } + + expectedValue := 1 + actualValue := value + if expectedValue != actualValue { + t.Errorf("expected content to see %v, saw %v", expectedValue, actualValue) + } +} + +func TestRaftFSM_Set(t *testing.T) { + tmp, err := ioutil.TempDir("", "") + if err != nil { + t.Errorf("%v", err) + } + defer func() { + err := os.RemoveAll(tmp) + if err != nil { + t.Errorf("%v", err) + } + }() + + logger := log.New(os.Stderr, "", 0) + + fsm, err := NewRaftFSM(tmp, logger) + if err != nil { + t.Errorf("%v", err) + } + err = fsm.Start() + defer func() { + err := fsm.Stop() + if err != nil { + t.Errorf("%v", err) + } + }() + if err != nil { + t.Errorf("%v", err) + } + + // set {"a": 1} + fsm.applySet("/", map[string]interface{}{ + "a": 1, + }, false) + val1, err := fsm.Get("/") + if err != nil { + t.Errorf("%v", err) + } + exp1 := map[string]interface{}{ + "a": 1, + } + act1 := val1 + if !reflect.DeepEqual(exp1, act1) { + t.Errorf("expected content to see %v, saw %v", exp1, act1) + } + + // merge {"a": "A"} + fsm.applySet("/", map[string]interface{}{ + "a": "A", + }, true) + val2, err := fsm.Get("/") + if err != nil { + t.Errorf("%v", err) + } + exp2 := map[string]interface{}{ + "a": "A", + } + act2 := val2 + if !reflect.DeepEqual(exp2, act2) { + t.Errorf("expected content to see %v, saw %v", exp2, act2) + } + + // set {"a": {"b": "AB"}} + fsm.applySet("/", map[string]interface{}{ + "a": map[string]interface{}{ + "b": "AB", + }, + }, false) + val3, err := fsm.Get("/") + if err != nil { + t.Errorf("%v", err) + } + exp3 := map[string]interface{}{ + "a": map[string]interface{}{ + "b": "AB", + }, + } + act3 := val3 + if !reflect.DeepEqual(exp3, act3) { + t.Errorf("expected content to see %v, saw %v", exp3, act3) + } + + // merge {"a": {"c": "AC"}} + fsm.applySet("/", map[string]interface{}{ + "a": map[string]interface{}{ + "c": "AC", + }, + }, true) + val4, err := fsm.Get("/") + if err != nil { + t.Errorf("%v", err) + } + exp4 := map[string]interface{}{ + "a": map[string]interface{}{ + "b": "AB", + "c": "AC", + }, + } + act4 := val4 + if !reflect.DeepEqual(exp4, act4) { + t.Errorf("expected content to see %v, saw %v", exp4, act4) + } + + // set {"a": 1} + fsm.applySet("/", map[string]interface{}{ + "a": 1, + }, false) + val5, err := fsm.Get("/") + if err != nil { + t.Errorf("%v", err) + } + exp5 := map[string]interface{}{ + "a": 1, + } + act5 := val5 + if !reflect.DeepEqual(exp5, act5) { + t.Errorf("expected content to see %v, saw %v", exp5, act5) + } + + // TODO: merge {"a": {"c": "AC"}} + //fsm.applySet("/", map[string]interface{}{ + // "a": map[string]interface{}{ + // "c": "AC", + // }, + //}, true) + //val6, err := fsm.Get("/") + //if err != nil { + // t.Errorf("%v", err) + //} + //exp6 := map[string]interface{}{ + // "a": map[string]interface{}{ + // "c": "AC", + // }, + //} + //act6 := val6 + //if !reflect.DeepEqual(exp6, act6) { + // t.Errorf("expected content to see %v, saw %v", exp6, act6) + //} +} + +func TestRaftFSM_Delete(t *testing.T) { + tmp, err := ioutil.TempDir("", "") + if err != nil { + t.Errorf("%v", err) + } + defer func() { + err := os.RemoveAll(tmp) + if err != nil { + t.Errorf("%v", err) + } + }() + + logger := log.New(os.Stderr, "", 0) + + fsm, err := NewRaftFSM(tmp, logger) + if err != nil { + t.Errorf("%v", err) + } + err = fsm.Start() + defer func() { + err := fsm.Stop() + if err != nil { + t.Errorf("%v", err) + } + }() + if err != nil { + t.Errorf("%v", err) + } + + fsm.applySet("/", map[string]interface{}{"a": 1}, false) + + value, err := fsm.Get("/a") + if err != nil { + t.Errorf("%v", err) + } + + expectedValue := 1 + actualValue := value + if expectedValue != actualValue { + t.Errorf("expected content to see %v, saw %v", expectedValue, actualValue) + } + + fsm.applyDelete("/a") + + value, err = fsm.Get("/a") + if err == nil { + t.Errorf("expected nil: %v", err) + } + + actualValue = value + if nil != actualValue { + t.Errorf("expected content to see %v, saw %v", expectedValue, actualValue) + } +} diff --git a/manager/raft_server.go b/manager/raft_server.go index 6ecc7be..75037e0 100644 --- a/manager/raft_server.go +++ b/manager/raft_server.go @@ -15,84 +15,99 @@ package manager import ( + "encoding/json" "log" "net" "path/filepath" + "sync" "time" - "github.com/golang/protobuf/proto" - "github.com/golang/protobuf/ptypes/any" "github.com/hashicorp/raft" raftboltdb "github.com/hashicorp/raft-boltdb" + _ "github.com/mosuka/blast/config" "github.com/mosuka/blast/errors" - "github.com/mosuka/blast/protobuf" - "github.com/mosuka/blast/protobuf/management" - blastraft "github.com/mosuka/blast/protobuf/raft" ) type RaftServer struct { - Node *blastraft.Node - bootstrap bool + id string + metadata map[string]interface{} - BindAddr string - DataDir string + bootstrap bool raft *raft.Raft fsm *RaftFSM + indexConfig map[string]interface{} + logger *log.Logger + mu sync.RWMutex } -func NewRaftServer(node *blastraft.Node, bootstrap bool, logger *log.Logger) (*RaftServer, error) { - fsm, err := NewRaftFSM(filepath.Join(node.DataDir, "kvs"), logger) - if err != nil { - return nil, err - } - +func NewRaftServer(id string, metadata map[string]interface{}, bootstrap bool, indexConfig map[string]interface{}, logger *log.Logger) (*RaftServer, error) { return &RaftServer{ - Node: node, + id: id, + metadata: metadata, + bootstrap: bootstrap, - fsm: fsm, - logger: logger, + + indexConfig: indexConfig, + logger: logger, }, nil } func (s *RaftServer) Start() error { + var err error + + s.logger.Print("[INFO] create finite state machine") + s.fsm, err = NewRaftFSM(filepath.Join(s.metadata["data_dir"].(string), "store"), s.logger) + if err != nil { + return err + } + + s.logger.Print("[INFO] start finite state machine") + err = s.fsm.Start() + if err != nil { + return err + } + + s.logger.Print("[INFO] initialize Raft") config := raft.DefaultConfig() - config.LocalID = raft.ServerID(s.Node.Id) + config.LocalID = raft.ServerID(s.id) config.SnapshotThreshold = 1024 config.Logger = s.logger - addr, err := net.ResolveTCPAddr("tcp", s.Node.BindAddr) + addr, err := net.ResolveTCPAddr("tcp", s.metadata["bind_addr"].(string)) if err != nil { return err } // create transport - transport, err := raft.NewTCPTransportWithLogger(s.Node.BindAddr, addr, 3, 10*time.Second, s.logger) + transport, err := raft.NewTCPTransportWithLogger(s.metadata["bind_addr"].(string), addr, 3, 10*time.Second, s.logger) if err != nil { return err } // create snapshot store - snapshotStore, err := raft.NewFileSnapshotStoreWithLogger(s.Node.DataDir, 2, s.logger) + snapshotStore, err := raft.NewFileSnapshotStoreWithLogger(s.metadata["data_dir"].(string), 2, s.logger) if err != nil { return err } // create raft log store - raftLogStore, err := raftboltdb.NewBoltStore(filepath.Join(s.Node.DataDir, "raft.db")) + raftLogStore, err := raftboltdb.NewBoltStore(filepath.Join(s.metadata["data_dir"].(string), "raft.db")) if err != nil { return err } // create raft + s.logger.Print("[INFO] start Raft") s.raft, err = raft.NewRaft(config, s.fsm, raftLogStore, raftLogStore, snapshotStore, transport) if err != nil { return err } if s.bootstrap { + s.logger.Print("[INFO] configure Raft as bootstrap") configuration := raft.Configuration{ Servers: []raft.Server{ { @@ -103,7 +118,8 @@ func (s *RaftServer) Start() error { } s.raft.BootstrapCluster(configuration) - // wait for detect a leader + // wait for become a leader + s.logger.Print("[INFO] wait for become a leader") err = s.WaitForDetectLeader(60 * time.Second) if err != nil { if err == errors.ErrTimeout { @@ -115,7 +131,16 @@ func (s *RaftServer) Start() error { } // set metadata - err = s.setMetadata(s.Node.Id, s.Node) + s.logger.Print("[INFO] register itself in a cluster") + err = s.setMetadata(s.id, s.metadata) + if err != nil { + s.logger.Printf("[ERR] %v", err) + return nil + } + + // set index config + s.logger.Print("[INFO] register index config") + err = s.setIndexConfig(s.indexConfig) if err != nil { s.logger.Printf("[ERR] %v", err) return nil @@ -126,34 +151,20 @@ func (s *RaftServer) Start() error { } func (s *RaftServer) Stop() error { - err := s.fsm.Close() + s.logger.Print("[INFO] shutdown Raft") + f := s.raft.Shutdown() + err := f.Error() if err != nil { return err } - return nil -} - -func (s *RaftServer) WaitForDetectLeader(timeout time.Duration) error { - ticker := time.NewTicker(1000 * time.Millisecond) - defer ticker.Stop() - timer := time.NewTimer(timeout) - defer timer.Stop() - - for { - select { - case <-ticker.C: - leaderAddr := s.raft.Leader() - if leaderAddr != "" { - s.logger.Printf("[INFO] detected %v as a leader", leaderAddr) - return nil - } else { - s.logger.Printf("[WARN] %v", errors.ErrNotFoundLeader) - } - case <-timer.C: - return errors.ErrTimeout - } + s.logger.Print("[INFO] stop finite state machine") + err = s.fsm.Stop() + if err != nil { + return err } + + return nil } func (s *RaftServer) LeaderAddress(timeout time.Duration) (raft.ServerAddress, error) { @@ -176,13 +187,13 @@ func (s *RaftServer) LeaderAddress(timeout time.Duration) (raft.ServerAddress, e } func (s *RaftServer) LeaderID(timeout time.Duration) (raft.ServerID, error) { - cf := s.raft.GetConfiguration() - err := cf.Error() + leaderAddr, err := s.LeaderAddress(timeout) if err != nil { return "", err } - leaderAddr, err := s.LeaderAddress(timeout) + cf := s.raft.GetConfiguration() + err = cf.Error() if err != nil { return "", err } @@ -196,34 +207,54 @@ func (s *RaftServer) LeaderID(timeout time.Duration) (raft.ServerID, error) { return "", errors.ErrNotFoundLeader } -func (s *RaftServer) getMetadata(nodeId string) (*blastraft.Node, error) { - node, err := s.fsm.GetMetadata(nodeId) +func (s *RaftServer) Stats() map[string]string { + return s.raft.Stats() +} + +func (s *RaftServer) State() string { + return s.raft.State().String() +} + +func (s *RaftServer) IsLeader() bool { + return s.raft.State() == raft.Leader +} + +func (s *RaftServer) WaitForDetectLeader(timeout time.Duration) error { + _, err := s.LeaderAddress(timeout) if err != nil { - return nil, err + return err } - return node, nil + return nil } -func (s *RaftServer) setMetadata(nodeId string, node *blastraft.Node) error { - // Node -> Any - nodeAny := &any.Any{} - err := protobuf.UnmarshalAny(node, nodeAny) +func (s *RaftServer) getMetadata(id string) (map[string]interface{}, error) { + metadata, err := s.fsm.GetMetadata(id) if err != nil { - return err + return nil, err } - c := &management.ManagementCommand{ - Type: management.ManagementCommand_SET_METADATA, - Data: nodeAny, + return metadata, nil +} + +func (s *RaftServer) setMetadata(id string, metadata map[string]interface{}) error { + msg, err := newMessage( + setNode, + map[string]interface{}{ + "id": id, + "metadata": metadata, + }, + ) + if err != nil { + return err } - msg, err := proto.Marshal(c) + msgBytes, err := json.Marshal(msg) if err != nil { return err } - f := s.raft.Apply(msg, 10*time.Second) + f := s.raft.Apply(msgBytes, 10*time.Second) err = f.Error() if err != nil { return err @@ -232,29 +263,23 @@ func (s *RaftServer) setMetadata(nodeId string, node *blastraft.Node) error { return nil } -func (s *RaftServer) deleteMetadata(nodeId string) error { - node := &blastraft.Node{ - Id: nodeId, - } - - // Node -> Any - nodeAny := &any.Any{} - err := protobuf.UnmarshalAny(node, nodeAny) +func (s *RaftServer) deleteMetadata(id string) error { + msg, err := newMessage( + deleteNode, + map[string]interface{}{ + "id": id, + }, + ) if err != nil { return err } - c := &management.ManagementCommand{ - Type: management.ManagementCommand_DELETE_METADATA, - Data: nodeAny, - } - - msg, err := proto.Marshal(c) + msgBytes, err := json.Marshal(msg) if err != nil { return err } - f := s.raft.Apply(msg, 10*time.Second) + f := s.raft.Apply(msgBytes, 10*time.Second) err = f.Error() if err != nil { return err @@ -263,39 +288,39 @@ func (s *RaftServer) deleteMetadata(nodeId string) error { return nil } -func (s *RaftServer) Join(node *blastraft.Node) error { - if s.raft.State() != raft.Leader { - // forward to leader node - leaderId, err := s.LeaderID(60 * time.Second) - if err != nil { - return err - } +func (s *RaftServer) setIndexConfig(indexConfig map[string]interface{}) error { + err := s.SetState("index_config", indexConfig) + if err != nil { + return err + } - node, err := s.getMetadata(string(leaderId)) - if err != nil { - s.logger.Printf("[ERR] %v", err) - return nil - } + return nil +} + +func (s *RaftServer) GetMetadata(id string) (map[string]interface{}, error) { + cf := s.raft.GetConfiguration() + err := cf.Error() + if err != nil { + return nil, err + } - client, err := NewGRPCClient(string(node.GrpcAddr)) - defer func() { - err := client.Close() + var metadata map[string]interface{} + for _, server := range cf.Configuration().Servers { + if server.ID == raft.ServerID(id) { + metadata, err = s.getMetadata(id) if err != nil { - s.logger.Printf("[ERR] %v", err) + return nil, err } - }() - if err != nil { - s.logger.Printf("[ERR] %v", err) - return nil + break } + } - err = client.Join(node) - if err != nil { - s.logger.Printf("[ERR] %v", err) - return nil - } + return metadata, nil +} - return nil +func (s *RaftServer) SetMetadata(id string, metadata map[string]interface{}) error { + if !s.IsLeader() { + return raft.ErrNotLeader } cf := s.raft.GetConfiguration() @@ -305,62 +330,32 @@ func (s *RaftServer) Join(node *blastraft.Node) error { } for _, server := range cf.Configuration().Servers { - if server.ID == raft.ServerID(node.Id) { - s.logger.Printf("[INFO] node %s already joined the cluster", node.Id) + if server.ID == raft.ServerID(id) { + s.logger.Printf("[INFO] node %v already joined the cluster", id) return nil } } - f := s.raft.AddVoter(raft.ServerID(node.Id), raft.ServerAddress(node.BindAddr), 0, 0) + f := s.raft.AddVoter(raft.ServerID(id), raft.ServerAddress(metadata["bind_addr"].(string)), 0, 0) err = f.Error() if err != nil { return err } // set metadata - err = s.setMetadata(node.Id, node) + err = s.setMetadata(id, metadata) if err != nil { s.logger.Printf("[ERR] %v", err) return nil } - s.logger.Printf("[INFO] node %s at %s joined successfully", node.Id, node.BindAddr) + s.logger.Printf("[INFO] node %v joined successfully", id) return nil } -func (s *RaftServer) Leave(node *blastraft.Node) error { - if s.raft.State() != raft.Leader { - // forward to leader node - leaderId, err := s.LeaderID(60 * time.Second) - if err != nil { - return err - } - - node, err := s.getMetadata(string(leaderId)) - if err != nil { - s.logger.Printf("[ERR] %v", err) - return nil - } - - client, err := NewGRPCClient(string(node.GrpcAddr)) - defer func() { - err := client.Close() - if err != nil { - s.logger.Printf("[ERR] %v", err) - } - }() - if err != nil { - s.logger.Printf("[ERR] %v", err) - return nil - } - - err = client.Leave(node) - if err != nil { - s.logger.Printf("[ERR] %v", err) - return nil - } - - return nil +func (s *RaftServer) DeleteMetadata(id string) error { + if !s.IsLeader() { + return raft.ErrNotLeader } cf := s.raft.GetConfiguration() @@ -370,97 +365,48 @@ func (s *RaftServer) Leave(node *blastraft.Node) error { } for _, server := range cf.Configuration().Servers { - if server.ID == raft.ServerID(node.Id) { + if server.ID == raft.ServerID(id) { f := s.raft.RemoveServer(server.ID, 0, 0) err = f.Error() if err != nil { return err } - s.logger.Printf("[INFO] node %s leaved successfully", node.Id) + s.logger.Printf("[INFO] node %v leaved successfully", id) return nil } } // delete metadata - err = s.deleteMetadata(node.Id) + err = s.deleteMetadata(id) if err != nil { s.logger.Printf("[ERR] %v", err) return nil } - s.logger.Printf("[INFO] node %s does not exists in the cluster", node.Id) + s.logger.Printf("[INFO] node %v does not exists in the cluster", id) return nil } -func (s *RaftServer) GetNode() (*blastraft.Node, error) { - cf := s.raft.GetConfiguration() - err := cf.Error() - if err != nil { - return nil, err - } - - leaderAddr, err := s.LeaderAddress(60 * time.Second) - if err != nil { - return nil, err - } - - node := &blastraft.Node{} - for _, server := range cf.Configuration().Servers { - if server.ID == raft.ServerID(s.Node.Id) { - node.Id = string(server.ID) - node.BindAddr = string(server.Address) - node.Leader = server.Address == leaderAddr - - nodeInfo, err := s.getMetadata(node.Id) - if err != nil { - s.logger.Printf("[WARN] %v", err) - break - } - node.GrpcAddr = nodeInfo.GrpcAddr - node.HttpAddr = nodeInfo.HttpAddr - node.DataDir = nodeInfo.DataDir - break - } - } - - return node, nil -} - -func (s *RaftServer) GetCluster() (*blastraft.Cluster, error) { +func (s *RaftServer) GetServers() (map[string]interface{}, error) { cf := s.raft.GetConfiguration() err := cf.Error() if err != nil { return nil, err } - leaderAddr, err := s.LeaderAddress(60 * time.Second) - if err != nil { - return nil, err - } - - nodes := make([]*blastraft.Node, 0) + servers := map[string]interface{}{} for _, server := range cf.Configuration().Servers { - node := &blastraft.Node{} - node.Id = string(server.ID) - node.BindAddr = string(server.Address) - node.Leader = server.Address == leaderAddr - - nodeInfo, err := s.getMetadata(node.Id) + metadata, err := s.GetMetadata(string(server.ID)) if err != nil { - s.logger.Printf("[WARN] %v", err) + // could not get metadata continue } - node.GrpcAddr = nodeInfo.GrpcAddr - node.HttpAddr = nodeInfo.HttpAddr - node.DataDir = nodeInfo.DataDir - nodes = append(nodes, node) + servers[string(server.ID)] = metadata } - return &blastraft.Cluster{ - Nodes: nodes, - }, nil + return servers, nil } func (s *RaftServer) Snapshot() error { @@ -473,82 +419,37 @@ func (s *RaftServer) Snapshot() error { return nil } -func (s *RaftServer) Get(kvp *management.KeyValuePair) (*management.KeyValuePair, error) { - value, err := s.fsm.Get(kvp.Key) +func (s *RaftServer) GetState(key string) (interface{}, error) { + value, err := s.fsm.Get(key) if err != nil { return nil, err } - s.logger.Printf("[DEBUG] %v", value) - - // map[string]interface{} -> Any - valueAny := &any.Any{} - err = protobuf.UnmarshalAny(value, valueAny) - if err != nil { - return nil, err - } - - retKVP := &management.KeyValuePair{ - Key: kvp.Key, - Value: valueAny, - } - - return retKVP, nil + return value, nil } -func (s *RaftServer) Set(kvp *management.KeyValuePair) error { - if s.raft.State() != raft.Leader { - // forward to leader node - leaderId, err := s.LeaderID(60 * time.Second) - if err != nil { - return err - } - - node, err := s.getMetadata(string(leaderId)) - if err != nil { - s.logger.Printf("[ERR] %v", err) - return err - } - - client, err := NewGRPCClient(string(node.GrpcAddr)) - defer func() { - err := client.Close() - if err != nil { - s.logger.Printf("[ERR] %v", err) - } - }() - if err != nil { - s.logger.Printf("[ERR] %v", err) - return err - } - - err = client.Set(kvp) - if err != nil { - s.logger.Printf("[ERR] %v", err) - return err - } - - return nil +func (s *RaftServer) SetState(key string, value interface{}) error { + if !s.IsLeader() { + return raft.ErrNotLeader } - // KeyValuePair -> Any - kvpAny := &any.Any{} - err := protobuf.UnmarshalAny(kvp, kvpAny) + msg, err := newMessage( + setKeyValue, + map[string]interface{}{ + "key": key, + "value": value, + }, + ) if err != nil { return err } - c := &management.ManagementCommand{ - Type: management.ManagementCommand_PUT_KEY_VALUE_PAIR, - Data: kvpAny, - } - - msg, err := proto.Marshal(c) + msgBytes, err := json.Marshal(msg) if err != nil { return err } - f := s.raft.Apply(msg, 10*time.Second) + f := s.raft.Apply(msgBytes, 10*time.Second) err = f.Error() if err != nil { return err @@ -557,59 +458,27 @@ func (s *RaftServer) Set(kvp *management.KeyValuePair) error { return nil } -func (s *RaftServer) Delete(kvp *management.KeyValuePair) error { - if s.raft.State() != raft.Leader { - // forward to leader node - leaderId, err := s.LeaderID(60 * time.Second) - if err != nil { - return err - } - - node, err := s.getMetadata(string(leaderId)) - if err != nil { - s.logger.Printf("[ERR] %v", err) - return err - } - - client, err := NewGRPCClient(string(node.GrpcAddr)) - defer func() { - err := client.Close() - if err != nil { - s.logger.Printf("[ERR] %v", err) - } - }() - if err != nil { - s.logger.Printf("[ERR] %v", err) - return err - } - - err = client.Delete(kvp) - if err != nil { - s.logger.Printf("[ERR] %v", err) - return err - } - - return nil +func (s *RaftServer) DeleteState(key string) error { + if !s.IsLeader() { + return raft.ErrNotLeader } - // KeyValuePair -> Any - kvpAny := &any.Any{} - err := protobuf.UnmarshalAny(kvp, kvpAny) + msg, err := newMessage( + deleteKeyValue, + map[string]interface{}{ + "key": key, + }, + ) if err != nil { return err } - c := &management.ManagementCommand{ - Type: management.ManagementCommand_DELETE_KEY_VALUE_PAIR, - Data: kvpAny, - } - - msg, err := proto.Marshal(c) + msgBytes, err := json.Marshal(msg) if err != nil { return err } - f := s.raft.Apply(msg, 10*time.Second) + f := s.raft.Apply(msgBytes, 10*time.Second) err = f.Error() if err != nil { return err diff --git a/manager/server.go b/manager/server.go index 0175ec6..ef979bf 100644 --- a/manager/server.go +++ b/manager/server.go @@ -18,90 +18,101 @@ import ( "log" accesslog "github.com/mash/go-accesslog" - "github.com/mosuka/blast/protobuf/raft" + "github.com/mosuka/blast/grpc" + "github.com/mosuka/blast/http" ) type Server struct { - node *raft.Node - bootstrap bool - joinAddr string + id string + metadata map[string]interface{} - raftServer *RaftServer + peerAddr string - grpcService *GRPCService - grpcServer *GRPCServer - grpcClient *GRPCClient + indexConfig map[string]interface{} - httpServer *HTTPServer + raftServer *RaftServer + grpcService *GRPCService + grpcServer *grpc.Server + httpRouter *http.Router + httpServer *http.Server logger *log.Logger httpLogger accesslog.Logger } -func NewServer(nodeId string, bindAddr string, grpcAddr string, httpAddr string, dataDir string, joinAddr string, logger *log.Logger, httpLogger accesslog.Logger) (*Server, error) { - var err error +func NewServer(id string, metadata map[string]interface{}, peerAddr string, indexConfig map[string]interface{}, logger *log.Logger, httpLogger accesslog.Logger) (*Server, error) { + return &Server{ + id: id, + metadata: metadata, + peerAddr: peerAddr, + indexConfig: indexConfig, + logger: logger, + httpLogger: httpLogger, + }, nil +} - server := &Server{ - bootstrap: joinAddr == "", - joinAddr: joinAddr, - logger: logger, - httpLogger: httpLogger, - } +func (s *Server) Start() { + var err error - // create node information - server.node = &raft.Node{ - Id: nodeId, - BindAddr: bindAddr, - GrpcAddr: grpcAddr, - HttpAddr: httpAddr, - DataDir: dataDir, - } + // bootstrap node? + bootstrap := s.peerAddr == "" + s.logger.Printf("[INFO] bootstrap: %v", bootstrap) // create raft server - server.raftServer, err = NewRaftServer(server.node, server.bootstrap, server.logger) + s.raftServer, err = NewRaftServer(s.id, s.metadata, bootstrap, s.indexConfig, s.logger) if err != nil { - return nil, err + s.logger.Printf("[ERR] %v", err) + return } // create gRPC service - server.grpcService, err = NewGRPCService(server.raftServer, server.logger) + s.grpcService, err = NewGRPCService(s.raftServer, s.logger) if err != nil { - return nil, err + s.logger.Printf("[ERR] %v", err) + return } // create gRPC server - server.grpcServer, err = NewGRPCServer(grpcAddr, server.grpcService, server.logger) + s.grpcServer, err = grpc.NewServer(s.metadata["grpc_addr"].(string), s.grpcService, s.logger) if err != nil { - return nil, err + s.logger.Printf("[ERR] %v", err) + return } - // create gRPC client for HTTP server - server.grpcClient, err = NewGRPCClient(grpcAddr) + // create HTTP router + s.httpRouter, err = NewRouter(s.metadata["grpc_addr"].(string), s.logger) if err != nil { - return nil, err + s.logger.Printf("[ERR] %v", err) + return } // create HTTP server - server.httpServer, err = NewHTTPServer(httpAddr, server.grpcClient, server.logger, server.httpLogger) + s.httpServer, err = http.NewServer(s.metadata["http_addr"].(string), s.httpRouter, s.logger, s.httpLogger) if err != nil { - return nil, err + s.logger.Printf("[ERR] %v", err) + return } - return server, nil -} - -func (s *Server) Start() { // start Raft server + s.logger.Print("[INFO] start Raft server") + err = s.raftServer.Start() + if err != nil { + s.logger.Printf("[ERR] %v", err) + return + } + + // start gRPC service + s.logger.Print("[INFO] start gRPC service") go func() { - err := s.raftServer.Start() + err := s.grpcService.Start() if err != nil { s.logger.Printf("[ERR] %v", err) return } }() - s.logger.Print("[INFO] Raft server started") // start gRPC server + s.logger.Print("[INFO] start gRPC server") go func() { err := s.grpcServer.Start() if err != nil { @@ -109,21 +120,16 @@ func (s *Server) Start() { return } }() - s.logger.Print("[INFO] gRPC server started") // start HTTP server + s.logger.Print("[INFO] start HTTP server") go func() { - err := s.httpServer.Start() - if err != nil { - s.logger.Printf("[ERR] %v", err) - return - } + _ = s.httpServer.Start() }() - s.logger.Print("[INFO] HTTP server started") - if !s.bootstrap { - // create gRPC client - client, err := NewGRPCClient(s.joinAddr) + // join to the existing cluster + if !bootstrap { + client, err := grpc.NewClient(s.peerAddr) defer func() { err := client.Close() if err != nil { @@ -135,8 +141,7 @@ func (s *Server) Start() { return } - // join to the existing cluster - err = client.Join(s.node) + err = client.SetNode(s.id, s.metadata) if err != nil { s.logger.Printf("[ERR] %v", err) return @@ -146,24 +151,34 @@ func (s *Server) Start() { func (s *Server) Stop() { // stop HTTP server + s.logger.Print("[INFO] stop HTTP server") err := s.httpServer.Stop() if err != nil { s.logger.Printf("[ERR] %v", err) } - // close gRPC client - err = s.grpcClient.Close() + // stop HTTP router + err = s.httpRouter.Close() if err != nil { s.logger.Printf("[ERR] %v", err) } // stop gRPC server + s.logger.Print("[INFO] stop gRPC server") err = s.grpcServer.Stop() if err != nil { s.logger.Printf("[ERR] %v", err) } + // stop gRPC service + s.logger.Print("[INFO] stop gRPC service") + err = s.grpcService.Stop() + if err != nil { + s.logger.Printf("[ERR] %v", err) + } + // stop Raft server + s.logger.Print("[INFO] stop Raft server") err = s.raftServer.Stop() if err != nil { s.logger.Printf("[ERR] %v", err) diff --git a/maputils/error.go b/maputils/error.go new file mode 100644 index 0000000..8de64bb --- /dev/null +++ b/maputils/error.go @@ -0,0 +1,7 @@ +package maputils + +import "errors" + +var ( + ErrNotFound = errors.New("not found") +) diff --git a/maputils/map.go b/maputils/map.go new file mode 100644 index 0000000..265ca37 --- /dev/null +++ b/maputils/map.go @@ -0,0 +1,283 @@ +package maputils + +import ( + "encoding/json" + "errors" + "strings" + + "github.com/imdario/mergo" + "github.com/stretchr/objx" + yaml "gopkg.in/yaml.v2" +) + +type Map map[string]interface{} + +func New() Map { + return Map{} +} + +func FromMap(src map[string]interface{}) Map { + return normalize(src).(Map) +} + +func FromJSON(src []byte) (Map, error) { + m := map[string]interface{}{} + err := json.Unmarshal(src, &m) + if err != nil { + return nil, err + } + + return FromMap(m), nil +} + +func FromYAML(src []byte) (Map, error) { + m := map[string]interface{}{} + err := yaml.Unmarshal(src, &m) + if err != nil { + return nil, err + } + + return FromMap(m), nil +} + +func splitKey(path string) []string { + keys := make([]string, 0) + for _, k := range strings.Split(path, "/") { + if k != "" { + keys = append(keys, k) + } + } + + return keys +} + +func makeSelector(key string) string { + return strings.Join(splitKey(key), objx.PathSeparator) +} + +func normalize(value interface{}) interface{} { + switch value.(type) { + case map[string]interface{}: + ret := Map{} + for k, v := range value.(map[string]interface{}) { + ret[k] = normalize(v) + } + return ret + case map[interface{}]interface{}: // when unmarshaled by yaml + ret := Map{} + for k, v := range value.(map[interface{}]interface{}) { + ret[k.(string)] = normalize(v) + } + return ret + case []interface{}: + ret := make([]interface{}, 0) + for _, v := range value.([]interface{}) { + ret = append(ret, normalize(v)) + } + return ret + case bool, string, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, uintptr, float32, float64, complex64, complex128: + return value + default: + return value + } +} + +func makeMap(path string, value interface{}) interface{} { + var ret interface{} + + keys := splitKey(path) + + if len(keys) >= 1 { + ret = Map{keys[0]: makeMap(strings.Join(keys[1:], "/"), value)} + } else if len(keys) == 0 { + ret = normalize(value) + } + + return ret +} + +func (m Map) Has(key string) (bool, error) { + _, err := m.Get(key) + if err != nil { + return false, err + } + + return true, nil +} + +func (m Map) Set(key string, value interface{}) error { + _ = m.Delete(key) + + err := m.Merge(key, value) + if err != nil { + return err + } + + return nil +} + +func (m Map) Merge(key string, value interface{}) error { + mm := makeMap(key, value).(Map) + + err := mergo.Merge(&m, mm, mergo.WithOverride) + if err != nil { + return err + } + + return nil +} + +func (m Map) Get(key string) (interface{}, error) { + var tmpMap interface{} + + tmpMap = m + + keys := splitKey(key) + + if len(keys) <= 0 { + return tmpMap.(Map).ToMap(), nil + } + + iter := newIterator(splitKey(key)) + var value interface{} + for { + k, err := iter.value() + if err != nil { + return nil, err + } + + if _, ok := tmpMap.(Map)[k]; !ok { + return nil, ErrNotFound + } + + if iter.hasNext() { + tmpMap = tmpMap.(Map)[k] + iter.next() + } else { + value = tmpMap.(Map)[k] + break + } + } + + return value, nil +} + +func (m Map) Delete(key string) error { + var tmpMap interface{} + + tmpMap = m + + keys := splitKey(key) + + if len(keys) <= 0 { + // clear map + err := m.Clear() + if err != nil { + return err + } + return nil + } + + iter := newIterator(splitKey(key)) + for { + k, err := iter.value() + if err != nil { + return err + } + + if _, ok := tmpMap.(Map)[k]; !ok { + return ErrNotFound + } + + if iter.hasNext() { + tmpMap = tmpMap.(Map)[k] + iter.next() + } else { + delete(tmpMap.(Map), k) + break + } + } + + return nil +} + +func (m Map) Clear() error { + for k := range m { + delete(m, k) + } + + return nil +} + +func (m Map) toMap(value interface{}) interface{} { + switch value.(type) { + case Map: + ret := map[string]interface{}{} + for k, v := range value.(Map) { + ret[k] = m.toMap(v) + } + return ret + case []interface{}: + ret := make([]interface{}, 0) + for _, v := range value.([]interface{}) { + ret = append(ret, m.toMap(v)) + } + return ret + case bool, string, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, uintptr, float32, float64, complex64, complex128: + return value + default: + return value + } +} + +func (m Map) ToMap() map[string]interface{} { + return m.toMap(m).(map[string]interface{}) +} + +func (m Map) ToJSON() ([]byte, error) { + mm := m.ToMap() + b, err := json.Marshal(&mm) + if err != nil { + return nil, err + } + + return b, nil +} + +func (m Map) ToYAML() ([]byte, error) { + mm := m.ToMap() + b, err := yaml.Marshal(&mm) + if err != nil { + return nil, err + } + + return b, nil +} + +type iterator struct { + keys []string + pos int +} + +func newIterator(keys []string) *iterator { + return &iterator{ + keys: keys, + pos: 0, + } +} + +func (i *iterator) hasNext() bool { + return i.pos < len(i.keys)-1 +} + +func (i *iterator) next() bool { + i.pos++ + return i.pos < len(i.keys)-1 +} + +func (i *iterator) value() (string, error) { + if i.pos > len(i.keys)-1 { + return "", errors.New("value is not valid after iterator finished") + } + return i.keys[i.pos], nil +} diff --git a/maputils/map_test.go b/maputils/map_test.go new file mode 100644 index 0000000..0691e90 --- /dev/null +++ b/maputils/map_test.go @@ -0,0 +1,665 @@ +package maputils + +import ( + "bytes" + "reflect" + "testing" +) + +func Test_splitKey(t *testing.T) { + key1 := "/a/b/c/d" + keys1 := splitKey(key1) + exp1 := []string{"a", "b", "c", "d"} + act1 := keys1 + if !reflect.DeepEqual(exp1, act1) { + t.Errorf("expected content to see %v, saw %v", exp1, act1) + } + + key2 := "/" + keys2 := splitKey(key2) + exp2 := make([]string, 0) + act2 := keys2 + if !reflect.DeepEqual(exp2, act2) { + t.Errorf("expected content to see %v, saw %v", exp2, act2) + } + + key3 := "" + keys3 := splitKey(key3) + exp3 := make([]string, 0) + act3 := keys3 + if !reflect.DeepEqual(exp3, act3) { + t.Errorf("expected content to see %v, saw %v", exp3, act3) + } +} + +func Test_makeSelector(t *testing.T) { + key1 := "/a/b/c/d" + selector1 := makeSelector(key1) + exp1 := "a.b.c.d" + act1 := selector1 + if exp1 != act1 { + t.Errorf("expected content to see %v, saw %v", exp1, act1) + } + + key2 := "/" + selector2 := makeSelector(key2) + exp2 := "" + act2 := selector2 + if exp2 != act2 { + t.Errorf("expected content to see %v, saw %v", exp2, act2) + } + + key3 := "" + selector3 := makeSelector(key3) + exp3 := "" + act3 := selector3 + if exp3 != act3 { + t.Errorf("expected content to see %v, saw %v", exp3, act3) + } +} + +func Test_normalize(t *testing.T) { + data1 := map[string]interface{}{ + "a": map[string]interface{}{ + "b": map[string]interface{}{ + "c": "abc", + "d": "abd", + }, + "e": []interface{}{ + "ae1", + "ae2", + }, + }, + } + val1 := normalize(data1) + exp1 := Map{ + "a": Map{ + "b": Map{ + "c": "abc", + "d": "abd", + }, + "e": []interface{}{ + "ae1", + "ae2", + }, + }, + } + act1 := val1 + if !reflect.DeepEqual(exp1, act1) { + t.Errorf("expected content to see %v, saw %v", exp1, act1) + } +} + +func Test_makeMap(t *testing.T) { + val1 := makeMap("/a/b/c", "C").(Map) + exp1 := Map{ + "a": Map{ + "b": Map{ + "c": "C", + }, + }, + } + act1 := val1 + if !reflect.DeepEqual(exp1, act1) { + t.Errorf("expected content to see %v, saw %v", exp1, act1) + } + + val2 := makeMap("a/b", map[string]interface{}{"c": "C"}).(Map) + exp2 := Map{ + "a": Map{ + "b": Map{ + "c": "C", + }, + }, + } + act2 := val2 + if !reflect.DeepEqual(exp2, act2) { + t.Errorf("expected content to see %v, saw %v", exp2, act2) + } +} + +func Test_Has(t *testing.T) { + map1 := Map{ + "a": Map{ + "b": Map{ + "c": "abc", + "d": "abd", + }, + "e": []interface{}{ + "ae1", + "ae2", + }, + }, + } + + val1, err := map1.Has("a/b/c") + if err != nil { + t.Errorf("%v", err) + } + exp1 := true + act1 := val1 + if exp1 != act1 { + t.Errorf("expected content to see %v, saw %v", exp1, act1) + } + + val2, err := map1.Get("a/b/f") + if err != ErrNotFound { + t.Errorf("%v", err) + } + exp2 := false + act2 := val2 + if exp2 == act2 { + t.Errorf("expected content to see %v, saw %v", exp2, act2) + } +} + +func Test_Set(t *testing.T) { + map1 := Map{} + + err := map1.Set("/", Map{"a": "A"}) + if err != nil { + t.Errorf("%v", err) + } + exp1 := Map{ + "a": "A", + } + act1 := map1 + if !reflect.DeepEqual(exp1, act1) { + t.Errorf("expected content to see %v, saw %v", exp1, act1) + } + + err = map1.Set("/", Map{"A": "a"}) + if err != nil { + t.Errorf("%v", err) + } + exp2 := Map{ + "A": "a", + } + act2 := map1 + if !reflect.DeepEqual(exp2, act2) { + t.Errorf("expected content to see %v, saw %v", exp2, act2) + } + + err = map1.Set("/", Map{"A": 1}) + if err != nil { + t.Errorf("%v", err) + } + exp3 := Map{ + "A": 1, + } + act3 := map1 + if !reflect.DeepEqual(exp3, act3) { + t.Errorf("expected content to see %v, saw %v", exp2, act2) + } + + err = map1.Set("/A", "AAA") + if err != nil { + t.Errorf("%v", err) + } + exp4 := Map{ + "A": "AAA", + } + act4 := map1 + if !reflect.DeepEqual(exp4, act4) { + t.Errorf("expected content to see %v, saw %v", exp4, act4) + } + + err = map1.Set("/B", "BBB") + if err != nil { + t.Errorf("%v", err) + } + exp5 := Map{ + "A": "AAA", + "B": "BBB", + } + act5 := map1 + if !reflect.DeepEqual(exp5, act5) { + t.Errorf("expected content to see %v, saw %v", exp5, act5) + } + + err = map1.Set("/C", map[string]interface{}{"D": "CCC-DDD"}) + if err != nil { + t.Errorf("%v", err) + } + exp6 := Map{ + "A": "AAA", + "B": "BBB", + "C": Map{ + "D": "CCC-DDD", + }, + } + act6 := map1 + if !reflect.DeepEqual(exp6, act6) { + t.Errorf("expected content to see %v, saw %v", exp6, act6) + } +} + +func Test_Merge(t *testing.T) { + map1 := Map{} + + err := map1.Merge("/", Map{"a": "A"}) + if err != nil { + t.Errorf("%v", err) + } + exp1 := Map{ + "a": "A", + } + act1 := map1 + if !reflect.DeepEqual(exp1, act1) { + t.Errorf("expected content to see %v, saw %v", exp1, act1) + } + + err = map1.Merge("/a", "a") + if err != nil { + t.Errorf("%v", err) + } + exp2 := Map{ + "a": "a", + } + act2 := map1 + if !reflect.DeepEqual(exp2, act2) { + t.Errorf("expected content to see %v, saw %v", exp2, act2) + } + + err = map1.Merge("/", Map{"a": 1}) + if err != nil { + t.Errorf("%v", err) + } + exp3 := Map{ + "a": 1, + } + act3 := map1 + if !reflect.DeepEqual(exp3, act3) { + t.Errorf("expected content to see %v, saw %v", exp3, act3) + } + + err = map1.Merge("/", Map{"b": 2}) + if err != nil { + t.Errorf("%v", err) + } + exp4 := Map{ + "a": 1, + "b": 2, + } + act4 := map1 + if !reflect.DeepEqual(exp4, act4) { + t.Errorf("expected content to see %v, saw %v", exp4, act4) + } + + err = map1.Merge("/c", 3) + if err != nil { + t.Errorf("%v", err) + } + exp5 := Map{ + "a": 1, + "b": 2, + "c": 3, + } + act5 := map1 + if !reflect.DeepEqual(exp5, act5) { + t.Errorf("expected content to see %v, saw %v", exp5, act5) + } + +} + +func Test_Get(t *testing.T) { + map1 := Map{ + "a": Map{ + "b": Map{ + "c": "abc", + "d": "abd", + }, + "e": []interface{}{ + "ae1", + "ae2", + }, + }, + } + + val1, err := map1.Get("a/b/c") + if err != nil { + t.Errorf("%v", err) + } + exp1 := "abc" + act1 := val1 + if !reflect.DeepEqual(exp1, act1) { + t.Errorf("expected content to see %v, saw %v", exp1, act1) + } + + val2, err := map1.Get("a") + if err != nil { + t.Errorf("%v", err) + } + exp2 := Map{ + "b": Map{ + "c": "abc", + "d": "abd", + }, + "e": []interface{}{ + "ae1", + "ae2", + }, + } + act2 := val2 + if !reflect.DeepEqual(exp2, act2) { + t.Errorf("expected content to see %v, saw %v", exp2, act2) + } +} + +func Test_Delete(t *testing.T) { + map1 := Map{ + "a": Map{ + "b": Map{ + "c": "abc", + "d": "abd", + }, + "e": []interface{}{ + "ae1", + "ae2", + }, + }, + } + + err := map1.Delete("a/b/c") + if err != nil { + t.Errorf("%v", err) + } + exp1 := Map{ + "a": Map{ + "b": Map{ + "d": "abd", + }, + "e": []interface{}{ + "ae1", + "ae2", + }, + }, + } + act1 := map1 + if !reflect.DeepEqual(exp1, act1) { + t.Errorf("expected content to see %v, saw %v", exp1, act1) + } + +} + +func Test_ToMap(t *testing.T) { + map1 := Map{ + "a": Map{ + "b": Map{ + "c": "abc", + "d": "abd", + }, + "e": []interface{}{ + "ae1", + "ae2", + }, + }, + } + val1 := map1.ToMap() + exp1 := map[string]interface{}{ + "a": map[string]interface{}{ + "b": map[string]interface{}{ + "c": "abc", + "d": "abd", + }, + "e": []interface{}{ + "ae1", + "ae2", + }, + }, + } + act1 := val1 + if !reflect.DeepEqual(exp1, act1) { + t.Errorf("expected content to see %v, saw %v", exp1, act1) + } +} + +func Test_FromMap(t *testing.T) { + map1 := FromMap(map[string]interface{}{ + "a": map[string]interface{}{ + "b": map[string]interface{}{ + "c": "abc", + "d": "abd", + }, + "e": []interface{}{ + "ae1", + "ae2", + }, + }, + }) + exp1 := Map{ + "a": Map{ + "b": Map{ + "c": "abc", + "d": "abd", + }, + "e": []interface{}{ + "ae1", + "ae2", + }, + }, + } + act1 := map1 + if !reflect.DeepEqual(exp1, act1) { + t.Errorf("expected content to see %v, saw %v", exp1, act1) + } +} + +func Test_ToJSON(t *testing.T) { + map1 := Map{ + "a": Map{ + "b": Map{ + "c": "abc", + "d": "abd", + }, + "e": []interface{}{ + "ae1", + "ae2", + }, + }, + } + val1, err := map1.ToJSON() + if err != nil { + t.Errorf("%v", err) + } + exp1 := []byte(`{"a":{"b":{"c":"abc","d":"abd"},"e":["ae1","ae2"]}}`) + act1 := val1 + if !bytes.Equal(exp1, act1) { + t.Errorf("expected content to see %v, saw %v", exp1, act1) + } +} + +func Test_FromJSON(t *testing.T) { + map1, err := FromJSON([]byte(`{"a":{"b":{"c":"abc","d":"abd"},"e":["ae1","ae2"]}}`)) + if err != nil { + t.Errorf("%v", err) + } + exp1 := Map{ + "a": Map{ + "b": Map{ + "c": "abc", + "d": "abd", + }, + "e": []interface{}{ + "ae1", + "ae2", + }, + }, + } + act1 := map1 + if !reflect.DeepEqual(exp1, act1) { + t.Errorf("expected content to see %v, saw %v", exp1, act1) + } +} + +func Test_ToYAML(t *testing.T) { + map1 := Map{ + "a": Map{ + "b": Map{ + "c": "abc", + "d": "abd", + }, + "e": []interface{}{ + "ae1", + "ae2", + }, + }, + } + + val1, err := map1.ToYAML() + if err != nil { + t.Errorf("%v", err) + } + exp1 := []byte(`a: + b: + c: abc + d: abd + e: + - ae1 + - ae2 +`) + act1 := val1 + if !bytes.Equal(exp1, act1) { + t.Errorf("expected content to see %v, saw %v", exp1, act1) + } +} + +func Test_FromYAML(t *testing.T) { + map1, err := FromYAML([]byte(`a: + b: + c: abc + d: abd + e: + - ae1 + - ae2 +`)) + if err != nil { + t.Errorf("%v", err) + } + exp1 := Map{ + "a": Map{ + "b": Map{ + "c": "abc", + "d": "abd", + }, + "e": []interface{}{ + "ae1", + "ae2", + }, + }, + } + act1 := map1 + if !reflect.DeepEqual(exp1, act1) { + t.Errorf("expected content to see %v, saw %v", exp1, act1) + } +} + +//func Test_Get(t *testing.T) { +// data1 := objx.Map{ +// "a": objx.Map{ +// "b": objx.Map{ +// "c": "abc", +// "d": "abd", +// }, +// "e": []interface{}{ +// "ae1", +// "ae2", +// }, +// }, +// } +// key1 := "/" +// val1, err := Get(data1, key1) +// if err != nil { +// t.Errorf("%v", err) +// } +// exp1 := map[string]interface{}{ +// "a": map[string]interface{}{ +// "b": map[string]interface{}{ +// "c": "abc", +// "d": "abd", +// }, +// "e": []interface{}{ +// "ae1", +// "ae2", +// }, +// }, +// } +// act1 := val1 +// if !reflect.DeepEqual(exp1, act1) { +// t.Errorf("expected content to see %v, saw %v", exp1, act1) +// } +// +// key2 := "/a" +// val2, err := Get(data1, key2) +// if err != nil { +// t.Errorf("%v", err) +// } +// exp2 := map[string]interface{}{ +// "b": map[string]interface{}{ +// "c": "abc", +// "d": "abd", +// }, +// "e": []interface{}{ +// "ae1", +// "ae2", +// }, +// } +// act2 := val2 +// if !reflect.DeepEqual(exp2, act2) { +// t.Errorf("expected content to see %v, saw %v", exp2, act2) +// } +//} + +//func Test_Set(t *testing.T) { +// data := map[string]interface{}{} +// +// data, err := Set(data, "/", map[string]interface{}{"a": 1}, true) +// if err != nil { +// t.Errorf("%v", err) +// } +// +// exp1 := 1 +// act1 := val1 +// if exp1 != act1 { +// t.Errorf("expected content to see %v, saw %v", exp1, act1) +// } +// +// fsm.applySet("/b/bb", map[string]interface{}{"b": 1}, false) +// +// val2, err := fsm.Get("/b") +// if err != nil { +// t.Errorf("%v", err) +// } +// +// exp2 := map[string]interface{}{"bb": map[string]interface{}{"b": 1}} +// act2 := val2.(map[string]interface{}) +// if !reflect.DeepEqual(exp2, act2) { +// t.Errorf("expected content to see %v, saw %v", exp2, act2) +// } +// +// fsm.applySet("/", map[string]interface{}{"a": 1}, false) +// +// val3, err := fsm.Get("/") +// if err != nil { +// t.Errorf("%v", err) +// } +// +// exp3 := map[string]interface{}{"a": 1} +// act3 := val3 +// if !reflect.DeepEqual(exp3, act3) { +// t.Errorf("expected content to see %v, saw %v", exp3, act3) +// } +// +// fsm.applySet("/", map[string]interface{}{"b": 2}, true) +// +// val4, err := fsm.Get("/") +// if err != nil { +// t.Errorf("%v", err) +// } +// +// exp4 := map[string]interface{}{"a": 1, "b": 2} +// act4 := val4 +// if !reflect.DeepEqual(exp4, act4) { +// t.Errorf("expected content to see %v, saw %v", exp4, act4) +// } +//} diff --git a/protobuf/blast.pb.go b/protobuf/blast.pb.go new file mode 100644 index 0000000..566ace5 --- /dev/null +++ b/protobuf/blast.pb.go @@ -0,0 +1,2189 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: protobuf/blast.proto + +package protobuf + +import ( + context "context" + fmt "fmt" + proto "github.com/golang/protobuf/proto" + any "github.com/golang/protobuf/ptypes/any" + empty "github.com/golang/protobuf/ptypes/empty" + grpc "google.golang.org/grpc" + math "math" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type LivenessProbeResponse_State int32 + +const ( + LivenessProbeResponse_UNKNOWN LivenessProbeResponse_State = 0 + LivenessProbeResponse_ALIVE LivenessProbeResponse_State = 1 + LivenessProbeResponse_DEAD LivenessProbeResponse_State = 2 +) + +var LivenessProbeResponse_State_name = map[int32]string{ + 0: "UNKNOWN", + 1: "ALIVE", + 2: "DEAD", +} + +var LivenessProbeResponse_State_value = map[string]int32{ + "UNKNOWN": 0, + "ALIVE": 1, + "DEAD": 2, +} + +func (x LivenessProbeResponse_State) String() string { + return proto.EnumName(LivenessProbeResponse_State_name, int32(x)) +} + +func (LivenessProbeResponse_State) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{0, 0} +} + +type ReadinessProbeResponse_State int32 + +const ( + ReadinessProbeResponse_UNKNOWN ReadinessProbeResponse_State = 0 + ReadinessProbeResponse_READY ReadinessProbeResponse_State = 1 + ReadinessProbeResponse_NOT_READY ReadinessProbeResponse_State = 2 +) + +var ReadinessProbeResponse_State_name = map[int32]string{ + 0: "UNKNOWN", + 1: "READY", + 2: "NOT_READY", +} + +var ReadinessProbeResponse_State_value = map[string]int32{ + "UNKNOWN": 0, + "READY": 1, + "NOT_READY": 2, +} + +func (x ReadinessProbeResponse_State) String() string { + return proto.EnumName(ReadinessProbeResponse_State_name, int32(x)) +} + +func (ReadinessProbeResponse_State) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{1, 0} +} + +type WatchStateResponse_Command int32 + +const ( + WatchStateResponse_UNKNOWN WatchStateResponse_Command = 0 + WatchStateResponse_SET WatchStateResponse_Command = 1 + WatchStateResponse_DELETE WatchStateResponse_Command = 2 +) + +var WatchStateResponse_Command_name = map[int32]string{ + 0: "UNKNOWN", + 1: "SET", + 2: "DELETE", +} + +var WatchStateResponse_Command_value = map[string]int32{ + "UNKNOWN": 0, + "SET": 1, + "DELETE": 2, +} + +func (x WatchStateResponse_Command) String() string { + return proto.EnumName(WatchStateResponse_Command_name, int32(x)) +} + +func (WatchStateResponse_Command) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{16, 0} +} + +// use for health check +type LivenessProbeResponse struct { + State LivenessProbeResponse_State `protobuf:"varint,1,opt,name=state,proto3,enum=index.LivenessProbeResponse_State" json:"state,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *LivenessProbeResponse) Reset() { *m = LivenessProbeResponse{} } +func (m *LivenessProbeResponse) String() string { return proto.CompactTextString(m) } +func (*LivenessProbeResponse) ProtoMessage() {} +func (*LivenessProbeResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{0} +} + +func (m *LivenessProbeResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_LivenessProbeResponse.Unmarshal(m, b) +} +func (m *LivenessProbeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_LivenessProbeResponse.Marshal(b, m, deterministic) +} +func (m *LivenessProbeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_LivenessProbeResponse.Merge(m, src) +} +func (m *LivenessProbeResponse) XXX_Size() int { + return xxx_messageInfo_LivenessProbeResponse.Size(m) +} +func (m *LivenessProbeResponse) XXX_DiscardUnknown() { + xxx_messageInfo_LivenessProbeResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_LivenessProbeResponse proto.InternalMessageInfo + +func (m *LivenessProbeResponse) GetState() LivenessProbeResponse_State { + if m != nil { + return m.State + } + return LivenessProbeResponse_UNKNOWN +} + +// use for health check +type ReadinessProbeResponse struct { + State ReadinessProbeResponse_State `protobuf:"varint,1,opt,name=state,proto3,enum=index.ReadinessProbeResponse_State" json:"state,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ReadinessProbeResponse) Reset() { *m = ReadinessProbeResponse{} } +func (m *ReadinessProbeResponse) String() string { return proto.CompactTextString(m) } +func (*ReadinessProbeResponse) ProtoMessage() {} +func (*ReadinessProbeResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{1} +} + +func (m *ReadinessProbeResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ReadinessProbeResponse.Unmarshal(m, b) +} +func (m *ReadinessProbeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ReadinessProbeResponse.Marshal(b, m, deterministic) +} +func (m *ReadinessProbeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ReadinessProbeResponse.Merge(m, src) +} +func (m *ReadinessProbeResponse) XXX_Size() int { + return xxx_messageInfo_ReadinessProbeResponse.Size(m) +} +func (m *ReadinessProbeResponse) XXX_DiscardUnknown() { + xxx_messageInfo_ReadinessProbeResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_ReadinessProbeResponse proto.InternalMessageInfo + +func (m *ReadinessProbeResponse) GetState() ReadinessProbeResponse_State { + if m != nil { + return m.State + } + return ReadinessProbeResponse_UNKNOWN +} + +type GetMetadataRequest struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetMetadataRequest) Reset() { *m = GetMetadataRequest{} } +func (m *GetMetadataRequest) String() string { return proto.CompactTextString(m) } +func (*GetMetadataRequest) ProtoMessage() {} +func (*GetMetadataRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{2} +} + +func (m *GetMetadataRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetMetadataRequest.Unmarshal(m, b) +} +func (m *GetMetadataRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetMetadataRequest.Marshal(b, m, deterministic) +} +func (m *GetMetadataRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetMetadataRequest.Merge(m, src) +} +func (m *GetMetadataRequest) XXX_Size() int { + return xxx_messageInfo_GetMetadataRequest.Size(m) +} +func (m *GetMetadataRequest) XXX_DiscardUnknown() { + xxx_messageInfo_GetMetadataRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_GetMetadataRequest proto.InternalMessageInfo + +func (m *GetMetadataRequest) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +type GetMetadataResponse struct { + Metadata *any.Any `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetMetadataResponse) Reset() { *m = GetMetadataResponse{} } +func (m *GetMetadataResponse) String() string { return proto.CompactTextString(m) } +func (*GetMetadataResponse) ProtoMessage() {} +func (*GetMetadataResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{3} +} + +func (m *GetMetadataResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetMetadataResponse.Unmarshal(m, b) +} +func (m *GetMetadataResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetMetadataResponse.Marshal(b, m, deterministic) +} +func (m *GetMetadataResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetMetadataResponse.Merge(m, src) +} +func (m *GetMetadataResponse) XXX_Size() int { + return xxx_messageInfo_GetMetadataResponse.Size(m) +} +func (m *GetMetadataResponse) XXX_DiscardUnknown() { + xxx_messageInfo_GetMetadataResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_GetMetadataResponse proto.InternalMessageInfo + +func (m *GetMetadataResponse) GetMetadata() *any.Any { + if m != nil { + return m.Metadata + } + return nil +} + +type GetNodeStateRequest struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetNodeStateRequest) Reset() { *m = GetNodeStateRequest{} } +func (m *GetNodeStateRequest) String() string { return proto.CompactTextString(m) } +func (*GetNodeStateRequest) ProtoMessage() {} +func (*GetNodeStateRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{4} +} + +func (m *GetNodeStateRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetNodeStateRequest.Unmarshal(m, b) +} +func (m *GetNodeStateRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetNodeStateRequest.Marshal(b, m, deterministic) +} +func (m *GetNodeStateRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetNodeStateRequest.Merge(m, src) +} +func (m *GetNodeStateRequest) XXX_Size() int { + return xxx_messageInfo_GetNodeStateRequest.Size(m) +} +func (m *GetNodeStateRequest) XXX_DiscardUnknown() { + xxx_messageInfo_GetNodeStateRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_GetNodeStateRequest proto.InternalMessageInfo + +func (m *GetNodeStateRequest) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +type GetNodeStateResponse struct { + State string `protobuf:"bytes,1,opt,name=state,proto3" json:"state,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetNodeStateResponse) Reset() { *m = GetNodeStateResponse{} } +func (m *GetNodeStateResponse) String() string { return proto.CompactTextString(m) } +func (*GetNodeStateResponse) ProtoMessage() {} +func (*GetNodeStateResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{5} +} + +func (m *GetNodeStateResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetNodeStateResponse.Unmarshal(m, b) +} +func (m *GetNodeStateResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetNodeStateResponse.Marshal(b, m, deterministic) +} +func (m *GetNodeStateResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetNodeStateResponse.Merge(m, src) +} +func (m *GetNodeStateResponse) XXX_Size() int { + return xxx_messageInfo_GetNodeStateResponse.Size(m) +} +func (m *GetNodeStateResponse) XXX_DiscardUnknown() { + xxx_messageInfo_GetNodeStateResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_GetNodeStateResponse proto.InternalMessageInfo + +func (m *GetNodeStateResponse) GetState() string { + if m != nil { + return m.State + } + return "" +} + +// use for raft cluster status +type GetNodeRequest struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetNodeRequest) Reset() { *m = GetNodeRequest{} } +func (m *GetNodeRequest) String() string { return proto.CompactTextString(m) } +func (*GetNodeRequest) ProtoMessage() {} +func (*GetNodeRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{6} +} + +func (m *GetNodeRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetNodeRequest.Unmarshal(m, b) +} +func (m *GetNodeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetNodeRequest.Marshal(b, m, deterministic) +} +func (m *GetNodeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetNodeRequest.Merge(m, src) +} +func (m *GetNodeRequest) XXX_Size() int { + return xxx_messageInfo_GetNodeRequest.Size(m) +} +func (m *GetNodeRequest) XXX_DiscardUnknown() { + xxx_messageInfo_GetNodeRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_GetNodeRequest proto.InternalMessageInfo + +func (m *GetNodeRequest) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +// use for raft cluster status +type GetNodeResponse struct { + Metadata *any.Any `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + State string `protobuf:"bytes,2,opt,name=state,proto3" json:"state,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetNodeResponse) Reset() { *m = GetNodeResponse{} } +func (m *GetNodeResponse) String() string { return proto.CompactTextString(m) } +func (*GetNodeResponse) ProtoMessage() {} +func (*GetNodeResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{7} +} + +func (m *GetNodeResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetNodeResponse.Unmarshal(m, b) +} +func (m *GetNodeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetNodeResponse.Marshal(b, m, deterministic) +} +func (m *GetNodeResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetNodeResponse.Merge(m, src) +} +func (m *GetNodeResponse) XXX_Size() int { + return xxx_messageInfo_GetNodeResponse.Size(m) +} +func (m *GetNodeResponse) XXX_DiscardUnknown() { + xxx_messageInfo_GetNodeResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_GetNodeResponse proto.InternalMessageInfo + +func (m *GetNodeResponse) GetMetadata() *any.Any { + if m != nil { + return m.Metadata + } + return nil +} + +func (m *GetNodeResponse) GetState() string { + if m != nil { + return m.State + } + return "" +} + +// use for raft cluster status +type SetNodeRequest struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Metadata *any.Any `protobuf:"bytes,2,opt,name=metadata,proto3" json:"metadata,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SetNodeRequest) Reset() { *m = SetNodeRequest{} } +func (m *SetNodeRequest) String() string { return proto.CompactTextString(m) } +func (*SetNodeRequest) ProtoMessage() {} +func (*SetNodeRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{8} +} + +func (m *SetNodeRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_SetNodeRequest.Unmarshal(m, b) +} +func (m *SetNodeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_SetNodeRequest.Marshal(b, m, deterministic) +} +func (m *SetNodeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_SetNodeRequest.Merge(m, src) +} +func (m *SetNodeRequest) XXX_Size() int { + return xxx_messageInfo_SetNodeRequest.Size(m) +} +func (m *SetNodeRequest) XXX_DiscardUnknown() { + xxx_messageInfo_SetNodeRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_SetNodeRequest proto.InternalMessageInfo + +func (m *SetNodeRequest) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +func (m *SetNodeRequest) GetMetadata() *any.Any { + if m != nil { + return m.Metadata + } + return nil +} + +// use for raft cluster status +type DeleteNodeRequest struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DeleteNodeRequest) Reset() { *m = DeleteNodeRequest{} } +func (m *DeleteNodeRequest) String() string { return proto.CompactTextString(m) } +func (*DeleteNodeRequest) ProtoMessage() {} +func (*DeleteNodeRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{9} +} + +func (m *DeleteNodeRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DeleteNodeRequest.Unmarshal(m, b) +} +func (m *DeleteNodeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DeleteNodeRequest.Marshal(b, m, deterministic) +} +func (m *DeleteNodeRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteNodeRequest.Merge(m, src) +} +func (m *DeleteNodeRequest) XXX_Size() int { + return xxx_messageInfo_DeleteNodeRequest.Size(m) +} +func (m *DeleteNodeRequest) XXX_DiscardUnknown() { + xxx_messageInfo_DeleteNodeRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_DeleteNodeRequest proto.InternalMessageInfo + +func (m *DeleteNodeRequest) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +// use for raft cluster status +type GetClusterResponse struct { + Cluster *any.Any `protobuf:"bytes,1,opt,name=cluster,proto3" json:"cluster,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetClusterResponse) Reset() { *m = GetClusterResponse{} } +func (m *GetClusterResponse) String() string { return proto.CompactTextString(m) } +func (*GetClusterResponse) ProtoMessage() {} +func (*GetClusterResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{10} +} + +func (m *GetClusterResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetClusterResponse.Unmarshal(m, b) +} +func (m *GetClusterResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetClusterResponse.Marshal(b, m, deterministic) +} +func (m *GetClusterResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetClusterResponse.Merge(m, src) +} +func (m *GetClusterResponse) XXX_Size() int { + return xxx_messageInfo_GetClusterResponse.Size(m) +} +func (m *GetClusterResponse) XXX_DiscardUnknown() { + xxx_messageInfo_GetClusterResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_GetClusterResponse proto.InternalMessageInfo + +func (m *GetClusterResponse) GetCluster() *any.Any { + if m != nil { + return m.Cluster + } + return nil +} + +type GetStateRequest struct { + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetStateRequest) Reset() { *m = GetStateRequest{} } +func (m *GetStateRequest) String() string { return proto.CompactTextString(m) } +func (*GetStateRequest) ProtoMessage() {} +func (*GetStateRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{11} +} + +func (m *GetStateRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetStateRequest.Unmarshal(m, b) +} +func (m *GetStateRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetStateRequest.Marshal(b, m, deterministic) +} +func (m *GetStateRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetStateRequest.Merge(m, src) +} +func (m *GetStateRequest) XXX_Size() int { + return xxx_messageInfo_GetStateRequest.Size(m) +} +func (m *GetStateRequest) XXX_DiscardUnknown() { + xxx_messageInfo_GetStateRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_GetStateRequest proto.InternalMessageInfo + +func (m *GetStateRequest) GetKey() string { + if m != nil { + return m.Key + } + return "" +} + +type GetStateResponse struct { + Value *any.Any `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetStateResponse) Reset() { *m = GetStateResponse{} } +func (m *GetStateResponse) String() string { return proto.CompactTextString(m) } +func (*GetStateResponse) ProtoMessage() {} +func (*GetStateResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{12} +} + +func (m *GetStateResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetStateResponse.Unmarshal(m, b) +} +func (m *GetStateResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetStateResponse.Marshal(b, m, deterministic) +} +func (m *GetStateResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetStateResponse.Merge(m, src) +} +func (m *GetStateResponse) XXX_Size() int { + return xxx_messageInfo_GetStateResponse.Size(m) +} +func (m *GetStateResponse) XXX_DiscardUnknown() { + xxx_messageInfo_GetStateResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_GetStateResponse proto.InternalMessageInfo + +func (m *GetStateResponse) GetValue() *any.Any { + if m != nil { + return m.Value + } + return nil +} + +type SetStateRequest struct { + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Value *any.Any `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SetStateRequest) Reset() { *m = SetStateRequest{} } +func (m *SetStateRequest) String() string { return proto.CompactTextString(m) } +func (*SetStateRequest) ProtoMessage() {} +func (*SetStateRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{13} +} + +func (m *SetStateRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_SetStateRequest.Unmarshal(m, b) +} +func (m *SetStateRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_SetStateRequest.Marshal(b, m, deterministic) +} +func (m *SetStateRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_SetStateRequest.Merge(m, src) +} +func (m *SetStateRequest) XXX_Size() int { + return xxx_messageInfo_SetStateRequest.Size(m) +} +func (m *SetStateRequest) XXX_DiscardUnknown() { + xxx_messageInfo_SetStateRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_SetStateRequest proto.InternalMessageInfo + +func (m *SetStateRequest) GetKey() string { + if m != nil { + return m.Key + } + return "" +} + +func (m *SetStateRequest) GetValue() *any.Any { + if m != nil { + return m.Value + } + return nil +} + +type DeleteStateRequest struct { + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DeleteStateRequest) Reset() { *m = DeleteStateRequest{} } +func (m *DeleteStateRequest) String() string { return proto.CompactTextString(m) } +func (*DeleteStateRequest) ProtoMessage() {} +func (*DeleteStateRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{14} +} + +func (m *DeleteStateRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DeleteStateRequest.Unmarshal(m, b) +} +func (m *DeleteStateRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DeleteStateRequest.Marshal(b, m, deterministic) +} +func (m *DeleteStateRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteStateRequest.Merge(m, src) +} +func (m *DeleteStateRequest) XXX_Size() int { + return xxx_messageInfo_DeleteStateRequest.Size(m) +} +func (m *DeleteStateRequest) XXX_DiscardUnknown() { + xxx_messageInfo_DeleteStateRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_DeleteStateRequest proto.InternalMessageInfo + +func (m *DeleteStateRequest) GetKey() string { + if m != nil { + return m.Key + } + return "" +} + +type WatchStateRequest struct { + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *WatchStateRequest) Reset() { *m = WatchStateRequest{} } +func (m *WatchStateRequest) String() string { return proto.CompactTextString(m) } +func (*WatchStateRequest) ProtoMessage() {} +func (*WatchStateRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{15} +} + +func (m *WatchStateRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_WatchStateRequest.Unmarshal(m, b) +} +func (m *WatchStateRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_WatchStateRequest.Marshal(b, m, deterministic) +} +func (m *WatchStateRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_WatchStateRequest.Merge(m, src) +} +func (m *WatchStateRequest) XXX_Size() int { + return xxx_messageInfo_WatchStateRequest.Size(m) +} +func (m *WatchStateRequest) XXX_DiscardUnknown() { + xxx_messageInfo_WatchStateRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_WatchStateRequest proto.InternalMessageInfo + +func (m *WatchStateRequest) GetKey() string { + if m != nil { + return m.Key + } + return "" +} + +type WatchStateResponse struct { + Command WatchStateResponse_Command `protobuf:"varint,1,opt,name=command,proto3,enum=index.WatchStateResponse_Command" json:"command,omitempty"` + Key string `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` + Value *any.Any `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *WatchStateResponse) Reset() { *m = WatchStateResponse{} } +func (m *WatchStateResponse) String() string { return proto.CompactTextString(m) } +func (*WatchStateResponse) ProtoMessage() {} +func (*WatchStateResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{16} +} + +func (m *WatchStateResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_WatchStateResponse.Unmarshal(m, b) +} +func (m *WatchStateResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_WatchStateResponse.Marshal(b, m, deterministic) +} +func (m *WatchStateResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_WatchStateResponse.Merge(m, src) +} +func (m *WatchStateResponse) XXX_Size() int { + return xxx_messageInfo_WatchStateResponse.Size(m) +} +func (m *WatchStateResponse) XXX_DiscardUnknown() { + xxx_messageInfo_WatchStateResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_WatchStateResponse proto.InternalMessageInfo + +func (m *WatchStateResponse) GetCommand() WatchStateResponse_Command { + if m != nil { + return m.Command + } + return WatchStateResponse_UNKNOWN +} + +func (m *WatchStateResponse) GetKey() string { + if m != nil { + return m.Key + } + return "" +} + +func (m *WatchStateResponse) GetValue() *any.Any { + if m != nil { + return m.Value + } + return nil +} + +type GetDocumentRequest struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetDocumentRequest) Reset() { *m = GetDocumentRequest{} } +func (m *GetDocumentRequest) String() string { return proto.CompactTextString(m) } +func (*GetDocumentRequest) ProtoMessage() {} +func (*GetDocumentRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{17} +} + +func (m *GetDocumentRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetDocumentRequest.Unmarshal(m, b) +} +func (m *GetDocumentRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetDocumentRequest.Marshal(b, m, deterministic) +} +func (m *GetDocumentRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetDocumentRequest.Merge(m, src) +} +func (m *GetDocumentRequest) XXX_Size() int { + return xxx_messageInfo_GetDocumentRequest.Size(m) +} +func (m *GetDocumentRequest) XXX_DiscardUnknown() { + xxx_messageInfo_GetDocumentRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_GetDocumentRequest proto.InternalMessageInfo + +func (m *GetDocumentRequest) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +type GetDocumentResponse struct { + Fields *any.Any `protobuf:"bytes,1,opt,name=fields,proto3" json:"fields,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetDocumentResponse) Reset() { *m = GetDocumentResponse{} } +func (m *GetDocumentResponse) String() string { return proto.CompactTextString(m) } +func (*GetDocumentResponse) ProtoMessage() {} +func (*GetDocumentResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{18} +} + +func (m *GetDocumentResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetDocumentResponse.Unmarshal(m, b) +} +func (m *GetDocumentResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetDocumentResponse.Marshal(b, m, deterministic) +} +func (m *GetDocumentResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetDocumentResponse.Merge(m, src) +} +func (m *GetDocumentResponse) XXX_Size() int { + return xxx_messageInfo_GetDocumentResponse.Size(m) +} +func (m *GetDocumentResponse) XXX_DiscardUnknown() { + xxx_messageInfo_GetDocumentResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_GetDocumentResponse proto.InternalMessageInfo + +func (m *GetDocumentResponse) GetFields() *any.Any { + if m != nil { + return m.Fields + } + return nil +} + +type IndexDocumentRequest struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Fields *any.Any `protobuf:"bytes,2,opt,name=fields,proto3" json:"fields,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *IndexDocumentRequest) Reset() { *m = IndexDocumentRequest{} } +func (m *IndexDocumentRequest) String() string { return proto.CompactTextString(m) } +func (*IndexDocumentRequest) ProtoMessage() {} +func (*IndexDocumentRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{19} +} + +func (m *IndexDocumentRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_IndexDocumentRequest.Unmarshal(m, b) +} +func (m *IndexDocumentRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_IndexDocumentRequest.Marshal(b, m, deterministic) +} +func (m *IndexDocumentRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_IndexDocumentRequest.Merge(m, src) +} +func (m *IndexDocumentRequest) XXX_Size() int { + return xxx_messageInfo_IndexDocumentRequest.Size(m) +} +func (m *IndexDocumentRequest) XXX_DiscardUnknown() { + xxx_messageInfo_IndexDocumentRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_IndexDocumentRequest proto.InternalMessageInfo + +func (m *IndexDocumentRequest) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +func (m *IndexDocumentRequest) GetFields() *any.Any { + if m != nil { + return m.Fields + } + return nil +} + +type IndexDocumentResponse struct { + Count int32 `protobuf:"varint,1,opt,name=count,proto3" json:"count,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *IndexDocumentResponse) Reset() { *m = IndexDocumentResponse{} } +func (m *IndexDocumentResponse) String() string { return proto.CompactTextString(m) } +func (*IndexDocumentResponse) ProtoMessage() {} +func (*IndexDocumentResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{20} +} + +func (m *IndexDocumentResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_IndexDocumentResponse.Unmarshal(m, b) +} +func (m *IndexDocumentResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_IndexDocumentResponse.Marshal(b, m, deterministic) +} +func (m *IndexDocumentResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_IndexDocumentResponse.Merge(m, src) +} +func (m *IndexDocumentResponse) XXX_Size() int { + return xxx_messageInfo_IndexDocumentResponse.Size(m) +} +func (m *IndexDocumentResponse) XXX_DiscardUnknown() { + xxx_messageInfo_IndexDocumentResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_IndexDocumentResponse proto.InternalMessageInfo + +func (m *IndexDocumentResponse) GetCount() int32 { + if m != nil { + return m.Count + } + return 0 +} + +type DeleteDocumentRequest struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DeleteDocumentRequest) Reset() { *m = DeleteDocumentRequest{} } +func (m *DeleteDocumentRequest) String() string { return proto.CompactTextString(m) } +func (*DeleteDocumentRequest) ProtoMessage() {} +func (*DeleteDocumentRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{21} +} + +func (m *DeleteDocumentRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DeleteDocumentRequest.Unmarshal(m, b) +} +func (m *DeleteDocumentRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DeleteDocumentRequest.Marshal(b, m, deterministic) +} +func (m *DeleteDocumentRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteDocumentRequest.Merge(m, src) +} +func (m *DeleteDocumentRequest) XXX_Size() int { + return xxx_messageInfo_DeleteDocumentRequest.Size(m) +} +func (m *DeleteDocumentRequest) XXX_DiscardUnknown() { + xxx_messageInfo_DeleteDocumentRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_DeleteDocumentRequest proto.InternalMessageInfo + +func (m *DeleteDocumentRequest) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +type DeleteDocumentResponse struct { + Count int32 `protobuf:"varint,1,opt,name=count,proto3" json:"count,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DeleteDocumentResponse) Reset() { *m = DeleteDocumentResponse{} } +func (m *DeleteDocumentResponse) String() string { return proto.CompactTextString(m) } +func (*DeleteDocumentResponse) ProtoMessage() {} +func (*DeleteDocumentResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{22} +} + +func (m *DeleteDocumentResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DeleteDocumentResponse.Unmarshal(m, b) +} +func (m *DeleteDocumentResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DeleteDocumentResponse.Marshal(b, m, deterministic) +} +func (m *DeleteDocumentResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteDocumentResponse.Merge(m, src) +} +func (m *DeleteDocumentResponse) XXX_Size() int { + return xxx_messageInfo_DeleteDocumentResponse.Size(m) +} +func (m *DeleteDocumentResponse) XXX_DiscardUnknown() { + xxx_messageInfo_DeleteDocumentResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_DeleteDocumentResponse proto.InternalMessageInfo + +func (m *DeleteDocumentResponse) GetCount() int32 { + if m != nil { + return m.Count + } + return 0 +} + +type SearchRequest struct { + SearchRequest *any.Any `protobuf:"bytes,1,opt,name=search_request,json=searchRequest,proto3" json:"search_request,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SearchRequest) Reset() { *m = SearchRequest{} } +func (m *SearchRequest) String() string { return proto.CompactTextString(m) } +func (*SearchRequest) ProtoMessage() {} +func (*SearchRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{23} +} + +func (m *SearchRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_SearchRequest.Unmarshal(m, b) +} +func (m *SearchRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_SearchRequest.Marshal(b, m, deterministic) +} +func (m *SearchRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_SearchRequest.Merge(m, src) +} +func (m *SearchRequest) XXX_Size() int { + return xxx_messageInfo_SearchRequest.Size(m) +} +func (m *SearchRequest) XXX_DiscardUnknown() { + xxx_messageInfo_SearchRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_SearchRequest proto.InternalMessageInfo + +func (m *SearchRequest) GetSearchRequest() *any.Any { + if m != nil { + return m.SearchRequest + } + return nil +} + +type SearchResponse struct { + SearchResult *any.Any `protobuf:"bytes,1,opt,name=search_result,json=searchResult,proto3" json:"search_result,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SearchResponse) Reset() { *m = SearchResponse{} } +func (m *SearchResponse) String() string { return proto.CompactTextString(m) } +func (*SearchResponse) ProtoMessage() {} +func (*SearchResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{24} +} + +func (m *SearchResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_SearchResponse.Unmarshal(m, b) +} +func (m *SearchResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_SearchResponse.Marshal(b, m, deterministic) +} +func (m *SearchResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_SearchResponse.Merge(m, src) +} +func (m *SearchResponse) XXX_Size() int { + return xxx_messageInfo_SearchResponse.Size(m) +} +func (m *SearchResponse) XXX_DiscardUnknown() { + xxx_messageInfo_SearchResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_SearchResponse proto.InternalMessageInfo + +func (m *SearchResponse) GetSearchResult() *any.Any { + if m != nil { + return m.SearchResult + } + return nil +} + +type GetIndexConfigResponse struct { + IndexConfig *any.Any `protobuf:"bytes,1,opt,name=index_config,json=indexConfig,proto3" json:"index_config,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetIndexConfigResponse) Reset() { *m = GetIndexConfigResponse{} } +func (m *GetIndexConfigResponse) String() string { return proto.CompactTextString(m) } +func (*GetIndexConfigResponse) ProtoMessage() {} +func (*GetIndexConfigResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{25} +} + +func (m *GetIndexConfigResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetIndexConfigResponse.Unmarshal(m, b) +} +func (m *GetIndexConfigResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetIndexConfigResponse.Marshal(b, m, deterministic) +} +func (m *GetIndexConfigResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetIndexConfigResponse.Merge(m, src) +} +func (m *GetIndexConfigResponse) XXX_Size() int { + return xxx_messageInfo_GetIndexConfigResponse.Size(m) +} +func (m *GetIndexConfigResponse) XXX_DiscardUnknown() { + xxx_messageInfo_GetIndexConfigResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_GetIndexConfigResponse proto.InternalMessageInfo + +func (m *GetIndexConfigResponse) GetIndexConfig() *any.Any { + if m != nil { + return m.IndexConfig + } + return nil +} + +type GetIndexStatsResponse struct { + IndexStats *any.Any `protobuf:"bytes,1,opt,name=index_stats,json=indexStats,proto3" json:"index_stats,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetIndexStatsResponse) Reset() { *m = GetIndexStatsResponse{} } +func (m *GetIndexStatsResponse) String() string { return proto.CompactTextString(m) } +func (*GetIndexStatsResponse) ProtoMessage() {} +func (*GetIndexStatsResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{26} +} + +func (m *GetIndexStatsResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetIndexStatsResponse.Unmarshal(m, b) +} +func (m *GetIndexStatsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetIndexStatsResponse.Marshal(b, m, deterministic) +} +func (m *GetIndexStatsResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetIndexStatsResponse.Merge(m, src) +} +func (m *GetIndexStatsResponse) XXX_Size() int { + return xxx_messageInfo_GetIndexStatsResponse.Size(m) +} +func (m *GetIndexStatsResponse) XXX_DiscardUnknown() { + xxx_messageInfo_GetIndexStatsResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_GetIndexStatsResponse proto.InternalMessageInfo + +func (m *GetIndexStatsResponse) GetIndexStats() *any.Any { + if m != nil { + return m.IndexStats + } + return nil +} + +// use for creating snapshot +type Document struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Fields *any.Any `protobuf:"bytes,2,opt,name=fields,proto3" json:"fields,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Document) Reset() { *m = Document{} } +func (m *Document) String() string { return proto.CompactTextString(m) } +func (*Document) ProtoMessage() {} +func (*Document) Descriptor() ([]byte, []int) { + return fileDescriptor_406ca165ef12c7d5, []int{27} +} + +func (m *Document) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Document.Unmarshal(m, b) +} +func (m *Document) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Document.Marshal(b, m, deterministic) +} +func (m *Document) XXX_Merge(src proto.Message) { + xxx_messageInfo_Document.Merge(m, src) +} +func (m *Document) XXX_Size() int { + return xxx_messageInfo_Document.Size(m) +} +func (m *Document) XXX_DiscardUnknown() { + xxx_messageInfo_Document.DiscardUnknown(m) +} + +var xxx_messageInfo_Document proto.InternalMessageInfo + +func (m *Document) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +func (m *Document) GetFields() *any.Any { + if m != nil { + return m.Fields + } + return nil +} + +func init() { + proto.RegisterEnum("index.LivenessProbeResponse_State", LivenessProbeResponse_State_name, LivenessProbeResponse_State_value) + proto.RegisterEnum("index.ReadinessProbeResponse_State", ReadinessProbeResponse_State_name, ReadinessProbeResponse_State_value) + proto.RegisterEnum("index.WatchStateResponse_Command", WatchStateResponse_Command_name, WatchStateResponse_Command_value) + proto.RegisterType((*LivenessProbeResponse)(nil), "index.LivenessProbeResponse") + proto.RegisterType((*ReadinessProbeResponse)(nil), "index.ReadinessProbeResponse") + proto.RegisterType((*GetMetadataRequest)(nil), "index.GetMetadataRequest") + proto.RegisterType((*GetMetadataResponse)(nil), "index.GetMetadataResponse") + proto.RegisterType((*GetNodeStateRequest)(nil), "index.GetNodeStateRequest") + proto.RegisterType((*GetNodeStateResponse)(nil), "index.GetNodeStateResponse") + proto.RegisterType((*GetNodeRequest)(nil), "index.GetNodeRequest") + proto.RegisterType((*GetNodeResponse)(nil), "index.GetNodeResponse") + proto.RegisterType((*SetNodeRequest)(nil), "index.SetNodeRequest") + proto.RegisterType((*DeleteNodeRequest)(nil), "index.DeleteNodeRequest") + proto.RegisterType((*GetClusterResponse)(nil), "index.GetClusterResponse") + proto.RegisterType((*GetStateRequest)(nil), "index.GetStateRequest") + proto.RegisterType((*GetStateResponse)(nil), "index.GetStateResponse") + proto.RegisterType((*SetStateRequest)(nil), "index.SetStateRequest") + proto.RegisterType((*DeleteStateRequest)(nil), "index.DeleteStateRequest") + proto.RegisterType((*WatchStateRequest)(nil), "index.WatchStateRequest") + proto.RegisterType((*WatchStateResponse)(nil), "index.WatchStateResponse") + proto.RegisterType((*GetDocumentRequest)(nil), "index.GetDocumentRequest") + proto.RegisterType((*GetDocumentResponse)(nil), "index.GetDocumentResponse") + proto.RegisterType((*IndexDocumentRequest)(nil), "index.IndexDocumentRequest") + proto.RegisterType((*IndexDocumentResponse)(nil), "index.IndexDocumentResponse") + proto.RegisterType((*DeleteDocumentRequest)(nil), "index.DeleteDocumentRequest") + proto.RegisterType((*DeleteDocumentResponse)(nil), "index.DeleteDocumentResponse") + proto.RegisterType((*SearchRequest)(nil), "index.SearchRequest") + proto.RegisterType((*SearchResponse)(nil), "index.SearchResponse") + proto.RegisterType((*GetIndexConfigResponse)(nil), "index.GetIndexConfigResponse") + proto.RegisterType((*GetIndexStatsResponse)(nil), "index.GetIndexStatsResponse") + proto.RegisterType((*Document)(nil), "index.Document") +} + +func init() { proto.RegisterFile("protobuf/blast.proto", fileDescriptor_406ca165ef12c7d5) } + +var fileDescriptor_406ca165ef12c7d5 = []byte{ + // 986 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x57, 0xed, 0x72, 0xdb, 0x44, + 0x14, 0x95, 0x14, 0x1c, 0x3b, 0xd7, 0xb1, 0xeb, 0x2e, 0xb6, 0x69, 0xdd, 0x76, 0x26, 0x6c, 0x28, + 0x14, 0x28, 0x4a, 0x27, 0x0c, 0x53, 0xa0, 0xc0, 0xe0, 0xc4, 0x22, 0x64, 0x1a, 0x1c, 0x2a, 0x07, + 0x3a, 0xe5, 0x4f, 0x46, 0xb6, 0x37, 0x89, 0xa6, 0xb6, 0x64, 0x2c, 0xa9, 0x43, 0x66, 0xf8, 0xc7, + 0x6b, 0xf1, 0x50, 0x3c, 0x02, 0xa3, 0xdd, 0xd5, 0x6a, 0xf5, 0xdd, 0x69, 0xfe, 0x59, 0xbb, 0xe7, + 0x9e, 0xb3, 0xf7, 0xee, 0xde, 0x7b, 0xc6, 0xd0, 0x5d, 0xad, 0x5d, 0xdf, 0x9d, 0x06, 0x17, 0x7b, + 0xd3, 0x85, 0xe5, 0xf9, 0x3a, 0xfd, 0x44, 0x35, 0xdb, 0x99, 0x93, 0xbf, 0x06, 0x77, 0x2f, 0x5d, + 0xf7, 0x72, 0x41, 0xf6, 0x04, 0xc6, 0x72, 0xae, 0x19, 0x62, 0x70, 0x2f, 0xbd, 0x45, 0x96, 0x2b, + 0x9f, 0x6f, 0xe2, 0xbf, 0xa1, 0x77, 0x62, 0xbf, 0x21, 0x0e, 0xf1, 0xbc, 0x5f, 0xd7, 0xee, 0x94, + 0x98, 0xc4, 0x5b, 0xb9, 0x8e, 0x47, 0xd0, 0xd7, 0x50, 0xf3, 0x7c, 0xcb, 0x27, 0x77, 0xd4, 0x1d, + 0xf5, 0x51, 0x7b, 0x1f, 0xeb, 0x54, 0x47, 0xcf, 0x05, 0xeb, 0x93, 0x10, 0x69, 0xb2, 0x00, 0xfc, + 0x29, 0xd4, 0xe8, 0x37, 0x6a, 0x42, 0xfd, 0xb7, 0xf1, 0xf3, 0xf1, 0xe9, 0xcb, 0x71, 0x47, 0x41, + 0x5b, 0x50, 0x1b, 0x9e, 0x1c, 0xff, 0x6e, 0x74, 0x54, 0xd4, 0x80, 0xf7, 0x46, 0xc6, 0x70, 0xd4, + 0xd1, 0xf0, 0x3f, 0x2a, 0xf4, 0x4d, 0x62, 0xcd, 0xed, 0xac, 0xfe, 0x37, 0x49, 0xfd, 0x5d, 0xae, + 0x9f, 0x8f, 0x4e, 0x1e, 0x40, 0x2f, 0x3a, 0x80, 0x69, 0x0c, 0x47, 0xaf, 0x3a, 0x2a, 0x6a, 0xc1, + 0xd6, 0xf8, 0xf4, 0xec, 0x9c, 0x7d, 0x6a, 0xf8, 0x23, 0x40, 0x47, 0xc4, 0xff, 0x85, 0xf8, 0xd6, + 0xdc, 0xf2, 0x2d, 0x93, 0xfc, 0x19, 0x10, 0xcf, 0x47, 0x6d, 0xd0, 0xec, 0x39, 0x55, 0xdf, 0x32, + 0x35, 0x7b, 0x8e, 0x8f, 0xe0, 0xfd, 0x04, 0x8a, 0x9f, 0xf3, 0x09, 0x34, 0x96, 0x7c, 0x8d, 0x82, + 0x9b, 0xfb, 0x5d, 0x9d, 0x15, 0x5c, 0x8f, 0x0a, 0xae, 0x0f, 0x9d, 0x6b, 0x53, 0xa0, 0xf0, 0x43, + 0x4a, 0x34, 0x76, 0xe7, 0x84, 0x9d, 0xba, 0x40, 0xef, 0x31, 0x74, 0x93, 0x30, 0x2e, 0xd8, 0x95, + 0x0b, 0xb3, 0x15, 0xe5, 0xbc, 0x03, 0x6d, 0x8e, 0x2e, 0xe2, 0x7b, 0x05, 0xb7, 0x04, 0xe2, 0x5d, + 0xcf, 0x1e, 0x8b, 0x6b, 0xb2, 0xb8, 0x09, 0xed, 0x49, 0xa9, 0x78, 0x42, 0x49, 0x7b, 0xab, 0x2a, + 0xed, 0xc2, 0xed, 0x11, 0x59, 0x10, 0x9f, 0x94, 0xe5, 0x34, 0xa2, 0x37, 0x77, 0xb8, 0x08, 0x3c, + 0x9f, 0xac, 0x45, 0x5a, 0x3a, 0xd4, 0x67, 0x6c, 0xa9, 0x34, 0xab, 0x08, 0x84, 0x77, 0x69, 0x65, + 0x12, 0x97, 0xd1, 0x81, 0x8d, 0xd7, 0xe4, 0x9a, 0x2b, 0x85, 0x3f, 0xf1, 0x0f, 0xd0, 0x89, 0x41, + 0x5c, 0xe8, 0x33, 0xa8, 0xbd, 0xb1, 0x16, 0x01, 0x29, 0x95, 0x61, 0x10, 0x7c, 0x0a, 0xb7, 0x26, + 0x55, 0x22, 0x31, 0xa1, 0x56, 0x4d, 0xf8, 0x31, 0x20, 0x56, 0xa0, 0x8a, 0x83, 0x3f, 0x84, 0xdb, + 0x2f, 0x2d, 0x7f, 0x76, 0x55, 0x01, 0xfb, 0x57, 0x05, 0x24, 0xe3, 0x78, 0x8a, 0xcf, 0xa0, 0x3e, + 0x73, 0x97, 0x4b, 0xcb, 0x99, 0xf3, 0x46, 0xfc, 0x90, 0x37, 0x62, 0x16, 0xab, 0x1f, 0x32, 0xa0, + 0x19, 0x45, 0x44, 0x2a, 0x5a, 0x4e, 0x82, 0x1b, 0xd5, 0x09, 0x7e, 0x0e, 0x75, 0xce, 0x98, 0x6c, + 0xe4, 0x3a, 0x6c, 0x4c, 0x8c, 0xb3, 0x8e, 0x8a, 0x00, 0x36, 0x47, 0xc6, 0x89, 0x71, 0x66, 0x88, + 0x1e, 0x1e, 0xb9, 0xb3, 0x60, 0x49, 0x1c, 0xbf, 0xe8, 0xbd, 0x1c, 0xd2, 0xd6, 0x8b, 0x51, 0x3c, + 0xc9, 0xc7, 0xb0, 0x79, 0x61, 0x93, 0xc5, 0xdc, 0x2b, 0xbd, 0x48, 0x8e, 0xc1, 0x67, 0xd0, 0x3d, + 0x0e, 0x4b, 0x50, 0x21, 0x26, 0xb1, 0x6a, 0x6f, 0xc1, 0xfa, 0x05, 0xf4, 0x52, 0xac, 0x71, 0xbf, + 0xcf, 0xdc, 0xc0, 0xf1, 0x29, 0x73, 0xcd, 0x64, 0x1f, 0xf8, 0x13, 0xe8, 0xb1, 0xdb, 0xaf, 0x4a, + 0x59, 0x87, 0x7e, 0x1a, 0x58, 0x4a, 0x7c, 0x02, 0xad, 0x09, 0xb1, 0xd6, 0xb3, 0xab, 0x88, 0xf0, + 0x19, 0xb4, 0x3d, 0xba, 0x70, 0xbe, 0x66, 0x2b, 0xa5, 0x45, 0x6a, 0x79, 0x72, 0x30, 0x7e, 0x1e, + 0x4e, 0x06, 0xb6, 0x20, 0xe6, 0x7a, 0x4b, 0xd0, 0x79, 0xc1, 0xa2, 0x9c, 0x6d, 0x3b, 0x62, 0x0b, + 0x91, 0xf8, 0x05, 0xf4, 0x8f, 0x88, 0x4f, 0xab, 0x74, 0xe8, 0x3a, 0x17, 0xf6, 0xa5, 0x20, 0x7d, + 0x0a, 0xdb, 0xf4, 0x55, 0x9e, 0xcf, 0xe8, 0x7a, 0x29, 0x67, 0xd3, 0x8e, 0x09, 0xf0, 0x18, 0x7a, + 0x11, 0x65, 0xf8, 0x96, 0x3d, 0xc1, 0xf8, 0x15, 0x30, 0xdc, 0x79, 0x38, 0xe1, 0xca, 0xdf, 0x05, + 0xd8, 0x22, 0x1c, 0xff, 0x0c, 0x8d, 0xa8, 0xce, 0x37, 0x7b, 0x0f, 0xfb, 0xff, 0x01, 0xd4, 0x0e, + 0x42, 0x9f, 0x47, 0x47, 0xd0, 0x4a, 0xb8, 0x2e, 0xea, 0x67, 0x02, 0x8d, 0xd0, 0xd1, 0x07, 0xf7, + 0xcb, 0x3c, 0x1a, 0x2b, 0xe8, 0x18, 0xda, 0x49, 0xfb, 0x2c, 0x64, 0x7a, 0x50, 0xea, 0xb6, 0x58, + 0x41, 0x3f, 0x41, 0x53, 0x32, 0x43, 0x74, 0x97, 0xe3, 0xb3, 0x36, 0x3a, 0x18, 0xe4, 0x6d, 0x49, + 0x47, 0xda, 0x96, 0x4d, 0x0e, 0x49, 0xe8, 0xb4, 0x41, 0x0e, 0xee, 0xe5, 0xee, 0x09, 0xaa, 0x6f, + 0xa1, 0xce, 0x77, 0x50, 0x2f, 0x89, 0x8c, 0x08, 0xfa, 0xe9, 0x65, 0x39, 0x76, 0x92, 0x8a, 0x9d, + 0xa4, 0x63, 0x73, 0x2b, 0x85, 0x15, 0xf4, 0x23, 0x40, 0x6c, 0x54, 0xe8, 0x0e, 0x0f, 0xcf, 0x78, + 0x57, 0x09, 0xc3, 0x10, 0x20, 0x76, 0xb1, 0xc2, 0x3b, 0x91, 0x6a, 0x9c, 0x32, 0x3c, 0xac, 0x20, + 0x03, 0xb6, 0xe9, 0x40, 0xbe, 0x09, 0xc9, 0x13, 0x15, 0x7d, 0x07, 0x8d, 0x89, 0x63, 0xad, 0xbc, + 0x2b, 0xd7, 0x2f, 0xa4, 0x28, 0xce, 0xe3, 0x7b, 0x68, 0x44, 0x16, 0x89, 0xa4, 0x5a, 0x27, 0x2e, + 0xf1, 0x83, 0xcc, 0xba, 0xc8, 0x21, 0x14, 0x4f, 0x87, 0xa7, 0x2c, 0xb3, 0x44, 0xfc, 0x00, 0x9a, + 0x92, 0x1d, 0x8a, 0x17, 0x99, 0xb5, 0xc8, 0x12, 0x0e, 0x03, 0x20, 0xb6, 0x35, 0x71, 0x95, 0x19, + 0xf7, 0x14, 0x55, 0xcc, 0x7a, 0x20, 0xad, 0x22, 0x6b, 0x0e, 0x31, 0x07, 0xa4, 0x9a, 0xa7, 0x86, + 0xb5, 0xdc, 0x1c, 0xe9, 0xf1, 0x8c, 0x15, 0x34, 0x86, 0x56, 0xc2, 0x12, 0x50, 0xd4, 0x01, 0x79, + 0xf6, 0x23, 0xba, 0x3f, 0xd7, 0x45, 0xb0, 0xf2, 0x48, 0x45, 0x2f, 0xa0, 0x9d, 0xb4, 0x02, 0x74, + 0x3f, 0x51, 0xa5, 0x34, 0xe3, 0x83, 0x82, 0x5d, 0x89, 0xf2, 0x29, 0x6c, 0xb2, 0xf9, 0x8e, 0xba, + 0xe2, 0xc6, 0xa4, 0xf9, 0x3f, 0xe8, 0xa5, 0x56, 0xe5, 0x59, 0x94, 0x9c, 0xe5, 0x95, 0xb3, 0x28, + 0x7f, 0xf4, 0x63, 0x25, 0x9c, 0x8f, 0x89, 0x19, 0x5e, 0x39, 0x1f, 0x73, 0x27, 0x3e, 0x56, 0x0e, + 0xf0, 0x1f, 0x3b, 0x97, 0xb6, 0x7f, 0x15, 0x4c, 0xf5, 0x99, 0xbb, 0xdc, 0x5b, 0xba, 0x5e, 0xf0, + 0xda, 0x62, 0xff, 0xb5, 0xc4, 0x7f, 0xa7, 0xe9, 0x26, 0xfd, 0xf5, 0xe5, 0xff, 0x01, 0x00, 0x00, + 0xff, 0xff, 0xc6, 0x0e, 0xdf, 0x26, 0x8d, 0x0d, 0x00, 0x00, +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// BlastClient is the client API for Blast service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type BlastClient interface { + LivenessProbe(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*LivenessProbeResponse, error) + ReadinessProbe(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*ReadinessProbeResponse, error) + GetMetadata(ctx context.Context, in *GetMetadataRequest, opts ...grpc.CallOption) (*GetMetadataResponse, error) + GetNodeState(ctx context.Context, in *GetNodeStateRequest, opts ...grpc.CallOption) (*GetNodeStateResponse, error) + GetNode(ctx context.Context, in *GetNodeRequest, opts ...grpc.CallOption) (*GetNodeResponse, error) + SetNode(ctx context.Context, in *SetNodeRequest, opts ...grpc.CallOption) (*empty.Empty, error) + DeleteNode(ctx context.Context, in *DeleteNodeRequest, opts ...grpc.CallOption) (*empty.Empty, error) + GetCluster(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*GetClusterResponse, error) + WatchCluster(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (Blast_WatchClusterClient, error) + Snapshot(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) + GetState(ctx context.Context, in *GetStateRequest, opts ...grpc.CallOption) (*GetStateResponse, error) + SetState(ctx context.Context, in *SetStateRequest, opts ...grpc.CallOption) (*empty.Empty, error) + DeleteState(ctx context.Context, in *DeleteStateRequest, opts ...grpc.CallOption) (*empty.Empty, error) + WatchState(ctx context.Context, in *WatchStateRequest, opts ...grpc.CallOption) (Blast_WatchStateClient, error) + GetDocument(ctx context.Context, in *GetDocumentRequest, opts ...grpc.CallOption) (*GetDocumentResponse, error) + IndexDocument(ctx context.Context, opts ...grpc.CallOption) (Blast_IndexDocumentClient, error) + DeleteDocument(ctx context.Context, opts ...grpc.CallOption) (Blast_DeleteDocumentClient, error) + Search(ctx context.Context, in *SearchRequest, opts ...grpc.CallOption) (*SearchResponse, error) + GetIndexConfig(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*GetIndexConfigResponse, error) + GetIndexStats(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*GetIndexStatsResponse, error) +} + +type blastClient struct { + cc *grpc.ClientConn +} + +func NewBlastClient(cc *grpc.ClientConn) BlastClient { + return &blastClient{cc} +} + +func (c *blastClient) LivenessProbe(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*LivenessProbeResponse, error) { + out := new(LivenessProbeResponse) + err := c.cc.Invoke(ctx, "/index.Blast/LivenessProbe", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *blastClient) ReadinessProbe(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*ReadinessProbeResponse, error) { + out := new(ReadinessProbeResponse) + err := c.cc.Invoke(ctx, "/index.Blast/ReadinessProbe", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *blastClient) GetMetadata(ctx context.Context, in *GetMetadataRequest, opts ...grpc.CallOption) (*GetMetadataResponse, error) { + out := new(GetMetadataResponse) + err := c.cc.Invoke(ctx, "/index.Blast/GetMetadata", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *blastClient) GetNodeState(ctx context.Context, in *GetNodeStateRequest, opts ...grpc.CallOption) (*GetNodeStateResponse, error) { + out := new(GetNodeStateResponse) + err := c.cc.Invoke(ctx, "/index.Blast/GetNodeState", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *blastClient) GetNode(ctx context.Context, in *GetNodeRequest, opts ...grpc.CallOption) (*GetNodeResponse, error) { + out := new(GetNodeResponse) + err := c.cc.Invoke(ctx, "/index.Blast/GetNode", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *blastClient) SetNode(ctx context.Context, in *SetNodeRequest, opts ...grpc.CallOption) (*empty.Empty, error) { + out := new(empty.Empty) + err := c.cc.Invoke(ctx, "/index.Blast/SetNode", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *blastClient) DeleteNode(ctx context.Context, in *DeleteNodeRequest, opts ...grpc.CallOption) (*empty.Empty, error) { + out := new(empty.Empty) + err := c.cc.Invoke(ctx, "/index.Blast/DeleteNode", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *blastClient) GetCluster(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*GetClusterResponse, error) { + out := new(GetClusterResponse) + err := c.cc.Invoke(ctx, "/index.Blast/GetCluster", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *blastClient) WatchCluster(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (Blast_WatchClusterClient, error) { + stream, err := c.cc.NewStream(ctx, &_Blast_serviceDesc.Streams[0], "/index.Blast/WatchCluster", opts...) + if err != nil { + return nil, err + } + x := &blastWatchClusterClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type Blast_WatchClusterClient interface { + Recv() (*GetClusterResponse, error) + grpc.ClientStream +} + +type blastWatchClusterClient struct { + grpc.ClientStream +} + +func (x *blastWatchClusterClient) Recv() (*GetClusterResponse, error) { + m := new(GetClusterResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *blastClient) Snapshot(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) { + out := new(empty.Empty) + err := c.cc.Invoke(ctx, "/index.Blast/Snapshot", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *blastClient) GetState(ctx context.Context, in *GetStateRequest, opts ...grpc.CallOption) (*GetStateResponse, error) { + out := new(GetStateResponse) + err := c.cc.Invoke(ctx, "/index.Blast/GetState", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *blastClient) SetState(ctx context.Context, in *SetStateRequest, opts ...grpc.CallOption) (*empty.Empty, error) { + out := new(empty.Empty) + err := c.cc.Invoke(ctx, "/index.Blast/SetState", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *blastClient) DeleteState(ctx context.Context, in *DeleteStateRequest, opts ...grpc.CallOption) (*empty.Empty, error) { + out := new(empty.Empty) + err := c.cc.Invoke(ctx, "/index.Blast/DeleteState", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *blastClient) WatchState(ctx context.Context, in *WatchStateRequest, opts ...grpc.CallOption) (Blast_WatchStateClient, error) { + stream, err := c.cc.NewStream(ctx, &_Blast_serviceDesc.Streams[1], "/index.Blast/WatchState", opts...) + if err != nil { + return nil, err + } + x := &blastWatchStateClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type Blast_WatchStateClient interface { + Recv() (*WatchStateResponse, error) + grpc.ClientStream +} + +type blastWatchStateClient struct { + grpc.ClientStream +} + +func (x *blastWatchStateClient) Recv() (*WatchStateResponse, error) { + m := new(WatchStateResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *blastClient) GetDocument(ctx context.Context, in *GetDocumentRequest, opts ...grpc.CallOption) (*GetDocumentResponse, error) { + out := new(GetDocumentResponse) + err := c.cc.Invoke(ctx, "/index.Blast/GetDocument", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *blastClient) IndexDocument(ctx context.Context, opts ...grpc.CallOption) (Blast_IndexDocumentClient, error) { + stream, err := c.cc.NewStream(ctx, &_Blast_serviceDesc.Streams[2], "/index.Blast/IndexDocument", opts...) + if err != nil { + return nil, err + } + x := &blastIndexDocumentClient{stream} + return x, nil +} + +type Blast_IndexDocumentClient interface { + Send(*IndexDocumentRequest) error + CloseAndRecv() (*IndexDocumentResponse, error) + grpc.ClientStream +} + +type blastIndexDocumentClient struct { + grpc.ClientStream +} + +func (x *blastIndexDocumentClient) Send(m *IndexDocumentRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *blastIndexDocumentClient) CloseAndRecv() (*IndexDocumentResponse, error) { + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + m := new(IndexDocumentResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *blastClient) DeleteDocument(ctx context.Context, opts ...grpc.CallOption) (Blast_DeleteDocumentClient, error) { + stream, err := c.cc.NewStream(ctx, &_Blast_serviceDesc.Streams[3], "/index.Blast/DeleteDocument", opts...) + if err != nil { + return nil, err + } + x := &blastDeleteDocumentClient{stream} + return x, nil +} + +type Blast_DeleteDocumentClient interface { + Send(*DeleteDocumentRequest) error + CloseAndRecv() (*DeleteDocumentResponse, error) + grpc.ClientStream +} + +type blastDeleteDocumentClient struct { + grpc.ClientStream +} + +func (x *blastDeleteDocumentClient) Send(m *DeleteDocumentRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *blastDeleteDocumentClient) CloseAndRecv() (*DeleteDocumentResponse, error) { + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + m := new(DeleteDocumentResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *blastClient) Search(ctx context.Context, in *SearchRequest, opts ...grpc.CallOption) (*SearchResponse, error) { + out := new(SearchResponse) + err := c.cc.Invoke(ctx, "/index.Blast/Search", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *blastClient) GetIndexConfig(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*GetIndexConfigResponse, error) { + out := new(GetIndexConfigResponse) + err := c.cc.Invoke(ctx, "/index.Blast/GetIndexConfig", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *blastClient) GetIndexStats(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*GetIndexStatsResponse, error) { + out := new(GetIndexStatsResponse) + err := c.cc.Invoke(ctx, "/index.Blast/GetIndexStats", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// BlastServer is the server API for Blast service. +type BlastServer interface { + LivenessProbe(context.Context, *empty.Empty) (*LivenessProbeResponse, error) + ReadinessProbe(context.Context, *empty.Empty) (*ReadinessProbeResponse, error) + GetMetadata(context.Context, *GetMetadataRequest) (*GetMetadataResponse, error) + GetNodeState(context.Context, *GetNodeStateRequest) (*GetNodeStateResponse, error) + GetNode(context.Context, *GetNodeRequest) (*GetNodeResponse, error) + SetNode(context.Context, *SetNodeRequest) (*empty.Empty, error) + DeleteNode(context.Context, *DeleteNodeRequest) (*empty.Empty, error) + GetCluster(context.Context, *empty.Empty) (*GetClusterResponse, error) + WatchCluster(*empty.Empty, Blast_WatchClusterServer) error + Snapshot(context.Context, *empty.Empty) (*empty.Empty, error) + GetState(context.Context, *GetStateRequest) (*GetStateResponse, error) + SetState(context.Context, *SetStateRequest) (*empty.Empty, error) + DeleteState(context.Context, *DeleteStateRequest) (*empty.Empty, error) + WatchState(*WatchStateRequest, Blast_WatchStateServer) error + GetDocument(context.Context, *GetDocumentRequest) (*GetDocumentResponse, error) + IndexDocument(Blast_IndexDocumentServer) error + DeleteDocument(Blast_DeleteDocumentServer) error + Search(context.Context, *SearchRequest) (*SearchResponse, error) + GetIndexConfig(context.Context, *empty.Empty) (*GetIndexConfigResponse, error) + GetIndexStats(context.Context, *empty.Empty) (*GetIndexStatsResponse, error) +} + +func RegisterBlastServer(s *grpc.Server, srv BlastServer) { + s.RegisterService(&_Blast_serviceDesc, srv) +} + +func _Blast_LivenessProbe_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(empty.Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlastServer).LivenessProbe(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/index.Blast/LivenessProbe", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlastServer).LivenessProbe(ctx, req.(*empty.Empty)) + } + return interceptor(ctx, in, info, handler) +} + +func _Blast_ReadinessProbe_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(empty.Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlastServer).ReadinessProbe(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/index.Blast/ReadinessProbe", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlastServer).ReadinessProbe(ctx, req.(*empty.Empty)) + } + return interceptor(ctx, in, info, handler) +} + +func _Blast_GetMetadata_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetMetadataRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlastServer).GetMetadata(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/index.Blast/GetMetadata", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlastServer).GetMetadata(ctx, req.(*GetMetadataRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Blast_GetNodeState_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetNodeStateRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlastServer).GetNodeState(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/index.Blast/GetNodeState", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlastServer).GetNodeState(ctx, req.(*GetNodeStateRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Blast_GetNode_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetNodeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlastServer).GetNode(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/index.Blast/GetNode", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlastServer).GetNode(ctx, req.(*GetNodeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Blast_SetNode_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SetNodeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlastServer).SetNode(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/index.Blast/SetNode", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlastServer).SetNode(ctx, req.(*SetNodeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Blast_DeleteNode_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteNodeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlastServer).DeleteNode(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/index.Blast/DeleteNode", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlastServer).DeleteNode(ctx, req.(*DeleteNodeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Blast_GetCluster_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(empty.Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlastServer).GetCluster(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/index.Blast/GetCluster", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlastServer).GetCluster(ctx, req.(*empty.Empty)) + } + return interceptor(ctx, in, info, handler) +} + +func _Blast_WatchCluster_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(empty.Empty) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(BlastServer).WatchCluster(m, &blastWatchClusterServer{stream}) +} + +type Blast_WatchClusterServer interface { + Send(*GetClusterResponse) error + grpc.ServerStream +} + +type blastWatchClusterServer struct { + grpc.ServerStream +} + +func (x *blastWatchClusterServer) Send(m *GetClusterResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _Blast_Snapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(empty.Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlastServer).Snapshot(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/index.Blast/Snapshot", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlastServer).Snapshot(ctx, req.(*empty.Empty)) + } + return interceptor(ctx, in, info, handler) +} + +func _Blast_GetState_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetStateRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlastServer).GetState(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/index.Blast/GetState", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlastServer).GetState(ctx, req.(*GetStateRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Blast_SetState_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SetStateRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlastServer).SetState(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/index.Blast/SetState", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlastServer).SetState(ctx, req.(*SetStateRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Blast_DeleteState_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteStateRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlastServer).DeleteState(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/index.Blast/DeleteState", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlastServer).DeleteState(ctx, req.(*DeleteStateRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Blast_WatchState_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(WatchStateRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(BlastServer).WatchState(m, &blastWatchStateServer{stream}) +} + +type Blast_WatchStateServer interface { + Send(*WatchStateResponse) error + grpc.ServerStream +} + +type blastWatchStateServer struct { + grpc.ServerStream +} + +func (x *blastWatchStateServer) Send(m *WatchStateResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _Blast_GetDocument_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetDocumentRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlastServer).GetDocument(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/index.Blast/GetDocument", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlastServer).GetDocument(ctx, req.(*GetDocumentRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Blast_IndexDocument_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(BlastServer).IndexDocument(&blastIndexDocumentServer{stream}) +} + +type Blast_IndexDocumentServer interface { + SendAndClose(*IndexDocumentResponse) error + Recv() (*IndexDocumentRequest, error) + grpc.ServerStream +} + +type blastIndexDocumentServer struct { + grpc.ServerStream +} + +func (x *blastIndexDocumentServer) SendAndClose(m *IndexDocumentResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *blastIndexDocumentServer) Recv() (*IndexDocumentRequest, error) { + m := new(IndexDocumentRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _Blast_DeleteDocument_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(BlastServer).DeleteDocument(&blastDeleteDocumentServer{stream}) +} + +type Blast_DeleteDocumentServer interface { + SendAndClose(*DeleteDocumentResponse) error + Recv() (*DeleteDocumentRequest, error) + grpc.ServerStream +} + +type blastDeleteDocumentServer struct { + grpc.ServerStream +} + +func (x *blastDeleteDocumentServer) SendAndClose(m *DeleteDocumentResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *blastDeleteDocumentServer) Recv() (*DeleteDocumentRequest, error) { + m := new(DeleteDocumentRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _Blast_Search_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SearchRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlastServer).Search(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/index.Blast/Search", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlastServer).Search(ctx, req.(*SearchRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Blast_GetIndexConfig_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(empty.Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlastServer).GetIndexConfig(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/index.Blast/GetIndexConfig", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlastServer).GetIndexConfig(ctx, req.(*empty.Empty)) + } + return interceptor(ctx, in, info, handler) +} + +func _Blast_GetIndexStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(empty.Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlastServer).GetIndexStats(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/index.Blast/GetIndexStats", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlastServer).GetIndexStats(ctx, req.(*empty.Empty)) + } + return interceptor(ctx, in, info, handler) +} + +var _Blast_serviceDesc = grpc.ServiceDesc{ + ServiceName: "index.Blast", + HandlerType: (*BlastServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "LivenessProbe", + Handler: _Blast_LivenessProbe_Handler, + }, + { + MethodName: "ReadinessProbe", + Handler: _Blast_ReadinessProbe_Handler, + }, + { + MethodName: "GetMetadata", + Handler: _Blast_GetMetadata_Handler, + }, + { + MethodName: "GetNodeState", + Handler: _Blast_GetNodeState_Handler, + }, + { + MethodName: "GetNode", + Handler: _Blast_GetNode_Handler, + }, + { + MethodName: "SetNode", + Handler: _Blast_SetNode_Handler, + }, + { + MethodName: "DeleteNode", + Handler: _Blast_DeleteNode_Handler, + }, + { + MethodName: "GetCluster", + Handler: _Blast_GetCluster_Handler, + }, + { + MethodName: "Snapshot", + Handler: _Blast_Snapshot_Handler, + }, + { + MethodName: "GetState", + Handler: _Blast_GetState_Handler, + }, + { + MethodName: "SetState", + Handler: _Blast_SetState_Handler, + }, + { + MethodName: "DeleteState", + Handler: _Blast_DeleteState_Handler, + }, + { + MethodName: "GetDocument", + Handler: _Blast_GetDocument_Handler, + }, + { + MethodName: "Search", + Handler: _Blast_Search_Handler, + }, + { + MethodName: "GetIndexConfig", + Handler: _Blast_GetIndexConfig_Handler, + }, + { + MethodName: "GetIndexStats", + Handler: _Blast_GetIndexStats_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "WatchCluster", + Handler: _Blast_WatchCluster_Handler, + ServerStreams: true, + }, + { + StreamName: "WatchState", + Handler: _Blast_WatchState_Handler, + ServerStreams: true, + }, + { + StreamName: "IndexDocument", + Handler: _Blast_IndexDocument_Handler, + ClientStreams: true, + }, + { + StreamName: "DeleteDocument", + Handler: _Blast_DeleteDocument_Handler, + ClientStreams: true, + }, + }, + Metadata: "protobuf/blast.proto", +} diff --git a/protobuf/blast.proto b/protobuf/blast.proto new file mode 100644 index 0000000..f9bfe66 --- /dev/null +++ b/protobuf/blast.proto @@ -0,0 +1,190 @@ +// Copyright (c) 2019 Minoru Osuka +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +import "google/protobuf/any.proto"; +import "google/protobuf/empty.proto"; + +package index; + +option go_package = "github.com/mosuka/blast/protobuf"; + +service Blast { + rpc LivenessProbe (google.protobuf.Empty) returns (LivenessProbeResponse) {} + rpc ReadinessProbe (google.protobuf.Empty) returns (ReadinessProbeResponse) {} + + rpc GetMetadata (GetMetadataRequest) returns (GetMetadataResponse) {} + rpc GetNodeState (GetNodeStateRequest) returns (GetNodeStateResponse) {} + rpc GetNode (GetNodeRequest) returns (GetNodeResponse) {} + rpc SetNode (SetNodeRequest) returns (google.protobuf.Empty) {} + rpc DeleteNode (DeleteNodeRequest) returns (google.protobuf.Empty) {} + rpc GetCluster (google.protobuf.Empty) returns (GetClusterResponse) {} + rpc WatchCluster (google.protobuf.Empty) returns (stream GetClusterResponse) {} + rpc Snapshot (google.protobuf.Empty) returns (google.protobuf.Empty) {} + + rpc GetState (GetStateRequest) returns (GetStateResponse) {} + rpc SetState (SetStateRequest) returns (google.protobuf.Empty) {} + rpc DeleteState (DeleteStateRequest) returns (google.protobuf.Empty) {} + rpc WatchState (WatchStateRequest) returns (stream WatchStateResponse) {} + + rpc GetDocument (GetDocumentRequest) returns (GetDocumentResponse) {} + rpc IndexDocument (stream IndexDocumentRequest) returns (IndexDocumentResponse) {} + rpc DeleteDocument (stream DeleteDocumentRequest) returns (DeleteDocumentResponse) {} + rpc Search (SearchRequest) returns (SearchResponse) {} + rpc GetIndexConfig (google.protobuf.Empty) returns (GetIndexConfigResponse) {} + rpc GetIndexStats (google.protobuf.Empty) returns (GetIndexStatsResponse) {} +} + +// use for health check +message LivenessProbeResponse { + enum State { + UNKNOWN = 0; + ALIVE = 1; + DEAD = 2; + } + State state = 1; +} + +// use for health check +message ReadinessProbeResponse { + enum State { + UNKNOWN = 0; + READY = 1; + NOT_READY = 2; + } + State state = 1; +} + +message GetMetadataRequest { + string id = 1; +} + +message GetMetadataResponse { + google.protobuf.Any metadata = 1; +} + +message GetNodeStateRequest { + string id = 1; +} + +message GetNodeStateResponse { + string state = 1; +} + +// use for raft cluster status +message GetNodeRequest { + string id = 1; +} + +// use for raft cluster status +message GetNodeResponse { + google.protobuf.Any metadata = 1; + string state = 2; +} + +// use for raft cluster status +message SetNodeRequest { + string id = 1; + google.protobuf.Any metadata = 2; +} + +// use for raft cluster status +message DeleteNodeRequest { + string id = 1; +} + +// use for raft cluster status +message GetClusterResponse { + google.protobuf.Any cluster = 1; +} + +message GetStateRequest { + string key = 1; +} + +message GetStateResponse { + google.protobuf.Any value = 1; +} + +message SetStateRequest { + string key = 1; + google.protobuf.Any value = 2; +} + +message DeleteStateRequest { + string key = 1; +} + +message WatchStateRequest { + string key = 1; +} + +message WatchStateResponse { + enum Command { + UNKNOWN = 0; + SET = 1; + DELETE = 2; + } + Command command = 1; + string key = 2; + google.protobuf.Any value = 3; +} + +message GetDocumentRequest { + string id = 1; +} + +message GetDocumentResponse { + google.protobuf.Any fields = 1; +} + +message IndexDocumentRequest { + string id = 1; + google.protobuf.Any fields = 2; +} + +message IndexDocumentResponse { + int32 count = 1; +} + +message DeleteDocumentRequest { + string id = 1; +} + +message DeleteDocumentResponse { + int32 count = 1; +} + +message SearchRequest { + google.protobuf.Any search_request = 1; +} + +message SearchResponse { + google.protobuf.Any search_result = 1; +} + +message GetIndexConfigResponse { + google.protobuf.Any index_config = 1; +} + +message GetIndexStatsResponse { + google.protobuf.Any index_stats = 1; +} + +// use for creating snapshot +message Document { + string id = 1; + google.protobuf.Any fields = 2; +} diff --git a/protobuf/index/index.pb.go b/protobuf/index/index.pb.go deleted file mode 100644 index 2bbb7d1..0000000 --- a/protobuf/index/index.pb.go +++ /dev/null @@ -1,800 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// source: protobuf/index/index.proto - -package index - -import ( - context "context" - fmt "fmt" - proto "github.com/golang/protobuf/proto" - any "github.com/golang/protobuf/ptypes/any" - empty "github.com/golang/protobuf/ptypes/empty" - raft "github.com/mosuka/blast/protobuf/raft" - grpc "google.golang.org/grpc" - math "math" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package - -type IndexCommand_Type int32 - -const ( - IndexCommand_UNKNOWN_COMMAND IndexCommand_Type = 0 - IndexCommand_SET_METADATA IndexCommand_Type = 1 - IndexCommand_DELETE_METADATA IndexCommand_Type = 2 - IndexCommand_INDEX_DOCUMENT IndexCommand_Type = 3 - IndexCommand_DELETE_DOCUMENT IndexCommand_Type = 4 -) - -var IndexCommand_Type_name = map[int32]string{ - 0: "UNKNOWN_COMMAND", - 1: "SET_METADATA", - 2: "DELETE_METADATA", - 3: "INDEX_DOCUMENT", - 4: "DELETE_DOCUMENT", -} - -var IndexCommand_Type_value = map[string]int32{ - "UNKNOWN_COMMAND": 0, - "SET_METADATA": 1, - "DELETE_METADATA": 2, - "INDEX_DOCUMENT": 3, - "DELETE_DOCUMENT": 4, -} - -func (x IndexCommand_Type) String() string { - return proto.EnumName(IndexCommand_Type_name, int32(x)) -} - -func (IndexCommand_Type) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_7b2daf652facb3ae, []int{5, 0} -} - -type Document struct { - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` - Fields *any.Any `protobuf:"bytes,2,opt,name=fields,proto3" json:"fields,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Document) Reset() { *m = Document{} } -func (m *Document) String() string { return proto.CompactTextString(m) } -func (*Document) ProtoMessage() {} -func (*Document) Descriptor() ([]byte, []int) { - return fileDescriptor_7b2daf652facb3ae, []int{0} -} - -func (m *Document) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Document.Unmarshal(m, b) -} -func (m *Document) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Document.Marshal(b, m, deterministic) -} -func (m *Document) XXX_Merge(src proto.Message) { - xxx_messageInfo_Document.Merge(m, src) -} -func (m *Document) XXX_Size() int { - return xxx_messageInfo_Document.Size(m) -} -func (m *Document) XXX_DiscardUnknown() { - xxx_messageInfo_Document.DiscardUnknown(m) -} - -var xxx_messageInfo_Document proto.InternalMessageInfo - -func (m *Document) GetId() string { - if m != nil { - return m.Id - } - return "" -} - -func (m *Document) GetFields() *any.Any { - if m != nil { - return m.Fields - } - return nil -} - -type UpdateResult struct { - Count int32 `protobuf:"varint,1,opt,name=count,proto3" json:"count,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *UpdateResult) Reset() { *m = UpdateResult{} } -func (m *UpdateResult) String() string { return proto.CompactTextString(m) } -func (*UpdateResult) ProtoMessage() {} -func (*UpdateResult) Descriptor() ([]byte, []int) { - return fileDescriptor_7b2daf652facb3ae, []int{1} -} - -func (m *UpdateResult) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_UpdateResult.Unmarshal(m, b) -} -func (m *UpdateResult) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_UpdateResult.Marshal(b, m, deterministic) -} -func (m *UpdateResult) XXX_Merge(src proto.Message) { - xxx_messageInfo_UpdateResult.Merge(m, src) -} -func (m *UpdateResult) XXX_Size() int { - return xxx_messageInfo_UpdateResult.Size(m) -} -func (m *UpdateResult) XXX_DiscardUnknown() { - xxx_messageInfo_UpdateResult.DiscardUnknown(m) -} - -var xxx_messageInfo_UpdateResult proto.InternalMessageInfo - -func (m *UpdateResult) GetCount() int32 { - if m != nil { - return m.Count - } - return 0 -} - -type Stats struct { - Stats *any.Any `protobuf:"bytes,1,opt,name=stats,proto3" json:"stats,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Stats) Reset() { *m = Stats{} } -func (m *Stats) String() string { return proto.CompactTextString(m) } -func (*Stats) ProtoMessage() {} -func (*Stats) Descriptor() ([]byte, []int) { - return fileDescriptor_7b2daf652facb3ae, []int{2} -} - -func (m *Stats) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Stats.Unmarshal(m, b) -} -func (m *Stats) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Stats.Marshal(b, m, deterministic) -} -func (m *Stats) XXX_Merge(src proto.Message) { - xxx_messageInfo_Stats.Merge(m, src) -} -func (m *Stats) XXX_Size() int { - return xxx_messageInfo_Stats.Size(m) -} -func (m *Stats) XXX_DiscardUnknown() { - xxx_messageInfo_Stats.DiscardUnknown(m) -} - -var xxx_messageInfo_Stats proto.InternalMessageInfo - -func (m *Stats) GetStats() *any.Any { - if m != nil { - return m.Stats - } - return nil -} - -type SearchRequest struct { - SearchRequest *any.Any `protobuf:"bytes,1,opt,name=search_request,json=searchRequest,proto3" json:"search_request,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *SearchRequest) Reset() { *m = SearchRequest{} } -func (m *SearchRequest) String() string { return proto.CompactTextString(m) } -func (*SearchRequest) ProtoMessage() {} -func (*SearchRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_7b2daf652facb3ae, []int{3} -} - -func (m *SearchRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SearchRequest.Unmarshal(m, b) -} -func (m *SearchRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SearchRequest.Marshal(b, m, deterministic) -} -func (m *SearchRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_SearchRequest.Merge(m, src) -} -func (m *SearchRequest) XXX_Size() int { - return xxx_messageInfo_SearchRequest.Size(m) -} -func (m *SearchRequest) XXX_DiscardUnknown() { - xxx_messageInfo_SearchRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_SearchRequest proto.InternalMessageInfo - -func (m *SearchRequest) GetSearchRequest() *any.Any { - if m != nil { - return m.SearchRequest - } - return nil -} - -type SearchResponse struct { - SearchResult *any.Any `protobuf:"bytes,1,opt,name=search_result,json=searchResult,proto3" json:"search_result,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *SearchResponse) Reset() { *m = SearchResponse{} } -func (m *SearchResponse) String() string { return proto.CompactTextString(m) } -func (*SearchResponse) ProtoMessage() {} -func (*SearchResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_7b2daf652facb3ae, []int{4} -} - -func (m *SearchResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SearchResponse.Unmarshal(m, b) -} -func (m *SearchResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SearchResponse.Marshal(b, m, deterministic) -} -func (m *SearchResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_SearchResponse.Merge(m, src) -} -func (m *SearchResponse) XXX_Size() int { - return xxx_messageInfo_SearchResponse.Size(m) -} -func (m *SearchResponse) XXX_DiscardUnknown() { - xxx_messageInfo_SearchResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_SearchResponse proto.InternalMessageInfo - -func (m *SearchResponse) GetSearchResult() *any.Any { - if m != nil { - return m.SearchResult - } - return nil -} - -type IndexCommand struct { - Type IndexCommand_Type `protobuf:"varint,1,opt,name=type,proto3,enum=index.IndexCommand_Type" json:"type,omitempty"` - Data *any.Any `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *IndexCommand) Reset() { *m = IndexCommand{} } -func (m *IndexCommand) String() string { return proto.CompactTextString(m) } -func (*IndexCommand) ProtoMessage() {} -func (*IndexCommand) Descriptor() ([]byte, []int) { - return fileDescriptor_7b2daf652facb3ae, []int{5} -} - -func (m *IndexCommand) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_IndexCommand.Unmarshal(m, b) -} -func (m *IndexCommand) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_IndexCommand.Marshal(b, m, deterministic) -} -func (m *IndexCommand) XXX_Merge(src proto.Message) { - xxx_messageInfo_IndexCommand.Merge(m, src) -} -func (m *IndexCommand) XXX_Size() int { - return xxx_messageInfo_IndexCommand.Size(m) -} -func (m *IndexCommand) XXX_DiscardUnknown() { - xxx_messageInfo_IndexCommand.DiscardUnknown(m) -} - -var xxx_messageInfo_IndexCommand proto.InternalMessageInfo - -func (m *IndexCommand) GetType() IndexCommand_Type { - if m != nil { - return m.Type - } - return IndexCommand_UNKNOWN_COMMAND -} - -func (m *IndexCommand) GetData() *any.Any { - if m != nil { - return m.Data - } - return nil -} - -func init() { - proto.RegisterEnum("index.IndexCommand_Type", IndexCommand_Type_name, IndexCommand_Type_value) - proto.RegisterType((*Document)(nil), "index.Document") - proto.RegisterType((*UpdateResult)(nil), "index.UpdateResult") - proto.RegisterType((*Stats)(nil), "index.Stats") - proto.RegisterType((*SearchRequest)(nil), "index.SearchRequest") - proto.RegisterType((*SearchResponse)(nil), "index.SearchResponse") - proto.RegisterType((*IndexCommand)(nil), "index.IndexCommand") -} - -func init() { proto.RegisterFile("protobuf/index/index.proto", fileDescriptor_7b2daf652facb3ae) } - -var fileDescriptor_7b2daf652facb3ae = []byte{ - // 567 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x94, 0x51, 0x6f, 0x12, 0x41, - 0x10, 0xc7, 0x0f, 0xca, 0x61, 0x1d, 0x81, 0x92, 0x6d, 0x35, 0x78, 0xbe, 0x34, 0x1b, 0x63, 0xd0, - 0xe8, 0x11, 0x69, 0x8c, 0x31, 0xfa, 0x82, 0xdc, 0x05, 0x6b, 0xcb, 0x35, 0x01, 0x1a, 0x8d, 0x2f, - 0x64, 0xe1, 0xa6, 0xed, 0xa5, 0x77, 0xb7, 0x27, 0xbb, 0x67, 0xe4, 0x23, 0xf8, 0xf9, 0xfc, 0x42, - 0xe6, 0x76, 0x39, 0xa4, 0x6d, 0x8e, 0xe8, 0xcb, 0x86, 0xfd, 0xcf, 0x6f, 0xfe, 0x0c, 0xb3, 0x33, - 0x80, 0x95, 0x2c, 0xb8, 0xe4, 0xb3, 0xf4, 0xa2, 0x13, 0xc4, 0x3e, 0xfe, 0xd4, 0xa7, 0xad, 0x44, - 0x62, 0xaa, 0x8b, 0xf5, 0xf8, 0x92, 0xf3, 0xcb, 0x10, 0x3b, 0x6b, 0x92, 0xc5, 0x4b, 0x4d, 0x58, - 0x4f, 0x6e, 0x87, 0x30, 0x4a, 0x64, 0x1e, 0x6c, 0xad, 0xd5, 0x05, 0xbb, 0x90, 0xea, 0xd0, 0x11, - 0xfa, 0x09, 0x76, 0x1d, 0x3e, 0x4f, 0x23, 0x8c, 0x25, 0x69, 0x40, 0x39, 0xf0, 0x5b, 0xa5, 0xc3, - 0x52, 0xfb, 0xfe, 0xa8, 0x1c, 0xf8, 0xe4, 0x25, 0x54, 0x2f, 0x02, 0x0c, 0x7d, 0xd1, 0x2a, 0x1f, - 0x96, 0xda, 0x0f, 0xba, 0x07, 0xb6, 0xfe, 0x0e, 0x3b, 0x77, 0xb3, 0x7b, 0xf1, 0x72, 0xb4, 0x62, - 0xe8, 0x53, 0xa8, 0x9d, 0x27, 0x3e, 0x93, 0x38, 0x42, 0x91, 0x86, 0x92, 0x1c, 0x80, 0x39, 0xe7, - 0x69, 0x2c, 0x95, 0xa1, 0x39, 0xd2, 0x17, 0x7a, 0x04, 0xe6, 0x58, 0x32, 0x29, 0xc8, 0x0b, 0x30, - 0x45, 0xf6, 0x41, 0x85, 0x8b, 0xbc, 0x35, 0x42, 0x4f, 0xa1, 0x3e, 0x46, 0xb6, 0x98, 0x5f, 0x8d, - 0xf0, 0x7b, 0x8a, 0x42, 0x92, 0xf7, 0xd0, 0x10, 0x4a, 0x98, 0x2e, 0xb4, 0xb2, 0xd5, 0xa5, 0x2e, - 0x36, 0x93, 0xe9, 0x09, 0x34, 0x72, 0x37, 0x91, 0xf0, 0x58, 0x20, 0x79, 0x07, 0xf5, 0xb5, 0x5d, - 0x56, 0xfb, 0x56, 0xb7, 0x5a, 0xee, 0x96, 0x91, 0xf4, 0x77, 0x09, 0x6a, 0xc7, 0xd9, 0xdb, 0xf4, - 0x79, 0x14, 0xb1, 0x38, 0x6b, 0x5a, 0x45, 0x2e, 0x13, 0x54, 0x16, 0x8d, 0x6e, 0xcb, 0xd6, 0xaf, - 0xb8, 0x89, 0xd8, 0x93, 0x65, 0x82, 0x23, 0x45, 0x91, 0x36, 0x54, 0x7c, 0x26, 0xd9, 0xd6, 0x06, - 0x2b, 0x82, 0x5e, 0x43, 0x25, 0xcb, 0x23, 0xfb, 0xb0, 0x77, 0xee, 0x9d, 0x78, 0x67, 0x5f, 0xbc, - 0x69, 0xff, 0x6c, 0x38, 0xec, 0x79, 0x4e, 0xd3, 0x20, 0x4d, 0xa8, 0x8d, 0xdd, 0xc9, 0x74, 0xe8, - 0x4e, 0x7a, 0x4e, 0x6f, 0xd2, 0x6b, 0x96, 0x32, 0xcc, 0x71, 0x4f, 0xdd, 0x89, 0xfb, 0x57, 0x2c, - 0x13, 0x02, 0x8d, 0x63, 0xcf, 0x71, 0xbf, 0x4e, 0x9d, 0xb3, 0xfe, 0xf9, 0xd0, 0xf5, 0x26, 0xcd, - 0x9d, 0x0d, 0x70, 0x2d, 0x56, 0xba, 0xbf, 0x2a, 0x60, 0xaa, 0x92, 0xb3, 0x9f, 0xf3, 0x99, 0x07, - 0x31, 0x01, 0x5b, 0x0d, 0x8d, 0xc7, 0x7d, 0xb4, 0x1e, 0xdd, 0x29, 0xd3, 0xcd, 0x66, 0x8d, 0x1a, - 0xe4, 0x15, 0x98, 0xa7, 0xc8, 0x7e, 0xe0, 0x3f, 0xe2, 0x1d, 0xb8, 0x37, 0x40, 0x99, 0x41, 0xa4, - 0x00, 0xb2, 0x36, 0x8c, 0xa8, 0x41, 0xde, 0x00, 0x0c, 0x50, 0xf6, 0xc3, 0x54, 0x48, 0x5c, 0x14, - 0xe6, 0xd4, 0x75, 0xce, 0x0a, 0xa3, 0x06, 0xf9, 0x00, 0xbb, 0xe3, 0x98, 0x25, 0xe2, 0x8a, 0xcb, - 0xc2, 0xa4, 0xe2, 0x2a, 0x9f, 0xc3, 0xce, 0x00, 0x25, 0xd9, 0x5b, 0x3d, 0x65, 0xbe, 0x2e, 0xd6, - 0x6d, 0x81, 0x1a, 0xe4, 0x75, 0xde, 0xb6, 0x3b, 0xf0, 0xfe, 0x4a, 0xd8, 0x5c, 0x11, 0x6a, 0xb4, - 0x4b, 0xa4, 0x0b, 0x55, 0x07, 0x43, 0x94, 0xf8, 0x1f, 0x39, 0x6f, 0xa1, 0xaa, 0x27, 0x98, 0x1c, - 0xac, 0x90, 0x1b, 0xeb, 0x61, 0x3d, 0xbc, 0xa5, 0xea, 0x31, 0xa7, 0x06, 0xe9, 0xc2, 0xee, 0x00, - 0xa5, 0x5e, 0xc0, 0xa2, 0x46, 0xd4, 0xf2, 0x64, 0xb5, 0x7a, 0xc6, 0xc7, 0xf6, 0xb7, 0x67, 0x97, - 0x81, 0xbc, 0x4a, 0x67, 0xf6, 0x9c, 0x47, 0x9d, 0x88, 0x8b, 0xf4, 0x9a, 0x75, 0x66, 0x21, 0x13, - 0xb2, 0x73, 0xf3, 0x0f, 0x6b, 0x56, 0x55, 0xf7, 0xa3, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x8d, - 0xc7, 0x8d, 0x5b, 0xc9, 0x04, 0x00, 0x00, -} - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConn - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion4 - -// IndexClient is the client API for Index service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type IndexClient interface { - Join(ctx context.Context, in *raft.Node, opts ...grpc.CallOption) (*empty.Empty, error) - Leave(ctx context.Context, in *raft.Node, opts ...grpc.CallOption) (*empty.Empty, error) - GetNode(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*raft.Node, error) - GetCluster(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*raft.Cluster, error) - Snapshot(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) - Get(ctx context.Context, in *Document, opts ...grpc.CallOption) (*Document, error) - Index(ctx context.Context, opts ...grpc.CallOption) (Index_IndexClient, error) - Delete(ctx context.Context, opts ...grpc.CallOption) (Index_DeleteClient, error) - Search(ctx context.Context, in *SearchRequest, opts ...grpc.CallOption) (*SearchResponse, error) - GetStats(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*Stats, error) -} - -type indexClient struct { - cc *grpc.ClientConn -} - -func NewIndexClient(cc *grpc.ClientConn) IndexClient { - return &indexClient{cc} -} - -func (c *indexClient) Join(ctx context.Context, in *raft.Node, opts ...grpc.CallOption) (*empty.Empty, error) { - out := new(empty.Empty) - err := c.cc.Invoke(ctx, "/index.Index/Join", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *indexClient) Leave(ctx context.Context, in *raft.Node, opts ...grpc.CallOption) (*empty.Empty, error) { - out := new(empty.Empty) - err := c.cc.Invoke(ctx, "/index.Index/Leave", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *indexClient) GetNode(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*raft.Node, error) { - out := new(raft.Node) - err := c.cc.Invoke(ctx, "/index.Index/GetNode", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *indexClient) GetCluster(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*raft.Cluster, error) { - out := new(raft.Cluster) - err := c.cc.Invoke(ctx, "/index.Index/GetCluster", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *indexClient) Snapshot(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) { - out := new(empty.Empty) - err := c.cc.Invoke(ctx, "/index.Index/Snapshot", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *indexClient) Get(ctx context.Context, in *Document, opts ...grpc.CallOption) (*Document, error) { - out := new(Document) - err := c.cc.Invoke(ctx, "/index.Index/Get", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *indexClient) Index(ctx context.Context, opts ...grpc.CallOption) (Index_IndexClient, error) { - stream, err := c.cc.NewStream(ctx, &_Index_serviceDesc.Streams[0], "/index.Index/Index", opts...) - if err != nil { - return nil, err - } - x := &indexIndexClient{stream} - return x, nil -} - -type Index_IndexClient interface { - Send(*Document) error - CloseAndRecv() (*UpdateResult, error) - grpc.ClientStream -} - -type indexIndexClient struct { - grpc.ClientStream -} - -func (x *indexIndexClient) Send(m *Document) error { - return x.ClientStream.SendMsg(m) -} - -func (x *indexIndexClient) CloseAndRecv() (*UpdateResult, error) { - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - m := new(UpdateResult) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *indexClient) Delete(ctx context.Context, opts ...grpc.CallOption) (Index_DeleteClient, error) { - stream, err := c.cc.NewStream(ctx, &_Index_serviceDesc.Streams[1], "/index.Index/Delete", opts...) - if err != nil { - return nil, err - } - x := &indexDeleteClient{stream} - return x, nil -} - -type Index_DeleteClient interface { - Send(*Document) error - CloseAndRecv() (*UpdateResult, error) - grpc.ClientStream -} - -type indexDeleteClient struct { - grpc.ClientStream -} - -func (x *indexDeleteClient) Send(m *Document) error { - return x.ClientStream.SendMsg(m) -} - -func (x *indexDeleteClient) CloseAndRecv() (*UpdateResult, error) { - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - m := new(UpdateResult) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *indexClient) Search(ctx context.Context, in *SearchRequest, opts ...grpc.CallOption) (*SearchResponse, error) { - out := new(SearchResponse) - err := c.cc.Invoke(ctx, "/index.Index/Search", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *indexClient) GetStats(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*Stats, error) { - out := new(Stats) - err := c.cc.Invoke(ctx, "/index.Index/GetStats", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// IndexServer is the server API for Index service. -type IndexServer interface { - Join(context.Context, *raft.Node) (*empty.Empty, error) - Leave(context.Context, *raft.Node) (*empty.Empty, error) - GetNode(context.Context, *empty.Empty) (*raft.Node, error) - GetCluster(context.Context, *empty.Empty) (*raft.Cluster, error) - Snapshot(context.Context, *empty.Empty) (*empty.Empty, error) - Get(context.Context, *Document) (*Document, error) - Index(Index_IndexServer) error - Delete(Index_DeleteServer) error - Search(context.Context, *SearchRequest) (*SearchResponse, error) - GetStats(context.Context, *empty.Empty) (*Stats, error) -} - -func RegisterIndexServer(s *grpc.Server, srv IndexServer) { - s.RegisterService(&_Index_serviceDesc, srv) -} - -func _Index_Join_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(raft.Node) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(IndexServer).Join(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/index.Index/Join", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(IndexServer).Join(ctx, req.(*raft.Node)) - } - return interceptor(ctx, in, info, handler) -} - -func _Index_Leave_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(raft.Node) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(IndexServer).Leave(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/index.Index/Leave", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(IndexServer).Leave(ctx, req.(*raft.Node)) - } - return interceptor(ctx, in, info, handler) -} - -func _Index_GetNode_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(empty.Empty) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(IndexServer).GetNode(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/index.Index/GetNode", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(IndexServer).GetNode(ctx, req.(*empty.Empty)) - } - return interceptor(ctx, in, info, handler) -} - -func _Index_GetCluster_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(empty.Empty) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(IndexServer).GetCluster(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/index.Index/GetCluster", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(IndexServer).GetCluster(ctx, req.(*empty.Empty)) - } - return interceptor(ctx, in, info, handler) -} - -func _Index_Snapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(empty.Empty) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(IndexServer).Snapshot(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/index.Index/Snapshot", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(IndexServer).Snapshot(ctx, req.(*empty.Empty)) - } - return interceptor(ctx, in, info, handler) -} - -func _Index_Get_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(Document) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(IndexServer).Get(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/index.Index/Get", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(IndexServer).Get(ctx, req.(*Document)) - } - return interceptor(ctx, in, info, handler) -} - -func _Index_Index_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(IndexServer).Index(&indexIndexServer{stream}) -} - -type Index_IndexServer interface { - SendAndClose(*UpdateResult) error - Recv() (*Document, error) - grpc.ServerStream -} - -type indexIndexServer struct { - grpc.ServerStream -} - -func (x *indexIndexServer) SendAndClose(m *UpdateResult) error { - return x.ServerStream.SendMsg(m) -} - -func (x *indexIndexServer) Recv() (*Document, error) { - m := new(Document) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func _Index_Delete_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(IndexServer).Delete(&indexDeleteServer{stream}) -} - -type Index_DeleteServer interface { - SendAndClose(*UpdateResult) error - Recv() (*Document, error) - grpc.ServerStream -} - -type indexDeleteServer struct { - grpc.ServerStream -} - -func (x *indexDeleteServer) SendAndClose(m *UpdateResult) error { - return x.ServerStream.SendMsg(m) -} - -func (x *indexDeleteServer) Recv() (*Document, error) { - m := new(Document) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func _Index_Search_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(SearchRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(IndexServer).Search(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/index.Index/Search", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(IndexServer).Search(ctx, req.(*SearchRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Index_GetStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(empty.Empty) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(IndexServer).GetStats(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/index.Index/GetStats", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(IndexServer).GetStats(ctx, req.(*empty.Empty)) - } - return interceptor(ctx, in, info, handler) -} - -var _Index_serviceDesc = grpc.ServiceDesc{ - ServiceName: "index.Index", - HandlerType: (*IndexServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "Join", - Handler: _Index_Join_Handler, - }, - { - MethodName: "Leave", - Handler: _Index_Leave_Handler, - }, - { - MethodName: "GetNode", - Handler: _Index_GetNode_Handler, - }, - { - MethodName: "GetCluster", - Handler: _Index_GetCluster_Handler, - }, - { - MethodName: "Snapshot", - Handler: _Index_Snapshot_Handler, - }, - { - MethodName: "Get", - Handler: _Index_Get_Handler, - }, - { - MethodName: "Search", - Handler: _Index_Search_Handler, - }, - { - MethodName: "GetStats", - Handler: _Index_GetStats_Handler, - }, - }, - Streams: []grpc.StreamDesc{ - { - StreamName: "Index", - Handler: _Index_Index_Handler, - ClientStreams: true, - }, - { - StreamName: "Delete", - Handler: _Index_Delete_Handler, - ClientStreams: true, - }, - }, - Metadata: "protobuf/index/index.proto", -} diff --git a/protobuf/index/index.proto b/protobuf/index/index.proto deleted file mode 100644 index d369ae1..0000000 --- a/protobuf/index/index.proto +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) 2019 Minoru Osuka -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -import "google/protobuf/any.proto"; -import "google/protobuf/empty.proto"; -import "protobuf/raft/raft.proto"; - -package index; - -option go_package = "github.com/mosuka/blast/protobuf/index"; - -service Index { - rpc Join (raft.Node) returns (google.protobuf.Empty) {} - rpc Leave (raft.Node) returns (google.protobuf.Empty) {} - rpc GetNode (google.protobuf.Empty) returns (raft.Node) {} - rpc GetCluster (google.protobuf.Empty) returns (raft.Cluster) {} - rpc Snapshot (google.protobuf.Empty) returns (google.protobuf.Empty) {} - - rpc Get (Document) returns (Document) {} - rpc Index (stream Document) returns (UpdateResult) {} - rpc Delete (stream Document) returns (UpdateResult) {} - rpc Search (SearchRequest) returns (SearchResponse) {} - - rpc GetStats (google.protobuf.Empty) returns (Stats) {} -} - -message Document { - string id = 1; - google.protobuf.Any fields = 2; -} - -message UpdateResult { - int32 count = 1; -} - -message Stats { - google.protobuf.Any stats = 1; -} - -message SearchRequest { - google.protobuf.Any search_request = 1; -} - -message SearchResponse { - google.protobuf.Any search_result = 1; -} - -message IndexCommand { - enum Type { - UNKNOWN_COMMAND = 0; - SET_METADATA = 1; - DELETE_METADATA = 2; - INDEX_DOCUMENT = 3; - DELETE_DOCUMENT = 4; - } - Type type = 1; - google.protobuf.Any data = 2; -} diff --git a/protobuf/management/management.pb.go b/protobuf/management/management.pb.go deleted file mode 100644 index 5477d6f..0000000 --- a/protobuf/management/management.pb.go +++ /dev/null @@ -1,499 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// source: protobuf/management/management.proto - -package management - -import ( - context "context" - fmt "fmt" - proto "github.com/golang/protobuf/proto" - any "github.com/golang/protobuf/ptypes/any" - empty "github.com/golang/protobuf/ptypes/empty" - raft "github.com/mosuka/blast/protobuf/raft" - grpc "google.golang.org/grpc" - math "math" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package - -type ManagementCommand_Type int32 - -const ( - ManagementCommand_UNKNOWN_COMMAND ManagementCommand_Type = 0 - ManagementCommand_SET_METADATA ManagementCommand_Type = 1 - ManagementCommand_DELETE_METADATA ManagementCommand_Type = 2 - ManagementCommand_PUT_KEY_VALUE_PAIR ManagementCommand_Type = 3 - ManagementCommand_DELETE_KEY_VALUE_PAIR ManagementCommand_Type = 4 -) - -var ManagementCommand_Type_name = map[int32]string{ - 0: "UNKNOWN_COMMAND", - 1: "SET_METADATA", - 2: "DELETE_METADATA", - 3: "PUT_KEY_VALUE_PAIR", - 4: "DELETE_KEY_VALUE_PAIR", -} - -var ManagementCommand_Type_value = map[string]int32{ - "UNKNOWN_COMMAND": 0, - "SET_METADATA": 1, - "DELETE_METADATA": 2, - "PUT_KEY_VALUE_PAIR": 3, - "DELETE_KEY_VALUE_PAIR": 4, -} - -func (x ManagementCommand_Type) String() string { - return proto.EnumName(ManagementCommand_Type_name, int32(x)) -} - -func (ManagementCommand_Type) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_5e030ad796566078, []int{1, 0} -} - -type KeyValuePair struct { - Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` - Value *any.Any `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *KeyValuePair) Reset() { *m = KeyValuePair{} } -func (m *KeyValuePair) String() string { return proto.CompactTextString(m) } -func (*KeyValuePair) ProtoMessage() {} -func (*KeyValuePair) Descriptor() ([]byte, []int) { - return fileDescriptor_5e030ad796566078, []int{0} -} - -func (m *KeyValuePair) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_KeyValuePair.Unmarshal(m, b) -} -func (m *KeyValuePair) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_KeyValuePair.Marshal(b, m, deterministic) -} -func (m *KeyValuePair) XXX_Merge(src proto.Message) { - xxx_messageInfo_KeyValuePair.Merge(m, src) -} -func (m *KeyValuePair) XXX_Size() int { - return xxx_messageInfo_KeyValuePair.Size(m) -} -func (m *KeyValuePair) XXX_DiscardUnknown() { - xxx_messageInfo_KeyValuePair.DiscardUnknown(m) -} - -var xxx_messageInfo_KeyValuePair proto.InternalMessageInfo - -func (m *KeyValuePair) GetKey() string { - if m != nil { - return m.Key - } - return "" -} - -func (m *KeyValuePair) GetValue() *any.Any { - if m != nil { - return m.Value - } - return nil -} - -type ManagementCommand struct { - Type ManagementCommand_Type `protobuf:"varint,1,opt,name=type,proto3,enum=management.ManagementCommand_Type" json:"type,omitempty"` - Data *any.Any `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ManagementCommand) Reset() { *m = ManagementCommand{} } -func (m *ManagementCommand) String() string { return proto.CompactTextString(m) } -func (*ManagementCommand) ProtoMessage() {} -func (*ManagementCommand) Descriptor() ([]byte, []int) { - return fileDescriptor_5e030ad796566078, []int{1} -} - -func (m *ManagementCommand) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ManagementCommand.Unmarshal(m, b) -} -func (m *ManagementCommand) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ManagementCommand.Marshal(b, m, deterministic) -} -func (m *ManagementCommand) XXX_Merge(src proto.Message) { - xxx_messageInfo_ManagementCommand.Merge(m, src) -} -func (m *ManagementCommand) XXX_Size() int { - return xxx_messageInfo_ManagementCommand.Size(m) -} -func (m *ManagementCommand) XXX_DiscardUnknown() { - xxx_messageInfo_ManagementCommand.DiscardUnknown(m) -} - -var xxx_messageInfo_ManagementCommand proto.InternalMessageInfo - -func (m *ManagementCommand) GetType() ManagementCommand_Type { - if m != nil { - return m.Type - } - return ManagementCommand_UNKNOWN_COMMAND -} - -func (m *ManagementCommand) GetData() *any.Any { - if m != nil { - return m.Data - } - return nil -} - -func init() { - proto.RegisterEnum("management.ManagementCommand_Type", ManagementCommand_Type_name, ManagementCommand_Type_value) - proto.RegisterType((*KeyValuePair)(nil), "management.KeyValuePair") - proto.RegisterType((*ManagementCommand)(nil), "management.ManagementCommand") -} - -func init() { - proto.RegisterFile("protobuf/management/management.proto", fileDescriptor_5e030ad796566078) -} - -var fileDescriptor_5e030ad796566078 = []byte{ - // 448 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x93, 0x51, 0x6f, 0x93, 0x50, - 0x14, 0xc7, 0x61, 0xb0, 0xa9, 0xc7, 0xa9, 0x78, 0xd4, 0xa5, 0xc3, 0x97, 0x85, 0xf8, 0xd0, 0xa8, - 0x83, 0xa4, 0x46, 0x13, 0xe3, 0x5e, 0xb0, 0xbd, 0x69, 0xb4, 0x2d, 0x6b, 0x5a, 0x3a, 0xa3, 0x2f, - 0xe4, 0xd6, 0x9e, 0x75, 0xcd, 0x80, 0x4b, 0xda, 0xcb, 0x12, 0x3e, 0xa0, 0xdf, 0xc6, 0x0f, 0x61, - 0x80, 0xad, 0x25, 0x9b, 0x34, 0xc6, 0x17, 0x72, 0xf3, 0xff, 0xff, 0xfe, 0xe7, 0x9c, 0xcb, 0x01, - 0x78, 0x95, 0x2c, 0x85, 0x14, 0xd3, 0xf4, 0xdc, 0x89, 0x78, 0xcc, 0xe7, 0x14, 0x51, 0x2c, 0x2b, - 0x47, 0xbb, 0xb0, 0x11, 0x36, 0x8a, 0x79, 0x38, 0x17, 0x62, 0x1e, 0x92, 0xb3, 0x0e, 0xf2, 0x38, - 0x2b, 0x31, 0xf3, 0xe5, 0x6d, 0x8b, 0xa2, 0x44, 0xde, 0x98, 0x8d, 0xb5, 0xba, 0xe4, 0xe7, 0xb2, - 0x78, 0x94, 0x8e, 0xd5, 0x87, 0xfd, 0x1e, 0x65, 0x67, 0x3c, 0x4c, 0x69, 0xc8, 0x17, 0x4b, 0x34, - 0x40, 0xbb, 0xa4, 0xac, 0xa1, 0x1e, 0xa9, 0xcd, 0x07, 0xa3, 0xfc, 0x88, 0xaf, 0x61, 0xf7, 0x2a, - 0xb7, 0x1b, 0x3b, 0x47, 0x6a, 0xf3, 0x61, 0xeb, 0xb9, 0x5d, 0x36, 0xb2, 0x6f, 0x4a, 0xda, 0x6e, - 0x9c, 0x8d, 0x4a, 0xc4, 0xfa, 0xad, 0xc2, 0xd3, 0xc1, 0x7a, 0xdc, 0xb6, 0x88, 0x22, 0x1e, 0xcf, - 0xf0, 0x03, 0xe8, 0x32, 0x4b, 0xa8, 0x28, 0xfa, 0xb8, 0x65, 0xd9, 0x95, 0x2b, 0xde, 0x81, 0x6d, - 0x3f, 0x4b, 0x68, 0x54, 0xf0, 0xd8, 0x04, 0x7d, 0xc6, 0x25, 0xdf, 0xda, 0xb8, 0x20, 0xac, 0x14, - 0xf4, 0x3c, 0x87, 0xcf, 0xe0, 0xc9, 0xc4, 0xeb, 0x79, 0xa7, 0xdf, 0xbc, 0xa0, 0x7d, 0x3a, 0x18, - 0xb8, 0x5e, 0xc7, 0x50, 0xd0, 0x80, 0xfd, 0x31, 0xf3, 0x83, 0x01, 0xf3, 0xdd, 0x8e, 0xeb, 0xbb, - 0x86, 0x9a, 0x63, 0x1d, 0xd6, 0x67, 0x3e, 0xdb, 0x88, 0x3b, 0x78, 0x00, 0x38, 0x9c, 0xf8, 0x41, - 0x8f, 0x7d, 0x0f, 0xce, 0xdc, 0xfe, 0x84, 0x05, 0x43, 0xf7, 0xcb, 0xc8, 0xd0, 0xf0, 0x10, 0x5e, - 0x5c, 0xc3, 0xb7, 0x2c, 0xbd, 0xf5, 0x4b, 0x03, 0xd8, 0xdc, 0x00, 0xdf, 0x82, 0xfe, 0x55, 0x2c, - 0x62, 0x04, 0xbb, 0x78, 0xc1, 0x9e, 0x98, 0x91, 0x79, 0x70, 0x67, 0x6a, 0x96, 0xef, 0xc5, 0x52, - 0xf0, 0x18, 0x76, 0xfb, 0xc4, 0xaf, 0xe8, 0x1f, 0x71, 0x07, 0xee, 0x75, 0x49, 0xe6, 0x10, 0xd6, - 0x40, 0x66, 0xa5, 0x90, 0xa5, 0xe0, 0x7b, 0x80, 0x2e, 0xc9, 0x76, 0x98, 0xae, 0x24, 0x2d, 0x6b, - 0x33, 0x8f, 0xca, 0xcc, 0x35, 0x66, 0x29, 0x78, 0x02, 0xf7, 0xc7, 0x31, 0x4f, 0x56, 0x17, 0x42, - 0xd6, 0x86, 0xea, 0xa7, 0xfc, 0x04, 0x5a, 0x97, 0x24, 0x36, 0xaa, 0x3b, 0xae, 0x7e, 0x5f, 0x66, - 0xad, 0x63, 0x29, 0xf8, 0x11, 0xb4, 0xf1, 0xd6, 0x70, 0x7d, 0xdf, 0x13, 0xd8, 0xeb, 0x50, 0x48, - 0x92, 0xfe, 0x27, 0xfd, 0xf9, 0xf8, 0xc7, 0x9b, 0xf9, 0x42, 0x5e, 0xa4, 0x53, 0xfb, 0xa7, 0x88, - 0x9c, 0x48, 0xac, 0xd2, 0x4b, 0xee, 0x4c, 0x43, 0xbe, 0x92, 0xce, 0x5f, 0x7e, 0xd1, 0xe9, 0x5e, - 0x21, 0xbe, 0xfb, 0x13, 0x00, 0x00, 0xff, 0xff, 0x4e, 0x64, 0x5d, 0xf3, 0xc0, 0x03, 0x00, 0x00, -} - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConn - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion4 - -// ManagementClient is the client API for Management service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type ManagementClient interface { - Join(ctx context.Context, in *raft.Node, opts ...grpc.CallOption) (*empty.Empty, error) - Leave(ctx context.Context, in *raft.Node, opts ...grpc.CallOption) (*empty.Empty, error) - GetNode(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*raft.Node, error) - GetCluster(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*raft.Cluster, error) - Snapshot(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) - Get(ctx context.Context, in *KeyValuePair, opts ...grpc.CallOption) (*KeyValuePair, error) - Set(ctx context.Context, in *KeyValuePair, opts ...grpc.CallOption) (*empty.Empty, error) - Delete(ctx context.Context, in *KeyValuePair, opts ...grpc.CallOption) (*empty.Empty, error) -} - -type managementClient struct { - cc *grpc.ClientConn -} - -func NewManagementClient(cc *grpc.ClientConn) ManagementClient { - return &managementClient{cc} -} - -func (c *managementClient) Join(ctx context.Context, in *raft.Node, opts ...grpc.CallOption) (*empty.Empty, error) { - out := new(empty.Empty) - err := c.cc.Invoke(ctx, "/management.Management/Join", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *managementClient) Leave(ctx context.Context, in *raft.Node, opts ...grpc.CallOption) (*empty.Empty, error) { - out := new(empty.Empty) - err := c.cc.Invoke(ctx, "/management.Management/Leave", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *managementClient) GetNode(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*raft.Node, error) { - out := new(raft.Node) - err := c.cc.Invoke(ctx, "/management.Management/GetNode", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *managementClient) GetCluster(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*raft.Cluster, error) { - out := new(raft.Cluster) - err := c.cc.Invoke(ctx, "/management.Management/GetCluster", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *managementClient) Snapshot(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) { - out := new(empty.Empty) - err := c.cc.Invoke(ctx, "/management.Management/Snapshot", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *managementClient) Get(ctx context.Context, in *KeyValuePair, opts ...grpc.CallOption) (*KeyValuePair, error) { - out := new(KeyValuePair) - err := c.cc.Invoke(ctx, "/management.Management/Get", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *managementClient) Set(ctx context.Context, in *KeyValuePair, opts ...grpc.CallOption) (*empty.Empty, error) { - out := new(empty.Empty) - err := c.cc.Invoke(ctx, "/management.Management/Set", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *managementClient) Delete(ctx context.Context, in *KeyValuePair, opts ...grpc.CallOption) (*empty.Empty, error) { - out := new(empty.Empty) - err := c.cc.Invoke(ctx, "/management.Management/Delete", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// ManagementServer is the server API for Management service. -type ManagementServer interface { - Join(context.Context, *raft.Node) (*empty.Empty, error) - Leave(context.Context, *raft.Node) (*empty.Empty, error) - GetNode(context.Context, *empty.Empty) (*raft.Node, error) - GetCluster(context.Context, *empty.Empty) (*raft.Cluster, error) - Snapshot(context.Context, *empty.Empty) (*empty.Empty, error) - Get(context.Context, *KeyValuePair) (*KeyValuePair, error) - Set(context.Context, *KeyValuePair) (*empty.Empty, error) - Delete(context.Context, *KeyValuePair) (*empty.Empty, error) -} - -func RegisterManagementServer(s *grpc.Server, srv ManagementServer) { - s.RegisterService(&_Management_serviceDesc, srv) -} - -func _Management_Join_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(raft.Node) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(ManagementServer).Join(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/management.Management/Join", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ManagementServer).Join(ctx, req.(*raft.Node)) - } - return interceptor(ctx, in, info, handler) -} - -func _Management_Leave_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(raft.Node) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(ManagementServer).Leave(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/management.Management/Leave", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ManagementServer).Leave(ctx, req.(*raft.Node)) - } - return interceptor(ctx, in, info, handler) -} - -func _Management_GetNode_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(empty.Empty) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(ManagementServer).GetNode(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/management.Management/GetNode", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ManagementServer).GetNode(ctx, req.(*empty.Empty)) - } - return interceptor(ctx, in, info, handler) -} - -func _Management_GetCluster_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(empty.Empty) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(ManagementServer).GetCluster(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/management.Management/GetCluster", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ManagementServer).GetCluster(ctx, req.(*empty.Empty)) - } - return interceptor(ctx, in, info, handler) -} - -func _Management_Snapshot_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(empty.Empty) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(ManagementServer).Snapshot(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/management.Management/Snapshot", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ManagementServer).Snapshot(ctx, req.(*empty.Empty)) - } - return interceptor(ctx, in, info, handler) -} - -func _Management_Get_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(KeyValuePair) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(ManagementServer).Get(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/management.Management/Get", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ManagementServer).Get(ctx, req.(*KeyValuePair)) - } - return interceptor(ctx, in, info, handler) -} - -func _Management_Set_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(KeyValuePair) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(ManagementServer).Set(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/management.Management/Set", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ManagementServer).Set(ctx, req.(*KeyValuePair)) - } - return interceptor(ctx, in, info, handler) -} - -func _Management_Delete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(KeyValuePair) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(ManagementServer).Delete(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/management.Management/Delete", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ManagementServer).Delete(ctx, req.(*KeyValuePair)) - } - return interceptor(ctx, in, info, handler) -} - -var _Management_serviceDesc = grpc.ServiceDesc{ - ServiceName: "management.Management", - HandlerType: (*ManagementServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "Join", - Handler: _Management_Join_Handler, - }, - { - MethodName: "Leave", - Handler: _Management_Leave_Handler, - }, - { - MethodName: "GetNode", - Handler: _Management_GetNode_Handler, - }, - { - MethodName: "GetCluster", - Handler: _Management_GetCluster_Handler, - }, - { - MethodName: "Snapshot", - Handler: _Management_Snapshot_Handler, - }, - { - MethodName: "Get", - Handler: _Management_Get_Handler, - }, - { - MethodName: "Set", - Handler: _Management_Set_Handler, - }, - { - MethodName: "Delete", - Handler: _Management_Delete_Handler, - }, - }, - Streams: []grpc.StreamDesc{}, - Metadata: "protobuf/management/management.proto", -} diff --git a/protobuf/management/management.proto b/protobuf/management/management.proto deleted file mode 100644 index 6f4c19e..0000000 --- a/protobuf/management/management.proto +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) 2019 Minoru Osuka -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -import "google/protobuf/any.proto"; -import "google/protobuf/empty.proto"; -import "protobuf/raft/raft.proto"; - -package management; - -option go_package = "github.com/mosuka/blast/protobuf/management"; - -service Management { - rpc Join (raft.Node) returns (google.protobuf.Empty) {} - rpc Leave (raft.Node) returns (google.protobuf.Empty) {} - rpc GetNode (google.protobuf.Empty) returns (raft.Node) {} - rpc GetCluster (google.protobuf.Empty) returns (raft.Cluster) {} - rpc Snapshot (google.protobuf.Empty) returns (google.protobuf.Empty) {} - - rpc Get (KeyValuePair) returns (KeyValuePair) {} - rpc Set (KeyValuePair) returns (google.protobuf.Empty) {} - rpc Delete (KeyValuePair) returns (google.protobuf.Empty) {} -} - -message KeyValuePair { - string key = 1; - google.protobuf.Any value = 2; -} - -message ManagementCommand { - enum Type { - UNKNOWN_COMMAND = 0; - SET_METADATA = 1; - DELETE_METADATA = 2; - PUT_KEY_VALUE_PAIR = 3; - DELETE_KEY_VALUE_PAIR = 4; - } - Type type = 1; - google.protobuf.Any data = 2; -} - diff --git a/protobuf/raft/raft.pb.go b/protobuf/raft/raft.pb.go deleted file mode 100644 index e461106..0000000 --- a/protobuf/raft/raft.pb.go +++ /dev/null @@ -1,173 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// source: protobuf/raft/raft.proto - -package raft - -import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - math "math" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package - -type Node struct { - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` - BindAddr string `protobuf:"bytes,2,opt,name=bind_addr,json=bindAddr,proto3" json:"bind_addr,omitempty"` - GrpcAddr string `protobuf:"bytes,3,opt,name=grpc_addr,json=grpcAddr,proto3" json:"grpc_addr,omitempty"` - HttpAddr string `protobuf:"bytes,4,opt,name=http_addr,json=httpAddr,proto3" json:"http_addr,omitempty"` - Leader bool `protobuf:"varint,5,opt,name=leader,proto3" json:"leader,omitempty"` - DataDir string `protobuf:"bytes,6,opt,name=data_dir,json=dataDir,proto3" json:"data_dir,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Node) Reset() { *m = Node{} } -func (m *Node) String() string { return proto.CompactTextString(m) } -func (*Node) ProtoMessage() {} -func (*Node) Descriptor() ([]byte, []int) { - return fileDescriptor_028aa12295c796d4, []int{0} -} - -func (m *Node) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Node.Unmarshal(m, b) -} -func (m *Node) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Node.Marshal(b, m, deterministic) -} -func (m *Node) XXX_Merge(src proto.Message) { - xxx_messageInfo_Node.Merge(m, src) -} -func (m *Node) XXX_Size() int { - return xxx_messageInfo_Node.Size(m) -} -func (m *Node) XXX_DiscardUnknown() { - xxx_messageInfo_Node.DiscardUnknown(m) -} - -var xxx_messageInfo_Node proto.InternalMessageInfo - -func (m *Node) GetId() string { - if m != nil { - return m.Id - } - return "" -} - -func (m *Node) GetBindAddr() string { - if m != nil { - return m.BindAddr - } - return "" -} - -func (m *Node) GetGrpcAddr() string { - if m != nil { - return m.GrpcAddr - } - return "" -} - -func (m *Node) GetHttpAddr() string { - if m != nil { - return m.HttpAddr - } - return "" -} - -func (m *Node) GetLeader() bool { - if m != nil { - return m.Leader - } - return false -} - -func (m *Node) GetDataDir() string { - if m != nil { - return m.DataDir - } - return "" -} - -type Cluster struct { - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` - Nodes []*Node `protobuf:"bytes,2,rep,name=nodes,proto3" json:"nodes,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Cluster) Reset() { *m = Cluster{} } -func (m *Cluster) String() string { return proto.CompactTextString(m) } -func (*Cluster) ProtoMessage() {} -func (*Cluster) Descriptor() ([]byte, []int) { - return fileDescriptor_028aa12295c796d4, []int{1} -} - -func (m *Cluster) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Cluster.Unmarshal(m, b) -} -func (m *Cluster) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Cluster.Marshal(b, m, deterministic) -} -func (m *Cluster) XXX_Merge(src proto.Message) { - xxx_messageInfo_Cluster.Merge(m, src) -} -func (m *Cluster) XXX_Size() int { - return xxx_messageInfo_Cluster.Size(m) -} -func (m *Cluster) XXX_DiscardUnknown() { - xxx_messageInfo_Cluster.DiscardUnknown(m) -} - -var xxx_messageInfo_Cluster proto.InternalMessageInfo - -func (m *Cluster) GetId() string { - if m != nil { - return m.Id - } - return "" -} - -func (m *Cluster) GetNodes() []*Node { - if m != nil { - return m.Nodes - } - return nil -} - -func init() { - proto.RegisterType((*Node)(nil), "raft.Node") - proto.RegisterType((*Cluster)(nil), "raft.Cluster") -} - -func init() { proto.RegisterFile("protobuf/raft/raft.proto", fileDescriptor_028aa12295c796d4) } - -var fileDescriptor_028aa12295c796d4 = []byte{ - // 229 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x64, 0x90, 0x3d, 0x6b, 0xc3, 0x30, - 0x10, 0x86, 0xb1, 0xe3, 0x38, 0xce, 0x15, 0x3a, 0x78, 0x28, 0x2a, 0x5d, 0x4c, 0xa0, 0xd4, 0x93, - 0x05, 0xed, 0xd8, 0xa9, 0x1f, 0x73, 0x07, 0x8f, 0x5d, 0x82, 0x94, 0x53, 0x12, 0x51, 0x27, 0x32, - 0xa7, 0xd3, 0xff, 0xe9, 0x4f, 0x2d, 0x92, 0xda, 0xa1, 0x64, 0x11, 0x7a, 0x9f, 0x87, 0x83, 0x7b, - 0x0f, 0xc4, 0x4c, 0x8e, 0x9d, 0x0e, 0x7b, 0x49, 0x6a, 0xcf, 0xe9, 0x19, 0x12, 0x6a, 0xab, 0xf8, - 0xdf, 0x7c, 0x17, 0x50, 0x7d, 0x38, 0x34, 0xed, 0x35, 0x94, 0x16, 0x45, 0xd1, 0x15, 0xfd, 0x7a, - 0x2c, 0x2d, 0xb6, 0x77, 0xb0, 0xd6, 0xf6, 0x8c, 0x5b, 0x85, 0x48, 0xa2, 0x4c, 0xb8, 0x89, 0xe0, - 0x05, 0x91, 0xa2, 0x3c, 0xd0, 0xbc, 0xcb, 0x72, 0x91, 0x65, 0x04, 0x7f, 0xf2, 0xc8, 0x3c, 0x67, - 0x59, 0x65, 0x19, 0x41, 0x92, 0x37, 0x50, 0x4f, 0x46, 0xa1, 0x21, 0xb1, 0xec, 0x8a, 0xbe, 0x19, - 0x7f, 0x53, 0x7b, 0x0b, 0x0d, 0x2a, 0x56, 0x5b, 0xb4, 0x24, 0xea, 0x34, 0xb3, 0x8a, 0xf9, 0xdd, - 0xd2, 0xe6, 0x19, 0x56, 0x6f, 0x53, 0xf0, 0x6c, 0xe8, 0x62, 0xc9, 0x0e, 0x96, 0x67, 0x87, 0xc6, - 0x8b, 0xb2, 0x5b, 0xf4, 0x57, 0x8f, 0x30, 0xa4, 0x7e, 0xb1, 0xcf, 0x98, 0xc5, 0xeb, 0xc3, 0xe7, - 0xfd, 0xc1, 0xf2, 0x31, 0xe8, 0x61, 0xe7, 0x4e, 0xf2, 0xe4, 0x7c, 0xf8, 0x52, 0x52, 0x4f, 0xca, - 0xb3, 0xfc, 0x77, 0x19, 0x5d, 0xa7, 0xf8, 0xf4, 0x13, 0x00, 0x00, 0xff, 0xff, 0x1b, 0x84, 0x9b, - 0xf8, 0x31, 0x01, 0x00, 0x00, -} diff --git a/protobuf/util.go b/protobuf/util.go index 8163120..d3a6ca5 100644 --- a/protobuf/util.go +++ b/protobuf/util.go @@ -16,27 +16,11 @@ package protobuf import ( "encoding/json" - "reflect" - "github.com/blevesearch/bleve" "github.com/golang/protobuf/ptypes/any" - "github.com/mosuka/blast/protobuf/index" - "github.com/mosuka/blast/protobuf/management" - "github.com/mosuka/blast/protobuf/raft" "github.com/mosuka/blast/registry" ) -func init() { - registry.RegisterType("map[string]interface {}", reflect.TypeOf((map[string]interface{})(nil))) - - registry.RegisterType("management.KeyValuePair", reflect.TypeOf(management.KeyValuePair{})) - registry.RegisterType("index.Document", reflect.TypeOf(index.Document{})) - registry.RegisterType("raft.Node", reflect.TypeOf(raft.Node{})) - - registry.RegisterType("bleve.SearchRequest", reflect.TypeOf(bleve.SearchRequest{})) - registry.RegisterType("bleve.SearchResult", reflect.TypeOf(bleve.SearchResult{})) -} - func MarshalAny(message *any.Any) (interface{}, error) { if message == nil { return nil, nil @@ -56,17 +40,18 @@ func MarshalAny(message *any.Any) (interface{}, error) { } func UnmarshalAny(instance interface{}, message *any.Any) error { + var err error + if instance == nil { return nil } - value, err := json.Marshal(instance) + message.TypeUrl = registry.TypeNameByInstance(instance) + + message.Value, err = json.Marshal(instance) if err != nil { return err } - message.TypeUrl = registry.TypeNameByInstance(instance) - message.Value = value - return nil } diff --git a/protobuf/util_test.go b/protobuf/util_test.go index a9ca4f8..f8fb7e4 100644 --- a/protobuf/util_test.go +++ b/protobuf/util_test.go @@ -1,3 +1,17 @@ +// Copyright (c) 2019 Minoru Osuka +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package protobuf import ( @@ -7,242 +21,287 @@ import ( "github.com/blevesearch/bleve" "github.com/blevesearch/bleve/search/query" "github.com/golang/protobuf/ptypes/any" - "github.com/mosuka/blast/protobuf/index" - "github.com/mosuka/blast/protobuf/raft" ) -func TestMarshalAny(t *testing.T) { - // test map[string]interface{} - data := map[string]interface{}{"a": 1, "b": 2, "c": 3} +func TestMarshalAny_Slice(t *testing.T) { + data := []interface{}{"a", 1} - mapAny := &any.Any{} - err := UnmarshalAny(data, mapAny) + dataAny := &any.Any{} + err := UnmarshalAny(data, dataAny) if err != nil { t.Errorf("%v", err) } - expectedType := "map[string]interface {}" - actualType := mapAny.TypeUrl + expectedType := "[]interface {}" + actualType := dataAny.TypeUrl if expectedType != actualType { t.Errorf("expected content to see %s, saw %s", expectedType, actualType) } - expectedValue := []byte(`{"a":1,"b":2,"c":3}`) - actualValue := mapAny.Value + expectedValue := []byte(`["a",1]`) + actualValue := dataAny.Value if !bytes.Equal(expectedValue, actualValue) { t.Errorf("expected content to see %v, saw %v", expectedValue, actualValue) } +} - // test index.Document - fieldsMap := map[string]interface{}{"f1": "aaa", "f2": 222, "f3": "ccc"} - fieldsAny := &any.Any{} - err = UnmarshalAny(fieldsMap, fieldsAny) - if err != nil { - t.Errorf("%v", err) - } - - doc := &index.Document{ - Id: "1", - Fields: fieldsAny, - } +func TestMarshalAny_Map(t *testing.T) { + data := map[string]interface{}{"a": 1, "b": 2, "c": 3} - docAny := &any.Any{} - err = UnmarshalAny(doc, docAny) + dataAny := &any.Any{} + err := UnmarshalAny(data, dataAny) if err != nil { t.Errorf("%v", err) } - expectedType = "index.Document" - actualType = docAny.TypeUrl - if expectedType != actualType { - t.Errorf("expected content to see %s, saw %s", expectedType, actualType) + expectedMapType := "map[string]interface {}" + actualMapType := dataAny.TypeUrl + if expectedMapType != actualMapType { + t.Errorf("expected content to see %s, saw %s", expectedMapType, actualMapType) } - expectedValue = []byte(`{"id":"1","fields":{"type_url":"map[string]interface {}","value":"eyJmMSI6ImFhYSIsImYyIjoyMjIsImYzIjoiY2NjIn0="}}`) - actualValue = docAny.Value + expectedValue := []byte(`{"a":1,"b":2,"c":3}`) + actualValue := dataAny.Value if !bytes.Equal(expectedValue, actualValue) { t.Errorf("expected content to see %v, saw %v", expectedValue, actualValue) } +} - // test raft.Node - node := &raft.Node{ - Id: "node1", - GrpcAddr: ":5050", - DataDir: "/tmp/blast/index1", - BindAddr: ":6060", - HttpAddr: ":8080", - Leader: true, - } - - nodeAny := &any.Any{} - err = UnmarshalAny(node, nodeAny) +//func TestMarshalAny_Document(t *testing.T) { +// fieldsMap := map[string]interface{}{"f1": "aaa", "f2": 222, "f3": "ccc"} +// fieldsAny := &any.Any{} +// err := UnmarshalAny(fieldsMap, fieldsAny) +// if err != nil { +// t.Errorf("%v", err) +// } +// +// data := &index.Document{ +// Id: "1", +// Fields: fieldsAny, +// } +// +// dataAny := &any.Any{} +// err = UnmarshalAny(data, dataAny) +// if err != nil { +// t.Errorf("%v", err) +// } +// +// expectedType := "index.Document" +// actualType := dataAny.TypeUrl +// if expectedType != actualType { +// t.Errorf("expected content to see %s, saw %s", expectedType, actualType) +// } +// +// expectedValue := []byte(`{"id":"1","fields":{"type_url":"map[string]interface {}","value":"eyJmMSI6ImFhYSIsImYyIjoyMjIsImYzIjoiY2NjIn0="}}`) +// actualValue := dataAny.Value +// if !bytes.Equal(expectedValue, actualValue) { +// t.Errorf("expected content to see %v, saw %v", expectedValue, actualValue) +// } +//} + +//func TestMarshalAny_Node(t *testing.T) { +// data := &raft.Node{ +// Id: "node1", +// Metadata: &raft.Metadata{ +// GrpcAddr: ":5050", +// DataDir: "/tmp/blast/index1", +// BindAddr: ":6060", +// HttpAddr: ":8080", +// Leader: true, +// }, +// } +// +// dataAny := &any.Any{} +// err := UnmarshalAny(data, dataAny) +// if err != nil { +// t.Errorf("%v", err) +// } +// +// expectedType := "raft.Node" +// actualType := dataAny.TypeUrl +// if expectedType != actualType { +// t.Errorf("expected content to see %s, saw %s", expectedType, actualType) +// } +// +// expectedValue := []byte(`{"id":"node1","metadata":{"bind_addr":":6060","grpc_addr":":5050","http_addr":":8080","data_dir":"/tmp/blast/index1","leader":true}}`) +// actualValue := dataAny.Value +// if !bytes.Equal(expectedValue, actualValue) { +// t.Errorf("expected content to see %v, saw %v", expectedValue, actualValue) +// } +//} + +func TestMarshalAny_SearchRequest(t *testing.T) { + data := bleve.NewSearchRequest(bleve.NewQueryStringQuery("blast")) + + dataAny := &any.Any{} + err := UnmarshalAny(data, dataAny) if err != nil { t.Errorf("%v", err) } - expectedType = "raft.Node" - actualType = nodeAny.TypeUrl + expectedType := "bleve.SearchRequest" + actualType := dataAny.TypeUrl if expectedType != actualType { t.Errorf("expected content to see %s, saw %s", expectedType, actualType) } - expectedValue = []byte(`{"id":"node1","bind_addr":":6060","grpc_addr":":5050","http_addr":":8080","leader":true,"data_dir":"/tmp/blast/index1"}`) - actualValue = nodeAny.Value + expectedValue := []byte(`{"query":{"query":"blast"},"size":10,"from":0,"highlight":null,"fields":null,"facets":null,"explain":false,"sort":["-_score"],"includeLocations":false}`) + actualValue := dataAny.Value if !bytes.Equal(expectedValue, actualValue) { t.Errorf("expected content to see %v, saw %v", expectedValue, actualValue) } +} - // test bleve.SearchRequest - searchReq := bleve.NewSearchRequest(bleve.NewQueryStringQuery("blast")) +func TestMarshalAny_SearchResult(t *testing.T) { + data := &bleve.SearchResult{ + Total: 10, + } - searchReqAny := &any.Any{} - err = UnmarshalAny(searchReq, searchReqAny) + dataAny := &any.Any{} + err := UnmarshalAny(data, dataAny) if err != nil { t.Errorf("%v", err) } - expectedType = "bleve.SearchRequest" - actualType = searchReqAny.TypeUrl + expectedType := "bleve.SearchResult" + actualType := dataAny.TypeUrl if expectedType != actualType { t.Errorf("expected content to see %s, saw %s", expectedType, actualType) } - expectedValue = []byte(`{"query":{"query":"blast"},"size":10,"from":0,"highlight":null,"fields":null,"facets":null,"explain":false,"sort":["-_score"],"includeLocations":false}`) - actualValue = searchReqAny.Value + expectedValue := []byte(`{"status":null,"request":null,"hits":null,"total_hits":10,"max_score":0,"took":0,"facets":null}`) + actualValue := dataAny.Value if !bytes.Equal(expectedValue, actualValue) { t.Errorf("expected content to see %v, saw %v", expectedValue, actualValue) } +} - // test bleve.SearchResult - searchReslt := &bleve.SearchResult{ - Total: 10, +func TestUnmarshalAny_Slice(t *testing.T) { + dataAny := &any.Any{ + TypeUrl: "[]interface {}", + Value: []byte(`["a",1]`), } - searchResltAny := &any.Any{} - err = UnmarshalAny(searchReslt, searchResltAny) + ins, err := MarshalAny(dataAny) if err != nil { t.Errorf("%v", err) } - expectedType = "bleve.SearchResult" - actualType = searchResltAny.TypeUrl - if expectedType != actualType { - t.Errorf("expected content to see %s, saw %s", expectedType, actualType) + data := *ins.(*[]interface{}) + + expected1 := "a" + actual1 := data[0] + if expected1 != actual1 { + t.Errorf("expected content to see %v, saw %v", expected1, actual1) } - expectedValue = []byte(`{"status":null,"request":null,"hits":null,"total_hits":10,"max_score":0,"took":0,"facets":null}`) - actualValue = searchResltAny.Value - if !bytes.Equal(expectedValue, actualValue) { - t.Errorf("expected content to see %v, saw %v", expectedValue, actualValue) + expected2 := float64(1) + actual2 := data[1] + if expected2 != actual2 { + t.Errorf("expected content to see %v, saw %v", expected2, actual2) } } -func TestUnmarshalAny(t *testing.T) { - // test map[string]interface{} +func TestUnmarshalAny_Map(t *testing.T) { dataAny := &any.Any{ TypeUrl: "map[string]interface {}", Value: []byte(`{"a":1,"b":2,"c":3}`), } - data, err := MarshalAny(dataAny) + ins, err := MarshalAny(dataAny) if err != nil { t.Errorf("%v", err) } - dataMap := *data.(*map[string]interface{}) - if dataMap["a"] != float64(1) { - t.Errorf("expected content to see %v, saw %v", 1, dataMap["a"]) - } - if dataMap["b"] != float64(2) { - t.Errorf("expected content to see %v, saw %v", 2, dataMap["b"]) - } - if dataMap["c"] != float64(3) { - t.Errorf("expected content to see %v, saw %v", 3, dataMap["c"]) - } - - // index.Document - dataAny = &any.Any{ - TypeUrl: "index.Document", - Value: []byte(`{"id":"1","fields":{"type_url":"map[string]interface {}","value":"eyJmMSI6ImFhYSIsImYyIjoyMjIsImYzIjoiY2NjIn0="}}`), - } - - data, err = MarshalAny(dataAny) - if err != nil { - t.Errorf("%v", err) - } - dataDoc := data.(*index.Document) + data := *ins.(*map[string]interface{}) - if dataDoc.Id != "1" { - t.Errorf("expected content to see %v, saw %v", "1", dataDoc.Id) - } - if dataDoc.Fields.TypeUrl != "map[string]interface {}" { - t.Errorf("expected content to see %v, saw %v", "map[string]interface {}", dataDoc.Fields.TypeUrl) - } - if !bytes.Equal(dataDoc.Fields.Value, []byte(`{"f1":"aaa","f2":222,"f3":"ccc"}`)) { - t.Errorf("expected content to see %v, saw %v", []byte("eyJmMSI6ImFhYSIsImYyIjoyMjIsImYzIjoiY2NjIn0="), dataDoc.Fields.Value) + expected1 := float64(1) + actual1 := data["a"] + if expected1 != actual1 { + t.Errorf("expected content to see %v, saw %v", expected1, actual1) } - // raft.Node - dataAny = &any.Any{ - TypeUrl: "raft.Node", - Value: []byte(`{"id":"node1","bind_addr":":6060","grpc_addr":":5050","http_addr":":8080","leader":true,"data_dir":"/tmp/blast/index1"}`), + expected2 := float64(2) + actual2 := data["b"] + if expected2 != actual2 { + t.Errorf("expected content to see %v, saw %v", expected2, actual2) } - data, err = MarshalAny(dataAny) - if err != nil { - t.Errorf("%v", err) - } - dataNode := data.(*raft.Node) - - if dataNode.Id != "node1" { - t.Errorf("expected content to see %v, saw %v", "node1", dataNode.Id) - } - if dataNode.HttpAddr != ":8080" { - t.Errorf("expected content to see %v, saw %v", ":8080", dataNode.HttpAddr) - } - if dataNode.BindAddr != ":6060" { - t.Errorf("expected content to see %v, saw %v", ":6060", dataNode.BindAddr) - } - if dataNode.GrpcAddr != ":5050" { - t.Errorf("expected content to see %v, saw %v", ":5050", dataNode.BindAddr) - } - if dataNode.DataDir != "/tmp/blast/index1" { - t.Errorf("expected content to see %v, saw %v", "/tmp/blast/index1", dataNode.DataDir) - } - if dataNode.Leader != true { - t.Errorf("expected content to see %v, saw %v", true, dataNode.Leader) + expected3 := float64(3) + actual3 := data["c"] + if expected3 != actual3 { + t.Errorf("expected content to see %v, saw %v", expected3, actual3) } +} - // test bleve.SearchRequest - dataAny = &any.Any{ +//func TestUnmarshalAny_Document(t *testing.T) { +// dataAny := &any.Any{ +// TypeUrl: "index.Document", +// Value: []byte(`{"id":"1","fields":{"type_url":"map[string]interface {}","value":"eyJmMSI6ImFhYSIsImYyIjoyMjIsImYzIjoiY2NjIn0="}}`), +// } +// +// ins, err := MarshalAny(dataAny) +// if err != nil { +// t.Errorf("%v", err) +// } +// +// data := *ins.(*index.Document) +// +// expected1 := "1" +// actual1 := data.Id +// if expected1 != actual1 { +// t.Errorf("expected content to see %v, saw %v", expected1, actual1) +// } +// +// expected2 := "map[string]interface {}" +// actual2 := data.Fields.TypeUrl +// if expected2 != actual2 { +// t.Errorf("expected content to see %v, saw %v", expected2, actual2) +// } +// +// expected3 := []byte(`{"f1":"aaa","f2":222,"f3":"ccc"}`) +// actual3 := data.Fields.Value +// if !bytes.Equal(expected3, actual3) { +// t.Errorf("expected content to see %v, saw %v", expected3, actual3) +// } +//} + +func TestUnmarshalAny_SearchRequest(t *testing.T) { + dataAny := &any.Any{ TypeUrl: "bleve.SearchRequest", Value: []byte(`{"query":{"query":"blast"},"size":10,"from":0,"highlight":null,"fields":null,"facets":null,"explain":false,"sort":["-_score"],"includeLocations":false}`), } - data, err = MarshalAny(dataAny) + ins, err := MarshalAny(dataAny) if err != nil { t.Errorf("%v", err) } - searchRequest := data.(*bleve.SearchRequest) - if searchRequest.Query.(*query.QueryStringQuery).Query != bleve.NewQueryStringQuery("blast").Query { - t.Errorf("expected content to see %v, saw %v", bleve.NewQueryStringQuery("blast").Query, searchRequest.Query.(*query.QueryStringQuery).Query) + data := *ins.(*bleve.SearchRequest) + + expected1 := bleve.NewQueryStringQuery("blast").Query + actual1 := data.Query.(*query.QueryStringQuery).Query + if expected1 != actual1 { + t.Errorf("expected content to see %v, saw %v", expected1, actual1) } +} - // test blast.SearchResult - dataAny = &any.Any{ +func TestUnmarshalAny_SearchResult(t *testing.T) { + dataAny := &any.Any{ TypeUrl: "bleve.SearchResult", Value: []byte(`{"status":null,"request":null,"hits":null,"total_hits":10,"max_score":0,"took":0,"facets":null}`), } - data, err = MarshalAny(dataAny) + ins, err := MarshalAny(dataAny) if err != nil { t.Errorf("%v", err) } - searchResult := data.(*bleve.SearchResult) - if searchResult.Total != 10 { - t.Errorf("expected content to see %v, saw %v", 10, searchResult.Total) - } + data := *ins.(*bleve.SearchResult) + expected1 := uint64(10) + actual1 := data.Total + if expected1 != actual1 { + t.Errorf("expected content to see %v, saw %v", expected1, actual1) + } } diff --git a/registry/type.go b/registry/type.go index 216761a..5cb1206 100644 --- a/registry/type.go +++ b/registry/type.go @@ -18,8 +18,40 @@ import ( "errors" "fmt" "reflect" + + "github.com/blevesearch/bleve" + "github.com/blevesearch/bleve/mapping" ) +func init() { + RegisterType("bool", reflect.TypeOf(false)) + RegisterType("string", reflect.TypeOf("")) + RegisterType("int", reflect.TypeOf(int(0))) + RegisterType("int8", reflect.TypeOf(int8(0))) + RegisterType("int16", reflect.TypeOf(int16(0))) + RegisterType("int32", reflect.TypeOf(int32(0))) + RegisterType("int64", reflect.TypeOf(int64(0))) + RegisterType("uint", reflect.TypeOf(uint(0))) + RegisterType("uint8", reflect.TypeOf(uint8(0))) + RegisterType("uint16", reflect.TypeOf(uint16(0))) + RegisterType("uint32", reflect.TypeOf(uint32(0))) + RegisterType("uint64", reflect.TypeOf(uint64(0))) + RegisterType("uintptr", reflect.TypeOf(uintptr(0))) + RegisterType("byte", reflect.TypeOf(byte(0))) + RegisterType("rune", reflect.TypeOf(rune(0))) + RegisterType("float32", reflect.TypeOf(float32(0))) + RegisterType("float64", reflect.TypeOf(float64(0))) + RegisterType("complex64", reflect.TypeOf(complex64(0))) + RegisterType("complex128", reflect.TypeOf(complex128(0))) + + RegisterType("map[string]interface {}", reflect.TypeOf((map[string]interface{})(nil))) + RegisterType("[]interface {}", reflect.TypeOf(([]interface{})(nil))) + + RegisterType("mapping.IndexMappingImpl", reflect.TypeOf(mapping.IndexMappingImpl{})) + RegisterType("bleve.SearchRequest", reflect.TypeOf(bleve.SearchRequest{})) + RegisterType("bleve.SearchResult", reflect.TypeOf(bleve.SearchResult{})) +} + type TypeRegistry map[string]reflect.Type var Types = make(TypeRegistry, 0) @@ -36,11 +68,13 @@ func TypeByName(name string) reflect.Type { } func TypeNameByInstance(instance interface{}) string { - switch ins := instance.(type) { - case map[string]interface{}: - return reflect.TypeOf(ins).String() + switch instance.(type) { + case bool, string, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, uintptr, float32, float64, complex64, complex128: + return reflect.TypeOf(instance).Name() + case map[string]interface{}, []interface{}: + return reflect.TypeOf(instance).String() default: - return reflect.TypeOf(ins).Elem().String() + return reflect.TypeOf(instance).Elem().String() } } diff --git a/sortutils/sort.go b/sortutils/sort.go new file mode 100644 index 0000000..4e7966a --- /dev/null +++ b/sortutils/sort.go @@ -0,0 +1,42 @@ +// Copyright (c) 2019 Minoru Osuka +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sortutils + +import ( + "github.com/blevesearch/bleve/search" +) + +type MultiSearchHitSorter struct { + hits search.DocumentMatchCollection + sort search.SortOrder + cachedScoring []bool + cachedDesc []bool +} + +func NewMultiSearchHitSorter(sort search.SortOrder, hits search.DocumentMatchCollection) *MultiSearchHitSorter { + return &MultiSearchHitSorter{ + sort: sort, + hits: hits, + cachedScoring: sort.CacheIsScore(), + cachedDesc: sort.CacheDescending(), + } +} + +func (m *MultiSearchHitSorter) Len() int { return len(m.hits) } +func (m *MultiSearchHitSorter) Swap(i, j int) { m.hits[i], m.hits[j] = m.hits[j], m.hits[i] } +func (m *MultiSearchHitSorter) Less(i, j int) bool { + c := m.sort.Compare(m.cachedScoring, m.cachedDesc, m.hits[i], m.hits[j]) + return c < 0 +}