From a9386bb867c706a2b534879b1ed403ab328f2f85 Mon Sep 17 00:00:00 2001 From: luke miles Date: Mon, 29 Aug 2022 21:56:03 -0700 Subject: [PATCH 1/4] add PUT /api/v1/tree/pin endpoint this is a draft attempt at adding a /api/v1/tree/pin endpoint. this endpoint will pin a json representation of a merkle tree to ipfs. you can see an example of what this code currently produces pinned to `[ipfs://QmXdDyUSsM59MUoeZyDS5uVo8gg4WMenduvehUjP4Hs8KG](https://cloudflare-ipfs.com/ipfs/QmXdDyUSsM59MUoeZyDS5uVo8gg4WMenduvehUjP4Hs8KG)`. open ideas/questions: - should we save ipfs hashes to postgres when we pin them and return them via `/tree` and other endpoints? - should we save every tree by default to IPFS? if we do that, how do we depend on our pinning provider being down/retries/etc. also the latency isn't great as is - even locally pinning was taking 600ms-1s --- api/api.go | 5 +++ api/ipfs.go | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++++ api/tree.go | 35 +++++++++++++++++++++ 3 files changed, 128 insertions(+) create mode 100644 api/ipfs.go diff --git a/api/api.go b/api/api.go index 636fd8e..744dd1e 100644 --- a/api/api.go +++ b/api/api.go @@ -21,17 +21,22 @@ import ( type Server struct { db *pgxpool.Pool + hc *http.Client } func New(db *pgxpool.Pool) *Server { return &Server{ db: db, + hc: &http.Client{ + Timeout: time.Second * 10, + }, } } func (s *Server) Handler(env, gitSha string) http.Handler { mux := http.NewServeMux() mux.HandleFunc("/api/v1/tree", s.TreeHandler) + mux.HandleFunc("/api/v1/tree/pin", s.PinTree) mux.HandleFunc("/api/v1/proof", s.GetProof) mux.HandleFunc("/api/v1/root", s.GetRoot) mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { diff --git a/api/ipfs.go b/api/ipfs.go new file mode 100644 index 0000000..13ea788 --- /dev/null +++ b/api/ipfs.go @@ -0,0 +1,88 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "os" + + "github.com/ethereum/go-ethereum/common/hexutil" +) + +var ( + ipfsPinningServiceURL = os.Getenv("IPFS_PINNING_SERVICE_URL") + ipfsPinningSecret = os.Getenv("IPFS_PINNING_SECRET") +) + +func (s *Server) pinTree(ctx context.Context, root hexutil.Bytes) (string, error) { + if ipfsPinningServiceURL == "" { + return "", errors.New("error: IPFS_PINNING_SERVICE_URL not set") + } + + const q = ` + SELECT unhashed_leaves, ltd, packed + FROM trees + WHERE root = $1 + ` + tr := struct { + Root hexutil.Bytes `json:"root"` + UnhashedLeaves []hexutil.Bytes `json:"unhashedLeaves"` + Ltd []string `json:"leafTypeDescriptor"` + Packed jsonNullBool `json:"packedEncoding"` + }{ + Root: root, + } + + err := s.db.QueryRow(ctx, q, root).Scan( + &tr.UnhashedLeaves, + &tr.Ltd, + &tr.Packed, + ) + + if err != nil { + return "", err + } + + msg, err := json.Marshal(tr) + + if err != nil { + return "", err + } + + req, err := http.NewRequestWithContext( + ctx, "POST", ipfsPinningServiceURL, bytes.NewReader(msg), + ) + + if err != nil { + return "", err + } + + req.Header.Set("Authorization", "Bearer "+ipfsPinningSecret) + req.Header.Set("Content-Type", "application/json") + + res, err := s.hc.Do(req) + + if err != nil { + return "", err + } + + if res.StatusCode >= 400 { + return "", errors.New(res.Status) + } + + type resp struct { + Hash string `json:"IpfsHash"` + } + + defer res.Body.Close() + + var r resp + err = json.NewDecoder(res.Body).Decode(&r) + if err != nil { + return "", err + } + + return r.Hash, nil +} diff --git a/api/tree.go b/api/tree.go index a110896..9f3785b 100644 --- a/api/tree.go +++ b/api/tree.go @@ -228,3 +228,38 @@ func (s *Server) GetTree(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "public, max-age=3600") s.sendJSON(r, w, tr) } + +func (s *Server) PinTree(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPut { + http.Error(w, "unsupported method", http.StatusMethodNotAllowed) + return + } + + type pinTreeReq struct { + MerkleRoot hexutil.Bytes `json:"merkleRoot"` + } + + var ( + req pinTreeReq + ctx = r.Context() + ) + defer r.Body.Close() + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + s.sendJSONError(r, w, err, http.StatusBadRequest, "unhashedLeaves must be a list of hex strings") + return + } + + hash, err := s.pinTree(ctx, req.MerkleRoot) + + if errors.Is(err, pgx.ErrNoRows) { + s.sendJSONError(r, w, err, http.StatusNotFound, "tree not found for root") + return + } else if err != nil { + s.sendJSONError(r, w, err, http.StatusInternalServerError, "selecting tree") + return + } + + s.sendJSON(r, w, map[string]any{ + "ipfsHash": "ipfs://" + hash, + }) +} From 17a87151a7f528ac189136cfa7fe9086a51688e8 Mon Sep 17 00:00:00 2001 From: luke miles Date: Mon, 29 Aug 2022 22:10:16 -0700 Subject: [PATCH 2/4] move http client --- api/api.go | 4 ---- api/ipfs.go | 6 +++++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/api.go b/api/api.go index 744dd1e..6b15bdd 100644 --- a/api/api.go +++ b/api/api.go @@ -21,15 +21,11 @@ import ( type Server struct { db *pgxpool.Pool - hc *http.Client } func New(db *pgxpool.Pool) *Server { return &Server{ db: db, - hc: &http.Client{ - Timeout: time.Second * 10, - }, } } diff --git a/api/ipfs.go b/api/ipfs.go index 13ea788..346a8b3 100644 --- a/api/ipfs.go +++ b/api/ipfs.go @@ -7,6 +7,7 @@ import ( "errors" "net/http" "os" + "time" "github.com/ethereum/go-ethereum/common/hexutil" ) @@ -14,6 +15,9 @@ import ( var ( ipfsPinningServiceURL = os.Getenv("IPFS_PINNING_SERVICE_URL") ipfsPinningSecret = os.Getenv("IPFS_PINNING_SECRET") + hc = &http.Client{ + Timeout: time.Second * 10, + } ) func (s *Server) pinTree(ctx context.Context, root hexutil.Bytes) (string, error) { @@ -62,7 +66,7 @@ func (s *Server) pinTree(ctx context.Context, root hexutil.Bytes) (string, error req.Header.Set("Authorization", "Bearer "+ipfsPinningSecret) req.Header.Set("Content-Type", "application/json") - res, err := s.hc.Do(req) + res, err := hc.Do(req) if err != nil { return "", err From c20ff73719989c39c2f3264b15936ef46847c0f8 Mon Sep 17 00:00:00 2001 From: luke miles Date: Mon, 29 Aug 2022 22:12:04 -0700 Subject: [PATCH 3/4] tracing --- api/ipfs.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/ipfs.go b/api/ipfs.go index 346a8b3..c0ff562 100644 --- a/api/ipfs.go +++ b/api/ipfs.go @@ -9,6 +9,7 @@ import ( "os" "time" + "github.com/contextwtf/lanyard/api/tracing" "github.com/ethereum/go-ethereum/common/hexutil" ) @@ -66,7 +67,9 @@ func (s *Server) pinTree(ctx context.Context, root hexutil.Bytes) (string, error req.Header.Set("Authorization", "Bearer "+ipfsPinningSecret) req.Header.Set("Content-Type", "application/json") + span, ctx := tracing.SpanFromContext(ctx, "ipfs.pinTree") res, err := hc.Do(req) + span.Finish() if err != nil { return "", err From 545e6110ab37183dc4aa0ce6536c0f1c85ac9b7c Mon Sep 17 00:00:00 2001 From: luke miles Date: Mon, 29 Aug 2022 22:19:28 -0700 Subject: [PATCH 4/4] style --- api/ipfs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/ipfs.go b/api/ipfs.go index c0ff562..dcb65e0 100644 --- a/api/ipfs.go +++ b/api/ipfs.go @@ -23,7 +23,7 @@ var ( func (s *Server) pinTree(ctx context.Context, root hexutil.Bytes) (string, error) { if ipfsPinningServiceURL == "" { - return "", errors.New("error: IPFS_PINNING_SERVICE_URL not set") + return "", errors.New("IPFS_PINNING_SERVICE_URL not set") } const q = `