From 928891091a21799a8987edd00c9a58fb696f69a3 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 15 Aug 2023 13:37:53 -0400 Subject: [PATCH] feat: IPIP-402 based backpressure (#160) Rewrote the graph-based gateway backend to use CAR requests that are backpressured and incrementally verified (with errors detectable by Caboose) rather than racily stored and read from a shared cache. No longer switches to block-based fallbacks when running into problems with CAR requests and instead retries CAR requests. Also allows reading from CARs with empty roots and fixing escaping of paths for CAR file requests. Co-authored-by: Marcin Rataj Co-authored-by: Will Co-authored-by: Aarsh Shah --- .github/workflows/gateway-conformance.yml | 16 +- blockstore.go | 36 +- blockstore_proxy.go | 10 +- blockstore_test.go | 44 - go.mod | 89 +- go.sum | 312 +--- graph_gateway_test.go | 1195 +++++++++++++ handlers.go | 8 +- lib/files.go | 533 ++++++ ...h-multilayer-hamt-and-multiblock-files.car | Bin 0 -> 6045 bytes lib/gateway_traversal.go | 313 ++++ lib/gateway_utils.go | 73 + lib/gateway_utils_test.go | 94 + lib/graph_gateway.go | 1573 +++++++++++------ lib/pubsub.go | 146 -- test-backend/handlers.go | 627 ------- test-backend/main.go | 147 -- test-backend/version.go | 39 - 18 files changed, 3351 insertions(+), 1904 deletions(-) delete mode 100644 blockstore_test.go create mode 100644 graph_gateway_test.go create mode 100644 lib/files.go create mode 100644 lib/fixtures/directory-with-multilayer-hamt-and-multiblock-files.car create mode 100644 lib/gateway_traversal.go create mode 100644 lib/gateway_utils.go create mode 100644 lib/gateway_utils_test.go delete mode 100644 lib/pubsub.go delete mode 100644 test-backend/handlers.go delete mode 100644 test-backend/main.go delete mode 100644 test-backend/version.go diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 4d42df6..162ba98 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -14,8 +14,16 @@ jobs: car-env: ["GRAPH_BACKEND=false", "GRAPH_BACKEND=true"] steps: # 1. Start the Kubo gateway - - name: Download Kubo gateway - uses: ipfs/download-ipfs-distribution-action@v1 + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: 1.19.x + - uses: protocol/cache-go-action@v1 + + - name: Install Kubo gateway from source + #uses: ipfs/download-ipfs-distribution-action@v1 + run: | + go install github.com/ipfs/kubo/cmd/ipfs@73552dfd9c6f73fdab887c4d2e99cf7e848a3c23 - name: Start Kubo gateway uses: ipfs/start-ipfs-daemon-action@v1 @@ -46,10 +54,6 @@ jobs: echo "IPFS_NS_MAP=${IPFS_NS_MAP}" >> $GITHUB_ENV # 4. Build the bifrost-gateway - - name: Setup Go - uses: actions/setup-go@v3 - with: - go-version: 1.19.x - name: Checkout bifrost-gateway uses: actions/checkout@v3 with: diff --git a/blockstore.go b/blockstore.go index 571b436..825cb08 100644 --- a/blockstore.go +++ b/blockstore.go @@ -3,14 +3,11 @@ package main import ( "context" "errors" - "fmt" - "strings" "time" - "github.com/filecoin-saturn/caboose" + "github.com/ipfs/bifrost-gateway/lib" blockstore "github.com/ipfs/boxo/blockstore" exchange "github.com/ipfs/boxo/exchange" - "github.com/ipfs/boxo/gateway" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "go.uber.org/zap/zapcore" @@ -38,7 +35,7 @@ func (e *exchangeBsWrapper) GetBlock(ctx context.Context, c cid.Cid) (blocks.Blo blk, err := e.bstore.Get(ctx, c) if err != nil { - return nil, gatewayError(err) + return nil, lib.GatewayError(err) } return blk, nil } @@ -67,33 +64,4 @@ func (e *exchangeBsWrapper) Close() error { return nil } -// gatewayError translates underlying blockstore error into one that gateway code will return as HTTP 502 or 504 -// it also makes sure Retry-After hint from remote blockstore will be passed to HTTP client, if present. -func gatewayError(err error) error { - if errors.Is(err, &gateway.ErrorStatusCode{}) || - errors.Is(err, &gateway.ErrorRetryAfter{}) { - // already correct error - return err - } - - // All timeouts should produce 504 Gateway Timeout - if errors.Is(err, context.DeadlineExceeded) || - errors.Is(err, caboose.ErrSaturnTimeout) || - // Unfortunately this is not an exported type so we have to check for the content. - strings.Contains(err.Error(), "Client.Timeout exceeded") { - return fmt.Errorf("%w: %s", gateway.ErrGatewayTimeout, err.Error()) - } - - // (Saturn) errors that support the RetryAfter interface need to be converted - // to the correct gateway error, such that the HTTP header is set. - for v := err; v != nil; v = errors.Unwrap(v) { - if r, ok := v.(interface{ RetryAfter() time.Duration }); ok { - return gateway.NewErrorRetryAfter(err, r.RetryAfter()) - } - } - - // everything else returns 502 Bad Gateway - return fmt.Errorf("%w: %s", gateway.ErrBadGateway, err.Error()) -} - var _ exchange.Interface = (*exchangeBsWrapper)(nil) diff --git a/blockstore_proxy.go b/blockstore_proxy.go index 2b44ef4..deab4fb 100644 --- a/blockstore_proxy.go +++ b/blockstore_proxy.go @@ -43,14 +43,20 @@ func (ps *proxyBlockStore) Fetch(ctx context.Context, path string, cb lib.DataCa return err } goLog.Debugw("car fetch", "url", req.URL) - req.Header.Set("Accept", "application/vnd.ipld.car") + req.Header.Set("Accept", "application/vnd.ipld.car;order=dfs;dups=y") resp, err := ps.httpClient.Do(req) if err != nil { return err } if resp.StatusCode != http.StatusOK { - return fmt.Errorf("http error from car gateway: %s", resp.Status) + errData, err := io.ReadAll(resp.Body) + if err != nil { + err = fmt.Errorf("could not read error message: %w", err) + } else { + err = fmt.Errorf("%q", string(errData)) + } + return fmt.Errorf("http error from car gateway: %s: %w", resp.Status, err) } err = cb(path, resp.Body) diff --git a/blockstore_test.go b/blockstore_test.go deleted file mode 100644 index c47827a..0000000 --- a/blockstore_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "testing" - "time" - - "github.com/ipfs/boxo/gateway" - "github.com/stretchr/testify/assert" -) - -type testErr struct { - message string - retryAfter time.Duration -} - -func (e *testErr) Error() string { - return e.message -} - -func (e *testErr) RetryAfter() time.Duration { - return e.retryAfter -} - -func TestGatewayErrorRetryAfter(t *testing.T) { - originalErr := &testErr{message: "test", retryAfter: time.Minute} - var ( - convertedErr error - gatewayErr *gateway.ErrorRetryAfter - ) - - // Test unwrapped - convertedErr = gatewayError(originalErr) - ok := errors.As(convertedErr, &gatewayErr) - assert.True(t, ok) - assert.EqualValues(t, originalErr.retryAfter, gatewayErr.RetryAfter) - - // Test wrapped. - convertedErr = gatewayError(fmt.Errorf("wrapped error: %w", originalErr)) - ok = errors.As(convertedErr, &gatewayErr) - assert.True(t, ok) - assert.EqualValues(t, originalErr.retryAfter, gatewayErr.RetryAfter) -} diff --git a/go.mod b/go.mod index 1777661..3de628e 100644 --- a/go.mod +++ b/go.mod @@ -3,21 +3,20 @@ module github.com/ipfs/bifrost-gateway go 1.19 require ( - github.com/cskr/pubsub v1.0.2 - github.com/filecoin-saturn/caboose v0.0.4 + github.com/filecoin-saturn/caboose v0.0.5-0.20230808125624-a986b542aa10 + github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/golang-lru/v2 v2.0.1 - github.com/ipfs/boxo v0.11.0 + github.com/ipfs/boxo v0.11.1-0.20230811122208-5de8f5f5ca60 github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-ipld-format v0.5.0 github.com/ipfs/go-log/v2 v2.5.1 - github.com/ipfs/go-unixfsnode v1.7.1 - github.com/ipld/go-car v0.6.0 + github.com/ipfs/go-unixfsnode v1.7.4 + github.com/ipld/go-car v0.6.2 github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33 github.com/ipld/go-codec-dagpb v1.6.0 github.com/ipld/go-ipld-prime v0.20.0 github.com/libp2p/go-libp2p v0.26.3 - github.com/libp2p/go-libp2p-kad-dht v0.23.0 github.com/libp2p/go-libp2p-routing-helpers v0.7.0 github.com/mitchellh/go-server-timing v1.0.1 github.com/multiformats/go-multicodec v0.9.0 @@ -27,110 +26,81 @@ require ( github.com/spf13/cobra v1.6.1 github.com/stretchr/testify v1.8.4 github.com/willscott/go-requestcontext v0.0.1 - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.40.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 go.opentelemetry.io/contrib/propagators/autoprop v0.40.0 - go.opentelemetry.io/otel v1.14.0 - go.opentelemetry.io/otel/sdk v1.14.0 - go.opentelemetry.io/otel/trace v1.14.0 + go.opentelemetry.io/otel v1.16.0 + go.opentelemetry.io/otel/sdk v1.16.0 + go.opentelemetry.io/otel/trace v1.16.0 go.uber.org/atomic v1.10.0 - go.uber.org/multierr v1.9.0 go.uber.org/zap v1.24.0 ) require ( github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/asecurityteam/rolling v0.0.0-20230418204413-b4052899307d // indirect - github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.2.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/containerd/cgroups v1.0.4 // indirect - github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect - github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect - github.com/elastic/gosigar v0.14.2 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect - github.com/flynn/noise v1.0.0 // indirect - github.com/francoispqt/gojay v1.2.13 // indirect github.com/gabriel-vasile/mimetype v1.4.1 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect - github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/gddo v0.0.0-20180823221919-9d8ff1c67be5 // indirect - github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/gopacket v1.1.19 // indirect - github.com/google/pprof v0.0.0-20221203041831-ce31453925ec // indirect github.com/google/uuid v1.3.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect - github.com/huin/goupnp v1.0.3 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-blockservice v0.5.0 // indirect github.com/ipfs/go-datastore v0.6.0 // indirect github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect - github.com/ipfs/go-ipfs-delay v0.0.1 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect github.com/ipfs/go-ipfs-redirects-file v0.1.1 // indirect github.com/ipfs/go-ipfs-util v0.0.2 // indirect github.com/ipfs/go-ipld-cbor v0.0.6 // indirect github.com/ipfs/go-ipld-legacy v0.2.1 // indirect - github.com/ipfs/go-libipfs v0.6.0 // indirect github.com/ipfs/go-log v1.0.5 // indirect github.com/ipfs/go-merkledag v0.11.0 // indirect github.com/ipfs/go-metrics-interface v0.0.1 // indirect github.com/ipfs/go-verifcid v0.0.2 // indirect - github.com/jackpal/go-nat-pmp v1.0.2 // indirect - github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect github.com/klauspost/compress v1.15.15 // indirect github.com/klauspost/cpuid/v2 v2.2.3 // indirect - github.com/koron/go-ssdp v0.0.3 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-doh-resolver v0.4.0 // indirect - github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect + github.com/libp2p/go-libp2p-kad-dht v0.23.0 // indirect github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect github.com/libp2p/go-libp2p-record v0.2.0 // indirect github.com/libp2p/go-msgio v0.3.0 // indirect - github.com/libp2p/go-nat v0.1.0 // indirect github.com/libp2p/go-netroute v0.2.1 // indirect - github.com/libp2p/go-reuseport v0.2.0 // indirect - github.com/libp2p/go-yamux/v4 v4.0.0 // indirect - github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/miekg/dns v1.1.50 // indirect - github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect - github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multiaddr v0.8.0 // indirect github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect - github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect - github.com/onsi/ginkgo/v2 v2.5.1 // indirect github.com/opencontainers/runtime-spec v1.0.3-0.20211123151946-c2389c3cb60a // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/openzipkin/zipkin-go v0.4.1 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect - github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -138,12 +108,6 @@ require ( github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect - github.com/quic-go/qpack v0.4.0 // indirect - github.com/quic-go/qtls-go1-19 v0.2.1 // indirect - github.com/quic-go/qtls-go1-20 v0.1.1 // indirect - github.com/quic-go/quic-go v0.33.0 // indirect - github.com/quic-go/webtransport-go v0.5.2 // indirect - github.com/raulk/go-watchdog v1.3.0 // indirect github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect @@ -154,35 +118,34 @@ require ( github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.42.0 // indirect go.opentelemetry.io/contrib/propagators/aws v1.15.0 // indirect go.opentelemetry.io/contrib/propagators/b3 v1.15.0 // indirect go.opentelemetry.io/contrib/propagators/jaeger v1.15.0 // indirect go.opentelemetry.io/contrib/propagators/ot v1.15.0 // indirect go.opentelemetry.io/otel/exporters/jaeger v1.14.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.14.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 // indirect go.opentelemetry.io/otel/exporters/zipkin v1.14.0 // indirect - go.opentelemetry.io/otel/metric v0.37.0 // indirect + go.opentelemetry.io/otel/metric v1.16.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect - go.uber.org/dig v1.15.0 // indirect - go.uber.org/fx v1.18.2 // indirect + go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.6.0 // indirect golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb // indirect - golang.org/x/mod v0.7.0 // indirect - golang.org/x/net v0.7.0 // indirect + golang.org/x/mod v0.8.0 // indirect + golang.org/x/net v0.8.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/text v0.7.0 // indirect - golang.org/x/tools v0.5.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/text v0.8.0 // indirect + golang.org/x/tools v0.6.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gonum.org/v1/gonum v0.11.0 // indirect - google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect - google.golang.org/grpc v1.53.0 // indirect + google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect + google.golang.org/grpc v1.55.0 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect - nhooyr.io/websocket v1.8.7 // indirect ) diff --git a/go.sum b/go.sum index ffd6f7b..d5bc1cc 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,5 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= @@ -32,31 +30,21 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= -dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= -dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= -git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/asecurityteam/rolling v0.0.0-20230418204413-b4052899307d h1:OD9AM8JZUHPrkEa9/SgQW2cCX6Om8d6n0akPS5leWJA= github.com/asecurityteam/rolling v0.0.0-20230418204413-b4052899307d/go.mod h1:tWDU1S7csNXWrzNpkbCk/dXpZkVcL4PfKn6Akwrffok= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= -github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= -github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= -github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= -github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -65,7 +53,6 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -74,36 +61,24 @@ github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= -github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= -github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= -github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= -github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= -github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -114,66 +89,36 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/felixge/httpsnoop v1.0.0/go.mod h1:3+D9sFq0ahK/JeJPhCBUV1xlf4/eIYrUQaxulT0VzX8= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/filecoin-saturn/caboose v0.0.4 h1:RSJ1FLy3R2/WZkfuDHt3yAJKNh4YXKcii75c4obuTyY= -github.com/filecoin-saturn/caboose v0.0.4/go.mod h1:CSXZMijzD8z0Q/K5JQPeDZnb1o50MBK9VGZhOleqIDk= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/filecoin-saturn/caboose v0.0.5-0.20230808125624-a986b542aa10 h1:VZhzrXd6IuL52U18KtTQ/q4v48J90a1aDr/maLF3H9Y= +github.com/filecoin-saturn/caboose v0.0.5-0.20230808125624-a986b542aa10/go.mod h1:nmDICK/NKu6r3c6V6W/h6vi7Ep2b+jWbRbEg51ltkHs= github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= -github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= -github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q= github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= -github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= -github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= -github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= -github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= -github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= -github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= -github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= -github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= -github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= -github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= -github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= -github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= -github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= -github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/gddo v0.0.0-20180823221919-9d8ff1c67be5 h1:yrv1uUvgXH/tEat+wdvJMRJ4g51GlIydtDpU9pFjaaI= github.com/golang/gddo v0.0.0-20180823221919-9d8ff1c67be5/go.mod h1:xEhNfoBDX1hzLm2Nf80qUvZ2sVwoMZ8d6IE2SrsQfh4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -182,7 +127,6 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -210,15 +154,10 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -231,22 +170,15 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20221203041831-ce31453925ec h1:fR20TYVVwhK4O7r7y+McjRYyaTH6/vjwJOajE+XhlzM= -github.com/google/pprof v0.0.0-20221203041831-ce31453925ec/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= -github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= -github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= @@ -263,17 +195,14 @@ github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+l github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= -github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= -github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.11.0 h1:urMxhZ3xoF4HssJVD3+0ssGT9pptEfHfbL8DYdoWFlg= -github.com/ipfs/boxo v0.11.0/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w= +github.com/ipfs/boxo v0.11.1-0.20230811122208-5de8f5f5ca60 h1:R8EYqB71skk4IxUlxVifUegUVEFZK4j+7nTdZF37Nus= +github.com/ipfs/boxo v0.11.1-0.20230811122208-5de8f5f5ca60/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= @@ -299,7 +228,6 @@ github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IW github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= -github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-ds-help v1.1.0 h1:yLE2w9RAsl31LtfMt91tRZcrx+e61O5mDxFRR994w4Q= github.com/ipfs/go-ipfs-ds-help v1.1.0/go.mod h1:YR5+6EaebOhfcqVCyqemItCLthrpVNot+rsOU/5IatU= github.com/ipfs/go-ipfs-exchange-interface v0.2.0 h1:8lMSJmKogZYNo2jjhUs0izT+dck05pqUw4mWNW9Pw6Y= @@ -319,8 +247,6 @@ github.com/ipfs/go-ipld-format v0.5.0 h1:WyEle9K96MSrvr47zZHKKcDxJ/vlpET6PSiQsAF github.com/ipfs/go-ipld-format v0.5.0/go.mod h1:ImdZqJQaEouMjCvqCe0ORUS+uoBmf7Hf+EO/jh+nk3M= github.com/ipfs/go-ipld-legacy v0.2.1 h1:mDFtrBpmU7b//LzLSypVrXsD8QxkEWxu5qVxN99/+tk= github.com/ipfs/go-ipld-legacy v0.2.1/go.mod h1:782MOUghNzMO2DER0FlBR94mllfdCJCkTtDtPM51otM= -github.com/ipfs/go-libipfs v0.6.0 h1:3FuckAJEm+zdHbHbf6lAyk0QUzc45LsFcGw102oBCZM= -github.com/ipfs/go-libipfs v0.6.0/go.mod h1:UjjDIuehp2GzlNP0HEr5I9GfFT7zWgst+YfpUEIThtw= github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8= github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo= github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= @@ -332,12 +258,12 @@ github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fG github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU= -github.com/ipfs/go-unixfsnode v1.7.1 h1:RRxO2b6CSr5UQ/kxnGzaChTjp5LWTdf3Y4n8ANZgB/s= -github.com/ipfs/go-unixfsnode v1.7.1/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= +github.com/ipfs/go-unixfsnode v1.7.4 h1:iLvKyAVKUYOIAW2t4kDYqsT7VLGj31eXJE2aeqGfbwA= +github.com/ipfs/go-unixfsnode v1.7.4/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU= -github.com/ipld/go-car v0.6.0 h1:d5QrGLnHAxiNLHor+DKGrLdqnM0dQJh2whfSXRDq6J0= -github.com/ipld/go-car v0.6.0/go.mod h1:tBrW1XZ3L2XipLxA69RnTVGW3rve6VX4TbaTYkq8aEA= +github.com/ipld/go-car v0.6.2 h1:Hlnl3Awgnq8icK+ze3iRghk805lu8YNq3wlREDTF2qc= +github.com/ipld/go-car v0.6.2/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8= github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33 h1:0OZwzSYWIuiKEOXd/2vm5cMcEmmGLFn+1h6lHELCm3s= github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33/go.mod h1:sQEkXVM3csejlb1kCCb+vQ/pWBKX9QtvsrysMQjOgOg= github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc= @@ -346,44 +272,30 @@ github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3u github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= -github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= -github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= -github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= -github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.3 h1:JivLMY45N76b4p/vsWGOKewBQu6uf39y8l+AQ7sDKx8= -github.com/koron/go-ssdp v0.0.3/go.mod h1:b2MxI6yh02pKrsyNoQUsk4+YNikaGhe4894J+Q5lDvA= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= -github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= @@ -391,7 +303,6 @@ github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0 github.com/libp2p/go-doh-resolver v0.4.0 h1:gUBa1f1XsPwtpE1du0O+nnZCUqtG7oYi7Bb+0S7FQqw= github.com/libp2p/go-doh-resolver v0.4.0/go.mod h1:v1/jwsFusgsWIGX/c6vCRrnJ60x7bhTiq/fs2qt0cAg= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= -github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= github.com/libp2p/go-libp2p v0.26.3 h1:6g/psubqwdaBqNNoidbRKSTBEYgaOuKBhHl8Q5tO+PM= github.com/libp2p/go-libp2p v0.26.3/go.mod h1:x75BN32YbwuY0Awm2Uix4d4KOz+/4piInkp4Wr3yOo8= github.com/libp2p/go-libp2p-asn-util v0.2.0 h1:rg3+Os8jbnO5DxkC7K/Utdi+DkY3q/d1/1q+8WeNAsw= @@ -408,36 +319,21 @@ github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUI github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= github.com/libp2p/go-nat v0.1.0 h1:MfVsH6DLcpa04Xr+p8hmVRG4juse0s3J8HyNWYHffXg= -github.com/libp2p/go-nat v0.1.0/go.mod h1:X7teVkwRHNInVNWQiO/tAiAVRwSr5zoRz4YSTC3uRBM= -github.com/libp2p/go-netroute v0.1.2/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= github.com/libp2p/go-netroute v0.2.1 h1:V8kVrpD8GK0Riv15/7VN6RbUQ3URNZVosw7H2v9tksU= github.com/libp2p/go-netroute v0.2.1/go.mod h1:hraioZr0fhBjG0ZRXJJ6Zj2IVEVNx6tDTFQfSmcq7mQ= github.com/libp2p/go-reuseport v0.2.0 h1:18PRvIMlpY6ZK85nIAicSBuXXvrYoSw3dsBAR7zc560= -github.com/libp2p/go-reuseport v0.2.0/go.mod h1:bvVho6eLMm6Bz5hmU0LYN3ixd3nPPvtIlaURZZgOY4k= -github.com/libp2p/go-sockaddr v0.0.2/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k= github.com/libp2p/go-yamux/v4 v4.0.0 h1:+Y80dV2Yx/kv7Y7JKu0LECyVdMXm1VUoko+VQ9rBfZQ= -github.com/libp2p/go-yamux/v4 v4.0.0/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= -github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= -github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= -github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= -github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= -github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= -github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKoFL8DUUmalo2yJJUCxbPKtm8OKfqr2/FTNU= github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc h1:PTfri+PuQmWDqERdnNMiD9ZejrlswWrCpBEZgWOiTrc= -github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc/go.mod h1:cGKTAVKx4SxOuR/czcZ/E2RSJ3sfHs8FpHhQ5CWMf9s= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= @@ -445,12 +341,6 @@ github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/go-server-timing v1.0.1 h1:f00/aIe8T3MrnLhQHu3tSWvnwc5GV/p5eutuu3hF/tE= github.com/mitchellh/go-server-timing v1.0.1/go.mod h1:Mo6GKi9FSLwWFAMn3bqVPWe20y5ri5QGQuO9D9MCOxk= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= @@ -462,7 +352,6 @@ github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYg github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= -github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= github.com/multiformats/go-multiaddr v0.8.0 h1:aqjksEcqK+iD/Foe1RRFsGZh8+XFiGo7FgUCZlpv3LU= github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= @@ -470,7 +359,6 @@ github.com/multiformats/go-multiaddr-dns v0.3.0/go.mod h1:mNzQ4eTGDg0ll1N9jKPOUo github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= -github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= @@ -489,23 +377,17 @@ github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXS github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= -github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= -github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.5.1 h1:auzK7OI497k6x4OvWq+TKAcpcSAlod0doAH72oIN0Jw= -github.com/onsi/ginkgo/v2 v2.5.1/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc= -github.com/onsi/gomega v1.24.0 h1:+0glovB9Jd6z3VR+ScSwQqXVTIfJcGA9UBM8yzQxhqg= -github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.3-0.20211123151946-c2389c3cb60a h1:9iT75RHhYHWwWRlVWU7wnmtFulYcURCglzQOpT+cAF8= github.com/opencontainers/runtime-spec v1.0.3-0.20211123151946-c2389c3cb60a/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/openzipkin/zipkin-go v0.4.1 h1:kNd/ST2yLLWhaWrkgchya40TJabe8Hioj9udfPcEO5A= github.com/openzipkin/zipkin-go v0.4.1/go.mod h1:qY0VqDSN1pOBN94dBc6w2GJlWLiovAyg7Qt6/I9HecM= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= -github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -516,74 +398,37 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4= github.com/polydawn/refmt v0.89.0/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw= -github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= -github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= -github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= github.com/quic-go/qtls-go1-19 v0.2.1 h1:aJcKNMkH5ASEJB9FXNeZCyTEIHU1J7MmHyz1Q1TSG1A= -github.com/quic-go/qtls-go1-19 v0.2.1/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI= github.com/quic-go/qtls-go1-20 v0.1.1 h1:KbChDlg82d3IHqaj2bn6GfKRj84Per2VGf5XV3wSwQk= -github.com/quic-go/qtls-go1-20 v0.1.1/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM= github.com/quic-go/quic-go v0.33.0 h1:ItNoTDN/Fm/zBlq769lLJc8ECe9gYaW40veHCCco7y0= -github.com/quic-go/quic-go v0.33.0/go.mod h1:YMuhaAV9/jIu0XclDXwZPAsP/2Kgr5yMYhe9oxhhOFA= github.com/quic-go/webtransport-go v0.5.2 h1:GA6Bl6oZY+g/flt00Pnu0XtivSD8vukOu3lYhJjnGEk= -github.com/quic-go/webtransport-go v0.5.2/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= -github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 h1:Lt9DzQALzHoDwMBGJ6v8ObDPR0dzr2a6sXTB1Fq7IHs= github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b h1:h+3JX2VoWTFuyQEo87pStk/a99dzIO1mM9KxIyLPGTU= github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc= -github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= -github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= -github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= -github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= -github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= -github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= -github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI= -github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU= -github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag= -github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg= -github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw= -github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y= -github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= -github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= -github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ= -github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I= -github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0= -github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ= -github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk= -github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= -github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= -github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= -github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -595,7 +440,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -605,20 +449,12 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0= github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb h1:Ywfo8sUltxogBpFuMOFRrrSifO788kAFxmvVw31PtQQ= github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb/go.mod h1:ikPs9bRWicNw3S7XpJ8sK/smGwU9WcSVU3dy9qahYBM= -github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= -github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= -github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= -github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= -github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/warpfork/go-testmark v0.11.0 h1:J6LnV8KpceDvo7spaNU4+DauH2n1x+6RaO2rJrmpQ9U= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= @@ -640,7 +476,6 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -648,8 +483,10 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.40.0 h1:lE9EJyw3/JhrjWH/hEy9FptnalDQgj7vpbgC2KCCCxE= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.40.0/go.mod h1:pcQ3MM3SWvrA71U4GDqv9UFDJ3HQsW7y5ZO3tDTlUdI= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.42.0 h1:0vzgiFDsCh/jxRCR1xcRrtMoeCu2itXz/PsXst5P8rI= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.42.0/go.mod h1:y0vOY2OKFMOTvwxKfurStPayUUKGHlNeVqNneHmFXr0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 h1:pginetY7+onl4qN1vl0xW/V/v6OBZ0vVdH+esuJgvmM= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0/go.mod h1:XiYsayHc36K3EByOO6nbAXnAWbrUxdjUROCEeeROOH8= go.opentelemetry.io/contrib/propagators/autoprop v0.40.0 h1:Lj33jj7eIrBfIShiK8NU91u2BglKnUS1UUxVemuQJtw= go.opentelemetry.io/contrib/propagators/autoprop v0.40.0/go.mod h1:6QO816FeZ+6zahs6hYqbUCCsnNBm7o+t4iwVySpzcdI= go.opentelemetry.io/contrib/propagators/aws v1.15.0 h1:FLe+bRTMAhEALItDQt1U2S/rdq8/rGGJTJpOpCDvMu0= @@ -660,28 +497,28 @@ go.opentelemetry.io/contrib/propagators/jaeger v1.15.0 h1:xdJjwy5t/8I+TZehMMQ+r2 go.opentelemetry.io/contrib/propagators/jaeger v1.15.0/go.mod h1:tU0nwW4QTvKceNUP60/PQm0FI8zDSwey7gIFt3RR/yw= go.opentelemetry.io/contrib/propagators/ot v1.15.0 h1:iBNejawWy7wWZ5msuZDNcMjBy14Wc0v3gCAXukGHN/Q= go.opentelemetry.io/contrib/propagators/ot v1.15.0/go.mod h1:0P7QQ+MHt6SXR1ATaMpewSiWlp8NbKErNLKcaU4EEKI= -go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM= -go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= +go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= +go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= go.opentelemetry.io/otel/exporters/jaeger v1.14.0 h1:CjbUNd4iN2hHmWekmOqZ+zSCU+dzZppG8XsV+A3oc8Q= go.opentelemetry.io/otel/exporters/jaeger v1.14.0/go.mod h1:4Ay9kk5vELRrbg5z4cpP9EtmQRFap2Wb0woPG4lujZA= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 h1:/fXHZHGvro6MVqV34fJzDhi7sHGpX3Ej/Qjmfn003ho= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0/go.mod h1:UFG7EBMRdXyFstOwH028U0sVf+AvukSGhF0g8+dmNG8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 h1:TKf2uAs2ueguzLaxOCBXNpHxfO/aC7PAdDsSH0IbeRQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0/go.mod h1:HrbCVv40OOLTABmOn1ZWty6CHXkU8DK/Urc43tHug70= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 h1:ap+y8RXX3Mu9apKVtOkM6WSFESLM8K3wNQyOU8sWHcc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0/go.mod h1:5w41DY6S9gZrbjuq6Y+753e96WfPha5IcsOSZTtullM= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.14.0 h1:3jAYbRHQAqzLjd9I4tzxwJ8Pk/N6AqBcF6m1ZHrxG94= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.14.0/go.mod h1:+N7zNjIJv4K+DeX67XXET0P+eIciESgaFDBqh+ZJFS4= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0/go.mod h1:vLarbg68dH2Wa77g71zmKQqlQ8+8Rq3GRG31uc0WcWI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo5x2wazq10SKz8hEbtCRPcU78= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0 h1:TVQp/bboR4mhZSav+MdgXB8FaRho1RC8UwVn3T0vjVc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0/go.mod h1:I33vtIe0sR96wfrUcilIzLoA3mLHhRmz9S9Te0S3gDo= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 h1:iqjq9LAB8aK++sKVcELezzn655JnBNdsDhghU4G/So8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0/go.mod h1:hGXzO5bhhSHZnKvrDaXB82Y9DRFour0Nz/KrBh7reWw= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0 h1:sEL90JjOO/4yhquXl5zTAkLLsZ5+MycAgX99SDsxGc8= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.14.0/go.mod h1:oCslUcizYdpKYyS9e8srZEqM6BB8fq41VJBjLAE6z1w= go.opentelemetry.io/otel/exporters/zipkin v1.14.0 h1:reEVE1upBF9tcujgvSqLJS0SrI7JQPaTKP4s4rymnSs= go.opentelemetry.io/otel/exporters/zipkin v1.14.0/go.mod h1:RcjvOAcvhzcufQP8aHmzRw1gE9g/VEZufDdo2w+s4sk= -go.opentelemetry.io/otel/metric v0.37.0 h1:pHDQuLQOZwYD+Km0eb657A25NaRzy0a+eLyKfDXedEs= -go.opentelemetry.io/otel/metric v0.37.0/go.mod h1:DmdaHfGt54iV6UKxsV9slj2bBRJcKC1B1uvDLIioc1s= -go.opentelemetry.io/otel/sdk v1.14.0 h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY= -go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM= -go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M= -go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= +go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= +go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= +go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= +go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= +go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= +go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= @@ -690,9 +527,7 @@ go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/dig v1.15.0 h1:vq3YWr8zRj1eFGC7Gvf907hE0eRjPTZ1d3xHadD6liE= -go.uber.org/dig v1.15.0/go.mod h1:pKHs0wMynzL6brANhB2hLMro+zalv1osARTviTcqHLM= go.uber.org/fx v1.18.2 h1:bUNI6oShr+OVFQeU8cDNbnN7VFsu+SsjHzUF51V/GAU= -go.uber.org/fx v1.18.2/go.mod h1:g0V1KMQ66zIRk8bLu3Ea5Jt2w/cHlOIp4wdRsgh0JaY= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -704,19 +539,13 @@ go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= -golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= -golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -733,7 +562,6 @@ golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb h1:PaBZQdo+iSDyHT053FjUCgZQ/ golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -753,18 +581,13 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= -golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -788,22 +611,18 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -816,16 +635,10 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -833,13 +646,10 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -850,7 +660,6 @@ golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -863,8 +672,8 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -875,16 +684,12 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -929,11 +734,10 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.5.0 h1:+bSpV5HIeWkuvgaMfI3UmKRThoTA5ODJTUd8T17NO+4= -golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -942,9 +746,6 @@ golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3j golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= -google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= -google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= -google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -962,18 +763,12 @@ google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= -google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1004,11 +799,8 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= -google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= -google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1026,8 +818,8 @@ google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= -google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= +google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1047,10 +839,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1059,8 +848,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1071,9 +858,6 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= -nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= -sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/graph_gateway_test.go b/graph_gateway_test.go new file mode 100644 index 0000000..88db33a --- /dev/null +++ b/graph_gateway_test.go @@ -0,0 +1,1195 @@ +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + _ "embed" + + "github.com/ipfs/bifrost-gateway/lib" + "github.com/ipfs/boxo/blockservice" + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/exchange/offline" + "github.com/ipfs/boxo/files" + "github.com/ipfs/boxo/gateway" + "github.com/ipfs/boxo/ipld/merkledag" + unixfile "github.com/ipfs/boxo/ipld/unixfs/file" + "github.com/ipfs/go-cid" + carv2 "github.com/ipld/go-car/v2" + carbs "github.com/ipld/go-car/v2/blockstore" + "github.com/ipld/go-car/v2/storage" +) + +//go:embed lib/fixtures/directory-with-multilayer-hamt-and-multiblock-files.car +var dirWithMultiblockHAMTandFiles []byte + +func TestTar(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + requestNum := 0 + s := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + requestNum++ + switch requestNum { + case 1: + // Expect the full request, but return one that terminates in the middle of the HAMT + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + "bafybeifdv255wmsrh75vcsrtkcwyktvewgihegeeyhhj2ju4lzt4lqfoze", // basicDir + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // exampleA + "bafkreie5noke3mb7hqxukzcy73nl23k6lxszxi5w3dtmuwz62wnvkpsscm", + "bafkreih4ephajybraj6wnxsbwjwa77fukurtpl7oj7t7pfq545duhot7cq", + "bafkreigu7buvm3cfunb35766dn7tmqyh2um62zcio63en2btvxuybgcpue", + "bafkreicll3huefkc3qnrzeony7zcfo7cr3nbx64hnxrqzsixpceg332fhe", + "bafkreifst3pqztuvj57lycamoi7z34b4emf7gawxs74nwrc2c7jncmpaqm", + "bafybeid3trcauvcp7fxaai23gkz3qexmlfxnnejgwm57hdvre472dafvha", // exampleB + "bafkreihgbi345degbcyxaf5b3boiyiaxhnuxdysvqmbdyaop2swmhh3s3m", + "bafkreiaugmh5gal5rgiems6gslcdt2ixfncahintrmcqvrgxqamwtlrmz4", + "bafkreiaxwwb7der2qvmteymgtlj7ww7w5vc44phdxfnexog3vnuwdkxuea", + "bafkreic5zyan5rk4ccfum4d4mu3h5eqsllbudlj4texlzj6xdgxvldzngi", + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamtDir + }); err != nil { + panic(err) + } + case 2: + // Expect a request for the HAMT only and give it + // Note: this is an implementation detail, it could be in the future that we request less or more data + // (e.g. requesting the blocks to fill out the HAMT, or with spec changes asking for HAMT ranges, or asking for the HAMT and its children) + expectedUri := "/ipfs/bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamtDir + "bafybeiccgo7euew77gkqkhezn3pozfrciiibqz2u3spdqmgjvd5wqskipm", + "bafybeihjydob4eq5j4m43whjgf5cgftthc42kjno3g24sa3wcw7vonbmfy", + }); err != nil { + panic(err) + } + case 3: + // Starting here expect requests for each file in the directory + expectedUri := "/ipfs/bafybeid3trcauvcp7fxaai23gkz3qexmlfxnnejgwm57hdvre472dafvha" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3trcauvcp7fxaai23gkz3qexmlfxnnejgwm57hdvre472dafvha", // exampleB + "bafkreihgbi345degbcyxaf5b3boiyiaxhnuxdysvqmbdyaop2swmhh3s3m", + "bafkreiaugmh5gal5rgiems6gslcdt2ixfncahintrmcqvrgxqamwtlrmz4", + "bafkreiaxwwb7der2qvmteymgtlj7ww7w5vc44phdxfnexog3vnuwdkxuea", + "bafkreic5zyan5rk4ccfum4d4mu3h5eqsllbudlj4texlzj6xdgxvldzngi", + }); err != nil { + panic(err) + } + case 4: + // Expect a request for one of the directory items and give it + expectedUri := "/ipfs/bafkreih2grj7p2bo5yk2guqazxfjzapv6hpm3mwrinv6s3cyayd72ke5he" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafkreih2grj7p2bo5yk2guqazxfjzapv6hpm3mwrinv6s3cyayd72ke5he", // exampleD + }); err != nil { + panic(err) + } + case 5: + // Expect a request for one of the directory items and give it + expectedUri := "/ipfs/bafkreidqhbqn5htm5qejxpb3hps7dookudo3nncfn6al6niqibi5lq6fee" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafkreidqhbqn5htm5qejxpb3hps7dookudo3nncfn6al6niqibi5lq6fee", // exampleC + }); err != nil { + panic(err) + } + case 6: + // Expect a request for one of the directory items and give part of it + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // exampleA + "bafkreie5noke3mb7hqxukzcy73nl23k6lxszxi5w3dtmuwz62wnvkpsscm", + "bafkreih4ephajybraj6wnxsbwjwa77fukurtpl7oj7t7pfq545duhot7cq", + }); err != nil { + panic(err) + } + case 7: + // Expect a partial request for one of the directory items and give it + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // exampleA + "bafkreigu7buvm3cfunb35766dn7tmqyh2um62zcio63en2btvxuybgcpue", + "bafkreicll3huefkc3qnrzeony7zcfo7cr3nbx64hnxrqzsixpceg332fhe", + "bafkreifst3pqztuvj57lycamoi7z34b4emf7gawxs74nwrc2c7jncmpaqm", + }); err != nil { + panic(err) + } + default: + t.Fatal("unsupported request number") + } + })) + defer s.Close() + + bs := newProxyBlockStore([]string{s.URL}, newCachedDNS(dnsCacheRefreshInterval)) + backend, err := lib.NewGraphGatewayBackend(&retryFetcher{inner: bs.(lib.CarFetcher), allowedRetries: 3, retriesRemaining: 3}) + if err != nil { + t.Fatal(err) + } + + p, err := gateway.NewImmutablePath(path.New("/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi")) + if err != nil { + t.Fatal(err) + } + _, nd, err := backend.GetAll(ctx, p) + if err != nil { + t.Fatal(err) + } + + assertNextEntryNameEquals := func(t *testing.T, dirIter files.DirIterator, expectedName string) { + t.Helper() + if !dirIter.Next() { + iterErr := dirIter.Err() + t.Fatalf("expected entry, but errored with %s", iterErr.Error()) + } + if expectedName != dirIter.Name() { + t.Fatalf("expected %s, got %s", expectedName, dirIter.Name()) + } + } + + robs, err := carbs.NewReadOnly(bytes.NewReader(dirWithMultiblockHAMTandFiles), nil) + if err != nil { + t.Fatal(err) + } + + dsrv := merkledag.NewDAGService(blockservice.New(robs, offline.Exchange(robs))) + assertFileEqual := func(t *testing.T, expectedCidString string, receivedFile files.File) { + t.Helper() + + expected := cid.MustParse(expectedCidString) + receivedFileData, err := io.ReadAll(receivedFile) + if err != nil { + t.Fatal(err) + } + nd, err := dsrv.Get(ctx, expected) + if err != nil { + t.Fatal(err) + } + expectedFile, err := unixfile.NewUnixfsFile(ctx, dsrv, nd) + if err != nil { + t.Fatal(err) + } + + expectedFileData, err := io.ReadAll(expectedFile.(files.File)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(expectedFileData, receivedFileData) { + t.Fatalf("expected %s, got %s", string(expectedFileData), string(receivedFileData)) + } + } + + rootDirIter := nd.(files.Directory).Entries() + assertNextEntryNameEquals(t, rootDirIter, "basicDir") + + basicDirIter := rootDirIter.Node().(files.Directory).Entries() + assertNextEntryNameEquals(t, basicDirIter, "exampleA") + assertFileEqual(t, "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", basicDirIter.Node().(files.File)) + + assertNextEntryNameEquals(t, basicDirIter, "exampleB") + assertFileEqual(t, "bafybeid3trcauvcp7fxaai23gkz3qexmlfxnnejgwm57hdvre472dafvha", basicDirIter.Node().(files.File)) + + assertNextEntryNameEquals(t, rootDirIter, "hamtDir") + hamtDirIter := rootDirIter.Node().(files.Directory).Entries() + + assertNextEntryNameEquals(t, hamtDirIter, "exampleB") + assertFileEqual(t, "bafybeid3trcauvcp7fxaai23gkz3qexmlfxnnejgwm57hdvre472dafvha", hamtDirIter.Node().(files.File)) + + assertNextEntryNameEquals(t, hamtDirIter, "exampleD-hamt-collide-exampleB-seed-364") + assertFileEqual(t, "bafkreih2grj7p2bo5yk2guqazxfjzapv6hpm3mwrinv6s3cyayd72ke5he", hamtDirIter.Node().(files.File)) + + assertNextEntryNameEquals(t, hamtDirIter, "exampleC-hamt-collide-exampleA-seed-52") + assertFileEqual(t, "bafkreidqhbqn5htm5qejxpb3hps7dookudo3nncfn6al6niqibi5lq6fee", hamtDirIter.Node().(files.File)) + + assertNextEntryNameEquals(t, hamtDirIter, "exampleA") + assertFileEqual(t, "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", hamtDirIter.Node().(files.File)) + + if rootDirIter.Next() || basicDirIter.Next() || hamtDirIter.Next() { + t.Fatal("expected directories to be fully enumerated") + } +} + +func TestTarAtEndOfPath(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + requestNum := 0 + s := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + requestNum++ + switch requestNum { + case 1: + // Expect the full request, but return one that terminates in the middle of the path + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + }); err != nil { + panic(err) + } + case 2: + // Expect the full request and give the path and the children from one of the HAMT nodes but not the other + // Note: this is an implementation detail, it could be in the future that we request less or more data + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamtDir + "bafybeiccgo7euew77gkqkhezn3pozfrciiibqz2u3spdqmgjvd5wqskipm", + "bafybeid3trcauvcp7fxaai23gkz3qexmlfxnnejgwm57hdvre472dafvha", // exampleB + "bafkreihgbi345degbcyxaf5b3boiyiaxhnuxdysvqmbdyaop2swmhh3s3m", + "bafkreiaugmh5gal5rgiems6gslcdt2ixfncahintrmcqvrgxqamwtlrmz4", + "bafkreiaxwwb7der2qvmteymgtlj7ww7w5vc44phdxfnexog3vnuwdkxuea", + "bafkreic5zyan5rk4ccfum4d4mu3h5eqsllbudlj4texlzj6xdgxvldzngi", + "bafkreih2grj7p2bo5yk2guqazxfjzapv6hpm3mwrinv6s3cyayd72ke5he", // exampleD + }); err != nil { + panic(err) + } + case 3: + // Expect a request for the HAMT only and give it + // Note: this is an implementation detail, it could be in the future that we request less or more data + // (e.g. requesting the blocks to fill out the HAMT, or with spec changes asking for HAMT ranges, or asking for the HAMT and its children) + expectedUri := "/ipfs/bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamtDir + "bafybeiccgo7euew77gkqkhezn3pozfrciiibqz2u3spdqmgjvd5wqskipm", + "bafybeihjydob4eq5j4m43whjgf5cgftthc42kjno3g24sa3wcw7vonbmfy", + }); err != nil { + panic(err) + } + case 4: + // Expect a request for one of the directory items and give it + expectedUri := "/ipfs/bafkreidqhbqn5htm5qejxpb3hps7dookudo3nncfn6al6niqibi5lq6fee" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafkreidqhbqn5htm5qejxpb3hps7dookudo3nncfn6al6niqibi5lq6fee", // exampleC + }); err != nil { + panic(err) + } + case 5: + // Expect a request for the multiblock file in the directory and give some of it + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // exampleA + "bafkreie5noke3mb7hqxukzcy73nl23k6lxszxi5w3dtmuwz62wnvkpsscm", + "bafkreih4ephajybraj6wnxsbwjwa77fukurtpl7oj7t7pfq545duhot7cq", + "bafkreigu7buvm3cfunb35766dn7tmqyh2um62zcio63en2btvxuybgcpue", + }); err != nil { + panic(err) + } + case 6: + // Expect a request for the rest of the multiblock file in the directory and give it + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa?format=car&dag-scope=entity&entity-bytes=768:*" + if request.RequestURI != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // exampleA + "bafkreicll3huefkc3qnrzeony7zcfo7cr3nbx64hnxrqzsixpceg332fhe", + "bafkreifst3pqztuvj57lycamoi7z34b4emf7gawxs74nwrc2c7jncmpaqm", + }); err != nil { + panic(err) + } + default: + t.Fatal("unsupported request number") + } + })) + defer s.Close() + + bs := newProxyBlockStore([]string{s.URL}, newCachedDNS(dnsCacheRefreshInterval)) + backend, err := lib.NewGraphGatewayBackend(&retryFetcher{inner: bs.(lib.CarFetcher), allowedRetries: 3, retriesRemaining: 3}) + if err != nil { + t.Fatal(err) + } + + p, err := gateway.NewImmutablePath(path.New("/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir")) + if err != nil { + t.Fatal(err) + } + _, nd, err := backend.GetAll(ctx, p) + if err != nil { + t.Fatal(err) + } + + assertNextEntryNameEquals := func(t *testing.T, dirIter files.DirIterator, expectedName string) { + t.Helper() + if !dirIter.Next() { + t.Fatal("expected entry") + } + if expectedName != dirIter.Name() { + t.Fatalf("expected %s, got %s", expectedName, dirIter.Name()) + } + } + + robs, err := carbs.NewReadOnly(bytes.NewReader(dirWithMultiblockHAMTandFiles), nil) + if err != nil { + t.Fatal(err) + } + + dsrv := merkledag.NewDAGService(blockservice.New(robs, offline.Exchange(robs))) + assertFileEqual := func(t *testing.T, expectedCidString string, receivedFile files.File) { + t.Helper() + + expected := cid.MustParse(expectedCidString) + receivedFileData, err := io.ReadAll(receivedFile) + if err != nil { + t.Fatal(err) + } + nd, err := dsrv.Get(ctx, expected) + if err != nil { + t.Fatal(err) + } + expectedFile, err := unixfile.NewUnixfsFile(ctx, dsrv, nd) + if err != nil { + t.Fatal(err) + } + + expectedFileData, err := io.ReadAll(expectedFile.(files.File)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(expectedFileData, receivedFileData) { + t.Fatalf("expected %s, got %s", string(expectedFileData), string(receivedFileData)) + } + } + + hamtDirIter := nd.(files.Directory).Entries() + + assertNextEntryNameEquals(t, hamtDirIter, "exampleB") + assertFileEqual(t, "bafybeid3trcauvcp7fxaai23gkz3qexmlfxnnejgwm57hdvre472dafvha", hamtDirIter.Node().(files.File)) + + assertNextEntryNameEquals(t, hamtDirIter, "exampleD-hamt-collide-exampleB-seed-364") + assertFileEqual(t, "bafkreih2grj7p2bo5yk2guqazxfjzapv6hpm3mwrinv6s3cyayd72ke5he", hamtDirIter.Node().(files.File)) + + assertNextEntryNameEquals(t, hamtDirIter, "exampleC-hamt-collide-exampleA-seed-52") + assertFileEqual(t, "bafkreidqhbqn5htm5qejxpb3hps7dookudo3nncfn6al6niqibi5lq6fee", hamtDirIter.Node().(files.File)) + + assertNextEntryNameEquals(t, hamtDirIter, "exampleA") + assertFileEqual(t, "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", hamtDirIter.Node().(files.File)) + + if hamtDirIter.Next() { + t.Fatal("expected directories to be fully enumerated") + } +} + +func sendBlocks(ctx context.Context, carFixture []byte, writer io.Writer, cidStrList []string) error { + rd, err := storage.OpenReadable(bytes.NewReader(carFixture)) + if err != nil { + return err + } + + cw, err := storage.NewWritable(writer, []cid.Cid{cid.MustParse("bafkqaaa")}, carv2.WriteAsCarV1(true), carv2.AllowDuplicatePuts(true)) + if err != nil { + return err + } + + for _, s := range cidStrList { + c := cid.MustParse(s) + blockData, err := rd.Get(ctx, c.KeyString()) + if err != nil { + return err + } + + if err := cw.Put(ctx, c.KeyString(), blockData); err != nil { + return err + } + } + return nil +} + +func TestGetFile(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + requestNum := 0 + s := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + requestNum++ + switch requestNum { + case 1: + // Expect the full request, but return one that terminates in the middle of the path + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/exampleA" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + }); err != nil { + panic(err) + } + case 2: + // Expect the full request, but return one that terminates in the middle of the file + // Note: this is an implementation detail, it could be in the future that we request less data (e.g. partial path) + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/exampleA" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamt root + }); err != nil { + panic(err) + } + + case 3: + // Expect the full request and return the path and most of the file + // Note: this is an implementation detail, it could be in the future that we request less data (e.g. partial path and file range) + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/exampleA" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamt root + "bafybeihjydob4eq5j4m43whjgf5cgftthc42kjno3g24sa3wcw7vonbmfy", // inner hamt + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // file root + "bafkreie5noke3mb7hqxukzcy73nl23k6lxszxi5w3dtmuwz62wnvkpsscm", // file chunks start here + "bafkreih4ephajybraj6wnxsbwjwa77fukurtpl7oj7t7pfq545duhot7cq", + }); err != nil { + panic(err) + } + + case 4: + // Expect a request for the remainder of the file + // Note: this is an implementation detail, it could be that the requester really asks for more information + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // file root + "bafkreigu7buvm3cfunb35766dn7tmqyh2um62zcio63en2btvxuybgcpue", // middle of the file starts here + "bafkreicll3huefkc3qnrzeony7zcfo7cr3nbx64hnxrqzsixpceg332fhe", + "bafkreifst3pqztuvj57lycamoi7z34b4emf7gawxs74nwrc2c7jncmpaqm", + }); err != nil { + panic(err) + } + + default: + t.Fatal("unsupported request number") + } + })) + defer s.Close() + + bs := newProxyBlockStore([]string{s.URL}, newCachedDNS(dnsCacheRefreshInterval)) + backend, err := lib.NewGraphGatewayBackend(&retryFetcher{inner: bs.(lib.CarFetcher), allowedRetries: 3, retriesRemaining: 3}) + if err != nil { + t.Fatal(err) + } + + trustedGatewayServer := httptest.NewServer(gateway.NewHandler(gateway.Config{DeserializedResponses: true}, backend)) + defer trustedGatewayServer.Close() + + resp, err := http.Get(trustedGatewayServer.URL + "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/exampleA") + if err != nil { + t.Fatal(err) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + + robs, err := carbs.NewReadOnly(bytes.NewReader(dirWithMultiblockHAMTandFiles), nil) + if err != nil { + t.Fatal(err) + } + + dsrv := merkledag.NewDAGService(blockservice.New(robs, offline.Exchange(robs))) + fileRootNd, err := dsrv.Get(ctx, cid.MustParse("bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa")) + if err != nil { + t.Fatal(err) + } + uio, err := unixfile.NewUnixfsFile(ctx, dsrv, fileRootNd) + if err != nil { + t.Fatal(err) + } + f := uio.(files.File) + expectedFileData, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(data, expectedFileData) { + t.Fatalf("expected %s, got %s", string(expectedFileData), string(data)) + } +} + +func TestGetFileRangeRequest(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + requestNum := 0 + s := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + requestNum++ + switch requestNum { + case 1: + // Expect the full request, but return one that terminates at the root block + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // file root + }); err != nil { + panic(err) + } + case 2: + // Expect the full request, and return the whole file which should be invalid + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // file root + "bafkreie5noke3mb7hqxukzcy73nl23k6lxszxi5w3dtmuwz62wnvkpsscm", // file chunks start here + "bafkreih4ephajybraj6wnxsbwjwa77fukurtpl7oj7t7pfq545duhot7cq", + "bafkreigu7buvm3cfunb35766dn7tmqyh2um62zcio63en2btvxuybgcpue", + "bafkreicll3huefkc3qnrzeony7zcfo7cr3nbx64hnxrqzsixpceg332fhe", + "bafkreifst3pqztuvj57lycamoi7z34b4emf7gawxs74nwrc2c7jncmpaqm", + }); err != nil { + panic(err) + } + case 3: + // Expect the full request and return the first block + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // file root + "bafkreih4ephajybraj6wnxsbwjwa77fukurtpl7oj7t7pfq545duhot7cq", + }); err != nil { + panic(err) + } + + case 4: + // Expect a request for the remainder of the file + // Note: this is an implementation detail, it could be that the requester really asks for more information + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // file root + "bafkreigu7buvm3cfunb35766dn7tmqyh2um62zcio63en2btvxuybgcpue", + "bafkreicll3huefkc3qnrzeony7zcfo7cr3nbx64hnxrqzsixpceg332fhe", + }); err != nil { + panic(err) + } + + default: + t.Fatal("unsupported request number") + } + })) + defer s.Close() + + bs := newProxyBlockStore([]string{s.URL}, newCachedDNS(dnsCacheRefreshInterval)) + backend, err := lib.NewGraphGatewayBackend(&retryFetcher{inner: bs.(lib.CarFetcher), allowedRetries: 3, retriesRemaining: 3}) + if err != nil { + t.Fatal(err) + } + + trustedGatewayServer := httptest.NewServer(gateway.NewHandler(gateway.Config{DeserializedResponses: true}, backend)) + defer trustedGatewayServer.Close() + + req, err := http.NewRequestWithContext(ctx, "GET", trustedGatewayServer.URL+"/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", nil) + if err != nil { + t.Fatal(err) + } + startIndex := 256 + endIndex := 750 + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", startIndex, endIndex)) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + + robs, err := carbs.NewReadOnly(bytes.NewReader(dirWithMultiblockHAMTandFiles), nil) + if err != nil { + t.Fatal(err) + } + + dsrv := merkledag.NewDAGService(blockservice.New(robs, offline.Exchange(robs))) + fileRootNd, err := dsrv.Get(ctx, cid.MustParse("bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa")) + if err != nil { + t.Fatal(err) + } + uio, err := unixfile.NewUnixfsFile(ctx, dsrv, fileRootNd) + if err != nil { + t.Fatal(err) + } + f := uio.(files.File) + if _, err := f.Seek(int64(startIndex), io.SeekStart); err != nil { + t.Fatal(err) + } + expectedFileData, err := io.ReadAll(io.LimitReader(f, int64(endIndex)-int64(startIndex)+1)) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(data, expectedFileData) { + t.Fatalf("expected %s, got %s", string(expectedFileData), string(data)) + } + + if requestNum != 4 { + t.Fatalf("expected exactly 4 requests, got %d", requestNum) + } +} + +func TestGetFileWithBadBlockReturned(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + requestNum := 0 + s := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + requestNum++ + switch requestNum { + case 1: + // Expect the full request, but return one that terminates at the root block + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // file root + }); err != nil { + panic(err) + } + case 2: + // Expect the full request, but return a totally unrelated block + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // file root + }); err != nil { + panic(err) + } + case 3: + // Expect the full request and return most of the file + // Note: this is an implementation detail, it could be in the future that we request less data (e.g. partial path and file range) + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // file root + "bafkreie5noke3mb7hqxukzcy73nl23k6lxszxi5w3dtmuwz62wnvkpsscm", // file chunks start here + "bafkreih4ephajybraj6wnxsbwjwa77fukurtpl7oj7t7pfq545duhot7cq", + }); err != nil { + panic(err) + } + + case 4: + // Expect a request for the remainder of the file + // Note: this is an implementation detail, it could be that the requester really asks for more information + expectedUri := "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // file root + "bafkreigu7buvm3cfunb35766dn7tmqyh2um62zcio63en2btvxuybgcpue", // middle of the file starts here + "bafkreicll3huefkc3qnrzeony7zcfo7cr3nbx64hnxrqzsixpceg332fhe", + "bafkreifst3pqztuvj57lycamoi7z34b4emf7gawxs74nwrc2c7jncmpaqm", + }); err != nil { + panic(err) + } + + default: + t.Fatal("unsupported request number") + } + })) + defer s.Close() + + bs := newProxyBlockStore([]string{s.URL}, newCachedDNS(dnsCacheRefreshInterval)) + backend, err := lib.NewGraphGatewayBackend(&retryFetcher{inner: bs.(lib.CarFetcher), allowedRetries: 3, retriesRemaining: 3}) + if err != nil { + t.Fatal(err) + } + + trustedGatewayServer := httptest.NewServer(gateway.NewHandler(gateway.Config{DeserializedResponses: true}, backend)) + defer trustedGatewayServer.Close() + + resp, err := http.Get(trustedGatewayServer.URL + "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa") + if err != nil { + t.Fatal(err) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + + robs, err := carbs.NewReadOnly(bytes.NewReader(dirWithMultiblockHAMTandFiles), nil) + if err != nil { + t.Fatal(err) + } + + dsrv := merkledag.NewDAGService(blockservice.New(robs, offline.Exchange(robs))) + fileRootNd, err := dsrv.Get(ctx, cid.MustParse("bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa")) + if err != nil { + t.Fatal(err) + } + uio, err := unixfile.NewUnixfsFile(ctx, dsrv, fileRootNd) + if err != nil { + t.Fatal(err) + } + f := uio.(files.File) + expectedFileData, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(data, expectedFileData) { + t.Fatalf("expected %s, got %s", string(expectedFileData), string(data)) + } +} + +func TestGetHAMTDirectory(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + requestNum := 0 + s := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + requestNum++ + switch requestNum { + case 1: + // Expect the full request, but return one that terminates in the middle of the path + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + }); err != nil { + panic(err) + } + case 2: + // Expect the full request, but return one that terminates in the middle of the HAMT + // Note: this is an implementation detail, it could be in the future that we request less data (e.g. partial path) + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamt root + "bafybeiccgo7euew77gkqkhezn3pozfrciiibqz2u3spdqmgjvd5wqskipm", // inner hamt nodes start here + }); err != nil { + panic(err) + } + case 3: + // Expect a request for a non-existent index.html file + // Note: this is an implementation detail related to the directory request above + // Note: the order of cases 3 and 4 here are implementation specific as well + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/index.html" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamt root + "bafybeiccgo7euew77gkqkhezn3pozfrciiibqz2u3spdqmgjvd5wqskipm", // inner hamt nodes start here + }); err != nil { + panic(err) + } + case 4: + // Expect a request for the full HAMT and return it + // Note: this is an implementation detail, it could be in the future that we request more or less data + // (e.g. ask for the full path, ask for index.html first, make a spec change to allow asking for index.html with a fallback to the directory, etc.) + expectedUri := "/ipfs/bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamt root + "bafybeiccgo7euew77gkqkhezn3pozfrciiibqz2u3spdqmgjvd5wqskipm", // inner hamt nodes start here + "bafybeihjydob4eq5j4m43whjgf5cgftthc42kjno3g24sa3wcw7vonbmfy", + }); err != nil { + panic(err) + } + + default: + t.Fatal("unsupported request number") + } + })) + defer s.Close() + + bs := newProxyBlockStore([]string{s.URL}, newCachedDNS(dnsCacheRefreshInterval)) + backend, err := lib.NewGraphGatewayBackend(&retryFetcher{inner: bs.(lib.CarFetcher), allowedRetries: 3, retriesRemaining: 3}) + if err != nil { + t.Fatal(err) + } + + trustedGatewayServer := httptest.NewServer(gateway.NewHandler(gateway.Config{DeserializedResponses: true}, backend)) + defer trustedGatewayServer.Close() + + resp, err := http.Get(trustedGatewayServer.URL + "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/") + if err != nil { + t.Fatal(err) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + + if strings.Count(string(data), ">exampleD-hamt-collide-exampleB-seed-364<") == 1 && + strings.Count(string(data), ">exampleC-hamt-collide-exampleA-seed-52<") == 1 && + strings.Count(string(data), ">exampleA<") == 1 && + strings.Count(string(data), ">exampleB<") == 1 { + return + } + t.Fatal("directory does not contain the expected links") +} + +func TestGetCAR(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + requestNum := 0 + s := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + requestNum++ + switch requestNum { + case 1: + // Expect the full request, but return one that terminates in the middle of the path + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + }); err != nil { + panic(err) + } + case 2: + // Expect the full request, but return one that terminates in the middle of the HAMT + // Note: this is an implementation detail, it could be in the future that we request less data (e.g. partial path) + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamt root + }); err != nil { + panic(err) + } + + case 3: + // Expect the full request and return the full HAMT + // Note: this is an implementation detail, it could be in the future that we request less data (e.g. requesting the blocks to fill out the HAMT, or with spec changes asking for HAMT ranges) + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + "bafybeifdv255wmsrh75vcsrtkcwyktvewgihegeeyhhj2ju4lzt4lqfoze", // basicDir + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // exampleA + "bafkreie5noke3mb7hqxukzcy73nl23k6lxszxi5w3dtmuwz62wnvkpsscm", + "bafkreih4ephajybraj6wnxsbwjwa77fukurtpl7oj7t7pfq545duhot7cq", + "bafkreigu7buvm3cfunb35766dn7tmqyh2um62zcio63en2btvxuybgcpue", + "bafkreicll3huefkc3qnrzeony7zcfo7cr3nbx64hnxrqzsixpceg332fhe", + "bafkreifst3pqztuvj57lycamoi7z34b4emf7gawxs74nwrc2c7jncmpaqm", + "bafybeid3trcauvcp7fxaai23gkz3qexmlfxnnejgwm57hdvre472dafvha", // exampleB + "bafkreihgbi345degbcyxaf5b3boiyiaxhnuxdysvqmbdyaop2swmhh3s3m", + "bafkreiaugmh5gal5rgiems6gslcdt2ixfncahintrmcqvrgxqamwtlrmz4", + "bafkreiaxwwb7der2qvmteymgtlj7ww7w5vc44phdxfnexog3vnuwdkxuea", + "bafkreic5zyan5rk4ccfum4d4mu3h5eqsllbudlj4texlzj6xdgxvldzngi", + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamtDir + "bafybeiccgo7euew77gkqkhezn3pozfrciiibqz2u3spdqmgjvd5wqskipm", + "bafybeid3trcauvcp7fxaai23gkz3qexmlfxnnejgwm57hdvre472dafvha", // exampleB + "bafkreihgbi345degbcyxaf5b3boiyiaxhnuxdysvqmbdyaop2swmhh3s3m", + "bafkreiaugmh5gal5rgiems6gslcdt2ixfncahintrmcqvrgxqamwtlrmz4", + "bafkreiaxwwb7der2qvmteymgtlj7ww7w5vc44phdxfnexog3vnuwdkxuea", + "bafkreic5zyan5rk4ccfum4d4mu3h5eqsllbudlj4texlzj6xdgxvldzngi", + "bafkreih2grj7p2bo5yk2guqazxfjzapv6hpm3mwrinv6s3cyayd72ke5he", // exampleD + "bafybeihjydob4eq5j4m43whjgf5cgftthc42kjno3g24sa3wcw7vonbmfy", + "bafkreidqhbqn5htm5qejxpb3hps7dookudo3nncfn6al6niqibi5lq6fee", // exampleC + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // exampleA + "bafkreie5noke3mb7hqxukzcy73nl23k6lxszxi5w3dtmuwz62wnvkpsscm", + "bafkreih4ephajybraj6wnxsbwjwa77fukurtpl7oj7t7pfq545duhot7cq", + "bafkreigu7buvm3cfunb35766dn7tmqyh2um62zcio63en2btvxuybgcpue", + "bafkreicll3huefkc3qnrzeony7zcfo7cr3nbx64hnxrqzsixpceg332fhe", + "bafkreifst3pqztuvj57lycamoi7z34b4emf7gawxs74nwrc2c7jncmpaqm", + }); err != nil { + panic(err) + } + + default: + t.Fatal("unsupported request number") + } + })) + defer s.Close() + + bs := newProxyBlockStore([]string{s.URL}, newCachedDNS(dnsCacheRefreshInterval)) + backend, err := lib.NewGraphGatewayBackend(&retryFetcher{inner: bs.(lib.CarFetcher), allowedRetries: 3, retriesRemaining: 3}) + if err != nil { + t.Fatal(err) + } + + p, err := gateway.NewImmutablePath(path.New("/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi")) + if err != nil { + t.Fatal(err) + } + var carReader io.Reader + _, carReader, err = backend.GetCAR(ctx, p, gateway.CarParams{Scope: gateway.DagScopeAll}) + if err != nil { + t.Fatal(err) + } + + carBytes, err := io.ReadAll(carReader) + if err != nil { + t.Fatal(err) + } + carReader = bytes.NewReader(carBytes) + + blkReader, err := carv2.NewBlockReader(carReader) + if err != nil { + t.Fatal(err) + } + + responseCarBlock := []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + "bafybeifdv255wmsrh75vcsrtkcwyktvewgihegeeyhhj2ju4lzt4lqfoze", // basicDir + "bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", // exampleA + "bafkreie5noke3mb7hqxukzcy73nl23k6lxszxi5w3dtmuwz62wnvkpsscm", + "bafkreih4ephajybraj6wnxsbwjwa77fukurtpl7oj7t7pfq545duhot7cq", + "bafkreigu7buvm3cfunb35766dn7tmqyh2um62zcio63en2btvxuybgcpue", + "bafkreicll3huefkc3qnrzeony7zcfo7cr3nbx64hnxrqzsixpceg332fhe", + "bafkreifst3pqztuvj57lycamoi7z34b4emf7gawxs74nwrc2c7jncmpaqm", + "bafybeid3trcauvcp7fxaai23gkz3qexmlfxnnejgwm57hdvre472dafvha", // exampleB + "bafkreihgbi345degbcyxaf5b3boiyiaxhnuxdysvqmbdyaop2swmhh3s3m", + "bafkreiaugmh5gal5rgiems6gslcdt2ixfncahintrmcqvrgxqamwtlrmz4", + "bafkreiaxwwb7der2qvmteymgtlj7ww7w5vc44phdxfnexog3vnuwdkxuea", + "bafkreic5zyan5rk4ccfum4d4mu3h5eqsllbudlj4texlzj6xdgxvldzngi", + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamtDir + "bafybeiccgo7euew77gkqkhezn3pozfrciiibqz2u3spdqmgjvd5wqskipm", + "bafkreih2grj7p2bo5yk2guqazxfjzapv6hpm3mwrinv6s3cyayd72ke5he", // exampleD + "bafybeihjydob4eq5j4m43whjgf5cgftthc42kjno3g24sa3wcw7vonbmfy", + "bafkreidqhbqn5htm5qejxpb3hps7dookudo3nncfn6al6niqibi5lq6fee", // exampleC + } + + for i := 0; i < len(responseCarBlock); i++ { + expectedCid := cid.MustParse(responseCarBlock[i]) + blk, err := blkReader.Next() + if err != nil { + t.Fatal(err) + } + if !blk.Cid().Equals(expectedCid) { + t.Fatalf("expected cid %s, got %s", expectedCid, blk.Cid()) + } + } + _, err = blkReader.Next() + if !errors.Is(err, io.EOF) { + t.Fatal("expected an EOF") + } +} + +func TestPassthroughErrors(t *testing.T) { + t.Run("PathTraversalError", func(t *testing.T) { + pathTraversalTest := func(t *testing.T, traversal func(ctx context.Context, p gateway.ImmutablePath, backend *lib.GraphGateway) error) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var requestNum int + s := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + requestNum++ + switch requestNum { + case 1: + // Expect the full request, but return one that terminates in the middle of the path + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/exampleA" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + }); err != nil { + panic(err) + } + case 2: + // Expect the full request, but return one that terminates in the middle of the file + // Note: this is an implementation detail, it could be in the future that we request less data (e.g. partial path) + expectedUri := "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/exampleA" + if request.URL.Path != expectedUri { + panic(fmt.Errorf("expected URI %s, got %s", expectedUri, request.RequestURI)) + } + + if err := sendBlocks(ctx, dirWithMultiblockHAMTandFiles, writer, []string{ + "bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi", // root dir + "bafybeignui4g7l6cvqyy4t6vnbl2fjtego4ejmpcia77jhhwnksmm4bejm", // hamt root + }); err != nil { + panic(err) + } + default: + t.Fatal("unsupported request number") + } + })) + defer s.Close() + + bs := newProxyBlockStore([]string{s.URL}, newCachedDNS(dnsCacheRefreshInterval)) + p, err := gateway.NewImmutablePath(path.New("/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/exampleA")) + if err != nil { + t.Fatal(err) + } + + bogusErr := gateway.NewErrorStatusCode(fmt.Errorf("this is a test error"), 418) + + clientRequestNum := 0 + backend, err := lib.NewGraphGatewayBackend(&retryFetcher{ + inner: &fetcherWrapper{fn: func(ctx context.Context, path string, cb lib.DataCallback) error { + clientRequestNum++ + if clientRequestNum > 2 { + return bogusErr + } + return bs.(lib.CarFetcher).Fetch(ctx, path, cb) + }}, + allowedRetries: 3, retriesRemaining: 3}) + if err != nil { + t.Fatal(err) + } + + err = traversal(ctx, p, backend) + parsedErr := &gateway.ErrorStatusCode{} + if errors.As(err, &parsedErr) { + if parsedErr.StatusCode == bogusErr.StatusCode { + return + } + } + t.Fatal("error did not pass through") + } + t.Run("Block", func(t *testing.T) { + pathTraversalTest(t, func(ctx context.Context, p gateway.ImmutablePath, backend *lib.GraphGateway) error { + _, _, err := backend.GetBlock(ctx, p) + return err + }) + }) + t.Run("File", func(t *testing.T) { + pathTraversalTest(t, func(ctx context.Context, p gateway.ImmutablePath, backend *lib.GraphGateway) error { + _, _, err := backend.Get(ctx, p) + return err + }) + }) + }) +} + +type fetcherWrapper struct { + fn func(ctx context.Context, path string, cb lib.DataCallback) error +} + +func (w *fetcherWrapper) Fetch(ctx context.Context, path string, cb lib.DataCallback) error { + return w.fn(ctx, path, cb) +} + +type retryFetcher struct { + inner lib.CarFetcher + allowedRetries int + retriesRemaining int +} + +func (r *retryFetcher) Fetch(ctx context.Context, path string, cb lib.DataCallback) error { + err := r.inner.Fetch(ctx, path, cb) + if err == nil { + return nil + } + + if r.retriesRemaining > 0 { + r.retriesRemaining-- + } else { + return fmt.Errorf("retry fetcher out of retries: %w", err) + } + + switch t := err.(type) { + case lib.ErrPartialResponse: + if len(t.StillNeed) > 1 { + panic("only a single request at a time supported") + } + + // Mimicking the Caboose logic reset the number of retries for partials + r.retriesRemaining = r.allowedRetries + + return r.Fetch(ctx, t.StillNeed[0], cb) + default: + return r.Fetch(ctx, path, cb) + } +} + +var _ lib.CarFetcher = (*retryFetcher)(nil) diff --git a/handlers.go b/handlers.go index 0397a39..fb844dd 100644 --- a/handlers.go +++ b/handlers.go @@ -98,13 +98,7 @@ func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockC return nil, err } } else { - // Sets up an exchange based on the given Block Store - exch, err := newExchange(bs) - if err != nil { - return nil, err - } - - gwAPI, err = lib.NewGraphGatewayBackend(bs.(lib.CarFetcher), exch, lib.WithValueStore(routing), lib.WithBlockstore(cacheBlockStore)) + gwAPI, err = lib.NewGraphGatewayBackend(bs.(lib.CarFetcher), lib.WithValueStore(routing), lib.WithPrometheusRegistry(prometheus.DefaultRegisterer)) if err != nil { return nil, err } diff --git a/lib/files.go b/lib/files.go new file mode 100644 index 0000000..602b818 --- /dev/null +++ b/lib/files.go @@ -0,0 +1,533 @@ +package lib + +import ( + "bytes" + "context" + "fmt" + "github.com/ipfs/boxo/files" + "github.com/ipfs/boxo/gateway" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-unixfsnode" + ufsData "github.com/ipfs/go-unixfsnode/data" + "github.com/ipfs/go-unixfsnode/hamt" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/node/basicnode" + "github.com/ipld/go-ipld-prime/schema" + "github.com/multiformats/go-multicodec" + "io" +) + +type AwaitCloser interface { + AwaitClose() <-chan error +} + +type backpressuredFile struct { + size int64 + f io.ReadSeeker + getLsys lsysGetter + + ctx context.Context + fileCid cid.Cid + byteRange gateway.DagByteRange + retErr error + + closed chan error +} + +func (b *backpressuredFile) AwaitClose() <-chan error { + return b.closed +} + +func (b *backpressuredFile) Close() error { + close(b.closed) + return nil +} + +func (b *backpressuredFile) Size() (int64, error) { + return b.size, nil +} + +func (b *backpressuredFile) Read(p []byte) (n int, err error) { + if b.retErr == nil { + n, err = b.f.Read(p) + if err == nil || err == io.EOF { + return n, err + } + + if n > 0 { + b.retErr = err + return n, nil + } + } else { + err = b.retErr + } + + from, seekErr := b.f.Seek(0, io.SeekCurrent) + if seekErr != nil { + // Return the seek error since by this point seeking failures like this should be impossible + return 0, seekErr + } + + // we had an error while reading so attempt to reset the underlying reader + for { + if b.ctx.Err() != nil { + return 0, b.ctx.Err() + } + + retry, processedErr := isRetryableError(err) + if !retry { + return 0, processedErr + } + + var nd files.Node + nd, err = loadTerminalUnixFSElementWithRecursiveDirectories(b.ctx, b.fileCid, nil, nil, gateway.CarParams{Scope: gateway.DagScopeEntity, Range: &gateway.DagByteRange{From: from, To: b.byteRange.To}}, b.getLsys) + if err != nil { + continue + } + + f, ok := nd.(files.File) + if !ok { + return 0, fmt.Errorf("not a file, should be unreachable") + } + + b.f = f + break + } + + // now that we've reset the reader try reading again + return b.Read(p) +} + +func (b *backpressuredFile) Seek(offset int64, whence int) (int64, error) { + return b.f.Seek(offset, whence) +} + +var _ files.File = (*backpressuredFile)(nil) +var _ AwaitCloser = (*backpressuredFile)(nil) + +type singleUseDirectory struct { + dirIter files.DirIterator + closed chan error +} + +func (b *singleUseDirectory) AwaitClose() <-chan error { + return b.closed +} + +func (b *singleUseDirectory) Close() error { + close(b.closed) + return nil +} + +func (b *singleUseDirectory) Size() (int64, error) { + //TODO implement me + panic("implement me") +} + +func (b *singleUseDirectory) Entries() files.DirIterator { + return b.dirIter +} + +var _ files.Directory = (*singleUseDirectory)(nil) +var _ AwaitCloser = (*singleUseDirectory)(nil) + +type backpressuredFlatDirIter struct { + linksItr *dagpb.PBLinks__Itr + lsys *ipld.LinkSystem + getLsys lsysGetter + ctx context.Context + + curName string + curFile files.Node + + err error +} + +func (it *backpressuredFlatDirIter) Name() string { + return it.curName +} + +func (it *backpressuredFlatDirIter) Node() files.Node { + return it.curFile +} + +func (it *backpressuredFlatDirIter) Next() bool { + if it.err != nil { + return false + } + + iter := it.linksItr + if iter.Done() { + return false + } + + _, v := iter.Next() + c := v.Hash.Link().(cidlink.Link).Cid + var name string + if v.Name.Exists() { + name = v.Name.Must().String() + } + + var nd files.Node + var err error + params := gateway.CarParams{Scope: gateway.DagScopeAll} + for { + if it.ctx.Err() != nil { + it.err = it.ctx.Err() + return false + } + if err != nil { + it.lsys, err = it.getLsys(it.ctx, c, params) + continue + } + nd, err = loadTerminalUnixFSElementWithRecursiveDirectories(it.ctx, c, nil, it.lsys, params, it.getLsys) + if err != nil { + if ctxErr := it.ctx.Err(); ctxErr != nil { + continue + } + retry, processedErr := isRetryableError(err) + if retry { + err = processedErr + continue + } + it.err = processedErr + return false + } + break + } + + it.curName = name + it.curFile = nd + return true +} + +func (it *backpressuredFlatDirIter) Err() error { + return it.err +} + +var _ files.DirIterator = (*backpressuredFlatDirIter)(nil) + +type backpressuredHAMTDirIter struct { + linksItr ipld.MapIterator + dirCid cid.Cid + + lsys *ipld.LinkSystem + getLsys lsysGetter + ctx context.Context + + curName string + curFile files.Node + curProcessed int + + err error +} + +func (it *backpressuredHAMTDirIter) Name() string { + return it.curName +} + +func (it *backpressuredHAMTDirIter) Node() files.Node { + return it.curFile +} + +func (it *backpressuredHAMTDirIter) Next() bool { + if it.err != nil { + return false + } + + iter := it.linksItr + if iter.Done() { + return false + } + + /* + Since there is no way to make a graph request for part of a HAMT during errors we can either fill in the HAMT with + block requests, or we can re-request the HAMT and skip over the parts we already have. + + Here we choose the latter, however in the event of a re-request we request the entity rather than the entire DAG as + a compromise between more requests and over-fetching data. + */ + + var err error + for { + if it.ctx.Err() != nil { + it.err = it.ctx.Err() + return false + } + + retry, processedErr := isRetryableError(err) + if !retry { + it.err = processedErr + return false + } + + var nd ipld.Node + if err != nil { + var lsys *ipld.LinkSystem + lsys, err = it.getLsys(it.ctx, it.dirCid, gateway.CarParams{Scope: gateway.DagScopeEntity}) + if err != nil { + continue + } + + _, pbn, ufsFieldData, _, ufsBaseErr := loadUnixFSBase(it.ctx, it.dirCid, nil, lsys) + if ufsBaseErr != nil { + err = ufsBaseErr + continue + } + + nd, err = hamt.NewUnixFSHAMTShard(it.ctx, pbn, ufsFieldData, lsys) + if err != nil { + err = fmt.Errorf("could not reify sharded directory: %w", err) + continue + } + + iter = nd.MapIterator() + for i := 0; i < it.curProcessed; i++ { + _, _, err = iter.Next() + if err != nil { + continue + } + } + + it.linksItr = iter + } + + var k, v ipld.Node + k, v, err = iter.Next() + if err != nil { + retry, processedErr = isRetryableError(err) + if retry { + err = processedErr + continue + } + it.err = processedErr + return false + } + + var name string + name, err = k.AsString() + if err != nil { + it.err = err + return false + } + var lnk ipld.Link + lnk, err = v.AsLink() + if err != nil { + it.err = err + return false + } + + cl, ok := lnk.(cidlink.Link) + if !ok { + it.err = fmt.Errorf("link not a cidlink") + return false + } + + c := cl.Cid + params := gateway.CarParams{Scope: gateway.DagScopeAll} + var childNd files.Node + for { + if it.ctx.Err() != nil { + it.err = it.ctx.Err() + return false + } + + if err != nil { + retry, processedErr = isRetryableError(err) + if !retry { + it.err = processedErr + return false + } + + it.lsys, err = it.getLsys(it.ctx, c, params) + continue + } + + childNd, err = loadTerminalUnixFSElementWithRecursiveDirectories(it.ctx, c, nil, it.lsys, params, it.getLsys) + if err != nil { + continue + } + break + } + + it.curName = name + it.curFile = childNd + it.curProcessed++ + break + } + + return true +} + +func (it *backpressuredHAMTDirIter) Err() error { + return it.err +} + +var _ files.DirIterator = (*backpressuredHAMTDirIter)(nil) + +/* +1. Run traversal to get the top-level response +2. Response can do a callback for another response +*/ + +type lsysGetter = func(ctx context.Context, c cid.Cid, params gateway.CarParams) (*ipld.LinkSystem, error) + +func loadUnixFSBase(ctx context.Context, c cid.Cid, blk blocks.Block, lsys *ipld.LinkSystem) ([]byte, dagpb.PBNode, ufsData.UnixFSData, int64, error) { + lctx := ipld.LinkContext{Ctx: ctx} + pathTerminalCidLink := cidlink.Link{Cid: c} + + var blockData []byte + var err error + + if blk != nil { + blockData = blk.RawData() + } else { + blockData, err = lsys.LoadRaw(lctx, pathTerminalCidLink) + if err != nil { + return nil, nil, nil, 0, err + } + } + + if c.Type() == uint64(multicodec.Raw) { + return blockData, nil, nil, 0, nil + } + + // decode the terminal block into a node + pc := dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { + if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { + return tlnkNd.LinkTargetNodePrototype(), nil + } + return basicnode.Prototype.Any, nil + }) + + np, err := pc(pathTerminalCidLink, lctx) + if err != nil { + return nil, nil, nil, 0, err + } + + decoder, err := lsys.DecoderChooser(pathTerminalCidLink) + if err != nil { + return nil, nil, nil, 0, err + } + nb := np.NewBuilder() + if err := decoder(nb, bytes.NewReader(blockData)); err != nil { + return nil, nil, nil, 0, err + } + lastCidNode := nb.Build() + + if pbn, ok := lastCidNode.(dagpb.PBNode); !ok { + // If it's not valid dag-pb then we're done + return nil, nil, nil, 0, errNotUnixFS + } else if !pbn.FieldData().Exists() { + // If it's not valid UnixFS then we're done + return nil, nil, nil, 0, errNotUnixFS + } else if unixfsFieldData, decodeErr := ufsData.DecodeUnixFSData(pbn.Data.Must().Bytes()); decodeErr != nil { + return nil, nil, nil, 0, errNotUnixFS + } else { + switch fieldNum := unixfsFieldData.FieldDataType().Int(); fieldNum { + case ufsData.Data_Symlink, ufsData.Data_Metadata, ufsData.Data_Raw, ufsData.Data_File, ufsData.Data_Directory, ufsData.Data_HAMTShard: + return nil, pbn, unixfsFieldData, fieldNum, nil + default: + return nil, nil, nil, 0, errNotUnixFS + } + } +} + +func loadTerminalUnixFSElementWithRecursiveDirectories(ctx context.Context, c cid.Cid, blk blocks.Block, lsys *ipld.LinkSystem, params gateway.CarParams, getLsys lsysGetter) (files.Node, error) { + var err error + if lsys == nil { + lsys, err = getLsys(ctx, c, params) + if err != nil { + return nil, err + } + } + + lctx := ipld.LinkContext{Ctx: ctx} + blockData, pbn, ufsFieldData, fieldNum, err := loadUnixFSBase(ctx, c, blk, lsys) + if err != nil { + return nil, err + } + + if c.Type() == uint64(multicodec.Raw) { + return files.NewBytesFile(blockData), nil + } + + switch fieldNum { + case ufsData.Data_Symlink: + if !ufsFieldData.FieldData().Exists() { + return nil, fmt.Errorf("invalid UnixFS symlink object") + } + lnkTarget := string(ufsFieldData.FieldData().Must().Bytes()) + f := files.NewLinkFile(lnkTarget, nil) + return f, nil + case ufsData.Data_Metadata: + return nil, fmt.Errorf("UnixFS Metadata unsupported") + case ufsData.Data_HAMTShard, ufsData.Data_Directory: + switch fieldNum { + case ufsData.Data_Directory: + d := &singleUseDirectory{&backpressuredFlatDirIter{ + ctx: ctx, + linksItr: pbn.Links.Iterator(), + lsys: lsys, + getLsys: getLsys, + }, make(chan error)} + return d, nil + case ufsData.Data_HAMTShard: + dirNd, err := unixfsnode.Reify(lctx, pbn, lsys) + if err != nil { + return nil, fmt.Errorf("could not reify sharded directory: %w", err) + } + + d := &singleUseDirectory{ + &backpressuredHAMTDirIter{ + linksItr: dirNd.MapIterator(), + dirCid: c, + lsys: lsys, + getLsys: getLsys, + ctx: ctx, + }, make(chan error), + } + return d, nil + default: + return nil, fmt.Errorf("not a basic or HAMT directory: should be unreachable") + } + case ufsData.Data_Raw, ufsData.Data_File: + nd, err := unixfsnode.Reify(lctx, pbn, lsys) + if err != nil { + return nil, err + } + + fnd, ok := nd.(datamodel.LargeBytesNode) + if !ok { + return nil, fmt.Errorf("could not process file since it did not present as large bytes") + } + f, err := fnd.AsLargeBytes() + if err != nil { + return nil, err + } + + fileSize, err := f.Seek(0, io.SeekEnd) + if err != nil { + return nil, fmt.Errorf("unable to get UnixFS file size: %w", err) + } + + from := int64(0) + var byteRange gateway.DagByteRange + if params.Range != nil { + byteRange = *params.Range + from = params.Range.From + } + _, err = f.Seek(from, io.SeekStart) + if err != nil { + return nil, fmt.Errorf("unable to get reset UnixFS file reader: %w", err) + } + + return &backpressuredFile{ctx: ctx, fileCid: c, byteRange: byteRange, size: fileSize, f: f, getLsys: getLsys, closed: make(chan error)}, nil + default: + return nil, fmt.Errorf("unknown UnixFS field type") + } +} diff --git a/lib/fixtures/directory-with-multilayer-hamt-and-multiblock-files.car b/lib/fixtures/directory-with-multilayer-hamt-and-multiblock-files.car new file mode 100644 index 0000000000000000000000000000000000000000..cb2a4875dc9169e97cd906a472d069b5bfd0f50f GIT binary patch literal 6045 zcma)A3vg7`8BSJ_?G{F6fo3Vt9*81pvV{Z?gfWm;MXHfU6hWEZ+p zMQuAW6lk@jJTyRSrxg$o1}a4hVh3gjV}(M6Mg~SSGx(q?h}hcyKaab2wKFTjY?8g_ z{?G6KzW?4^*HhPVeDBc{H_g7jq%@l(i$;FBXldPveb06cJ9lw$EcwgD3!8^+pA}y< z_Hx@ccaw_)?kC#$B{T$z4H_1&AExNGCqzT!u}pw`rD-_nf*R5sR0uo=i?CJrZ zfB(zJ=Id1tEl-yGS^MCx?tHOk(hnMp+5dg(FV?&V-dVk2=ZSNN z=H7N}b@OdA`X@_L9kHZ5{6%@!)u%_t7Pod!*=An*#Y@fQw`}>K>D^1u41M>9b$@%i4cteb_KT9KK1!ylvgveY&1*dU#K~k6W|$qeV(Jbm$ z5Z$pos{7OrT%s8)%RHUg8A46wr^#J`trOd^3A3`)H5m1Y4PT|n-LAuIqUmHopjjk< zuUShQ+(cO`wGE<~Y(YT5z&+ISSxaDoW6k!dj3MhXi+Kbrv}%D4*FEC6IwNMF`|vw> z2~6K*I`v4_@dD~nq6aO^WE%W*H9C>Zqy{lrCNMRTP(>s{Ps=dh2m6|<2Vks%h$y<<}yWAfoTluipW0j+?r8_Pf4-0dr7Pw%b4XWi*7rNm&hGs&-CNu+UkO7aa)U~MX2bL;$fxle7 zwz^|kj^RMPSx&Ryb_`ir3Pi}tH!)boW}e3^ehT_;Fz!WEsTjyDGC%Nqa5dE?Z2*8a zOjh$eO0uTrQ|JeS1S}7lFLTUM6H5W!02Ap%LdDq105pMDL8uL}Xg)DDfFAT%)U87H z&xI=)SIaR&!xMOeLT~uJrDbeQU?{u>)k$-lAbw5ih?^19*bkCpz{0L35vfJmZfIGI zOP(azz|1jQb0ywHm5NPwK12@t5YC7pG%e;Q8@SLbgc;-{z~UjWofgMM03*gK!X$H0 zXE{M>G6g3L(y*AkKMa!wq=mHru3+Yb0cdef1E9i0=we)W_f%1NKf%920Ff9^9l61?X;1$4%`0;+zTeO_>Fn+~({`Qw6Vv|q2`M1!2f9kS z58U7HsmAP2=(vYhCg<#*vT4$)^q%LB58BfFi^?iEyfD!Z$okSXd8S?{a8oRWscryd zmxGCb-qLoY9Sddxh6h2c0WQL;l8&;##0K<0(G%kq_)%zhYmi|uKQMww=MC)?Xx7(! zSU!{!p$CHbfy#jTIY(*Sqm5V)5g@T;VFuxD&MW z5n@Bzb(-}lgyg7OMD?Qr&HG5Gs8TV|VwAG)Jm84XhpA8?Zm3N|Aj(+KGzA`2R;(=q z5A}xBfbWs(6$&T|JWwDYT*L$q8R!X+f;7dlBS^&Xg$_|mq zhVvNaR*+3SU$pox!}Xsquq?~eH_^$NujqDODYp39>b-X)-@g2GuOX}K?(@$KukDx0 z%sRDh{HTL}xY9QD_C?9q*xJ;G{gd)|&#R}tmmJzO=#~Z)FvJ zJ~>NlzMqeodFj3M2Lm_EC^>v+&7+@Q=svvdjk*VW%-PrU{?Ev@6OuR9*2>%JEAdgK zQpf%pv{D(X^e8ndtH+H=ElDJXK}M-)CXu8j^Q@jvAS#NvY`WOH72*p=Ot2>47B*c)Q0OgA|xNbHSVd-*L-ouF#nD!ox#)2y^3n5~&}$ zm1U5Whvb7MpeHIMmPO`*%}ws*obS9m=QypcAZVx*YR1t(VFtW9xLy<%A#+ zaYakUS&9dpCQUMRaSHWIM#Q&9uCx-CDm(=piEZd!ST?zA0*mvQ!w{qx3X`hMf|eE7 zL8SX8xFp~%6X^}umtc@J4>1uD*1Wo)+z2lIN}Nh7p@CpMZv=V(52c}j@B%Kr@N_37 zSFkzUh%$*=4N^4x8EyxKjpbm${R>X3rvp9}`~wt9$YMb&r@~V*Y)hyGR0*n-H&sXw zKDCXAlFG_NmIDb>4!baG_#;*InJU&J6 z_}Hq{zTQcxDMerTBpMr~QGCIaxDnW&bd*evkp9yCfpl1nYQOkf>EO@i| zroov2sV*b}i4eZH2)xRB6;9>2=a)xL{Tv?nvV^bwA!;Qm@tus=%=uxVLHt)atZLfw zRzqmP^davf1Mr~u2j?ZxZSfi6onKKm^MS}ui(n8HT$(iF#N=aB&=RyMAjJ>m@rqFZ z@_6W>OP;$}jnia03Ly-xs9oXoaR=~Ng4qHk0fOKCgw6wgSnI7X5346RykrDKOtG2`wmngkRBJgk-?8w5RlvM~_4S zilSmHP^Adf<;fNqRLOzVqbTX(dmea>#fK$)BH@n-;Wy#qqnsOFGL&sjqVNt8W2uu5 QS)$l^wpMjk0ER~W54hCPHUIzs literal 0 HcmV?d00001 diff --git a/lib/gateway_traversal.go b/lib/gateway_traversal.go new file mode 100644 index 0000000..c2a9a1a --- /dev/null +++ b/lib/gateway_traversal.go @@ -0,0 +1,313 @@ +package lib + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "sync" + "time" + + "github.com/filecoin-saturn/caboose" + bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice" + "github.com/ipfs/boxo/gateway" + "github.com/ipfs/boxo/verifcid" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-unixfsnode" + "github.com/ipfs/go-unixfsnode/data" + "github.com/ipld/go-car" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/linking" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/node/basicnode" + "github.com/ipld/go-ipld-prime/schema" + "github.com/ipld/go-ipld-prime/traversal" + "github.com/ipld/go-ipld-prime/traversal/selector" + selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" + "github.com/multiformats/go-multihash" +) + +type getBlock func(ctx context.Context, cid cid.Cid) (blocks.Block, error) + +var ErrNilBlock = caboose.ErrInvalidResponse{Message: "received a nil block with no error"} + +func carToLinearBlockGetter(ctx context.Context, reader io.Reader, metrics *GraphGatewayMetrics) (getBlock, error) { + cr, err := car.NewCarReaderWithOptions(reader, car.WithErrorOnEmptyRoots(false)) + if err != nil { + return nil, err + } + + cbCtx, cncl := context.WithCancel(ctx) + + type blockRead struct { + block blocks.Block + err error + } + + blkCh := make(chan blockRead, 1) + go func() { + defer cncl() + defer close(blkCh) + for { + blk, rdErr := cr.Next() + select { + case blkCh <- blockRead{blk, rdErr}: + if rdErr != nil { + cncl() + } + case <-cbCtx.Done(): + return + } + } + }() + + isFirstBlock := true + mx := sync.Mutex{} + + return func(ctx context.Context, c cid.Cid) (blocks.Block, error) { + mx.Lock() + defer mx.Unlock() + if err := verifcid.ValidateCid(c); err != nil { + return nil, err + } + + isId, bdata := extractIdentityMultihashCIDContents(c) + if isId { + return blocks.NewBlockWithCid(bdata, c) + } + + // initially set a higher timeout here so that if there's an initial timeout error we get it from the car reader. + var t *time.Timer + if isFirstBlock { + t = time.NewTimer(GetBlockTimeout * 2) + } else { + t = time.NewTimer(GetBlockTimeout) + } + var blkRead blockRead + var ok bool + select { + case blkRead, ok = <-blkCh: + if !t.Stop() { + <-t.C + } + t.Reset(GetBlockTimeout) + case <-t.C: + return nil, gateway.ErrGatewayTimeout + } + if !ok || blkRead.err != nil { + if !ok || errors.Is(blkRead.err, io.EOF) { + return nil, io.ErrUnexpectedEOF + } + return nil, GatewayError(blkRead.err) + } + if blkRead.block != nil { + metrics.carBlocksFetchedMetric.Inc() + if !blkRead.block.Cid().Equals(c) { + return nil, caboose.ErrInvalidResponse{Message: fmt.Sprintf("received block with cid %s, expected %s", blkRead.block.Cid(), c)} + } + return blkRead.block, nil + } + return nil, ErrNilBlock + }, nil +} + +// extractIdentityMultihashCIDContents will check if a given CID has an identity multihash and if so return true and +// the bytes encoded in the digest, otherwise will return false. +// Taken from https://github.com/ipfs/boxo/blob/b96767cc0971ca279feb36e7844e527a774309ab/blockstore/idstore.go#L30 +func extractIdentityMultihashCIDContents(k cid.Cid) (bool, []byte) { + // Pre-check by calling Prefix(), this much faster than extracting the hash. + if k.Prefix().MhType != multihash.IDENTITY { + return false, nil + } + + dmh, err := multihash.Decode(k.Hash()) + if err != nil || dmh.Code != multihash.IDENTITY { + return false, nil + } + return true, dmh.Digest +} + +func getLinksystem(fn getBlock) *ipld.LinkSystem { + lsys := cidlink.DefaultLinkSystem() + lsys.StorageReadOpener = func(linkContext linking.LinkContext, link datamodel.Link) (io.Reader, error) { + c := link.(cidlink.Link).Cid + blk, err := fn(linkContext.Ctx, c) + if err != nil { + return nil, err + } + return bytes.NewReader(blk.RawData()), nil + } + lsys.TrustedStorage = true + unixfsnode.AddUnixFSReificationToLinkSystem(&lsys) + return &lsys +} + +// walkGatewaySimpleSelector walks the subgraph described by the path and terminal element parameters +func walkGatewaySimpleSelector(ctx context.Context, terminalBlk blocks.Block, dagScope gateway.DagScope, entityRange *gateway.DagByteRange, lsys *ipld.LinkSystem) error { + lctx := ipld.LinkContext{Ctx: ctx} + var err error + + // If the scope is the block, we only need the root block of the last element of the path, which we have. + if dagScope == gateway.DagScopeBlock { + return nil + } + + // decode the terminal block into a node + pc := dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { + if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { + return tlnkNd.LinkTargetNodePrototype(), nil + } + return basicnode.Prototype.Any, nil + }) + + pathTerminalCidLink := cidlink.Link{Cid: terminalBlk.Cid()} + np, err := pc(pathTerminalCidLink, lctx) + if err != nil { + return err + } + + decoder, err := lsys.DecoderChooser(pathTerminalCidLink) + if err != nil { + return err + } + nb := np.NewBuilder() + blockData := terminalBlk.RawData() + if err := decoder(nb, bytes.NewReader(blockData)); err != nil { + return err + } + lastCidNode := nb.Build() + + // TODO: Evaluate: + // Does it matter that we're ignoring the "remainder" portion of the traversal in GetCAR? + // Does it matter that we're using a linksystem with the UnixFS reifier for dagscope=all? + + // If we're asking for everything then give it + if dagScope == gateway.DagScopeAll { + sel, err := selector.ParseSelector(selectorparse.CommonSelector_ExploreAllRecursively) + if err != nil { + return err + } + + progress := traversal.Progress{ + Cfg: &traversal.Config{ + Ctx: ctx, + LinkSystem: *lsys, + LinkTargetNodePrototypeChooser: bsfetcher.DefaultPrototypeChooser, + LinkVisitOnlyOnce: false, // Despite being safe for the "all" selector we do this walk anyway since this is how we will be receiving the blocks + }, + } + + if err := progress.WalkMatching(lastCidNode, sel, func(progress traversal.Progress, node datamodel.Node) error { + return nil + }); err != nil { + return err + } + return nil + } + + // From now on, dag-scope=entity! + // Since we need more of the graph load it to figure out what we have + // This includes determining if the terminal node is UnixFS or not + if pbn, ok := lastCidNode.(dagpb.PBNode); !ok { + // If it's not valid dag-pb then we're done + return nil + } else if !pbn.FieldData().Exists() { + // If it's not valid UnixFS then we're done + return nil + } else if unixfsFieldData, decodeErr := data.DecodeUnixFSData(pbn.Data.Must().Bytes()); decodeErr != nil { + // If it's not valid dag-pb and UnixFS then we're done + return nil + } else { + switch unixfsFieldData.FieldDataType().Int() { + case data.Data_Directory, data.Data_Symlink: + // These types are non-recursive so we're done + return nil + case data.Data_Raw, data.Data_Metadata: + // TODO: for now, we decided to return nil here. The different implementations are inconsistent + // and UnixFS is not properly specified: https://github.com/ipfs/specs/issues/316. + // - Is Data_Raw different from Data_File? + // - Data_Metadata is handled differently in boxo/ipld/unixfs and go-unixfsnode. + return nil + case data.Data_HAMTShard: + // Return all elements in the map + _, err := lsys.KnownReifiers["unixfs-preload"](lctx, lastCidNode, lsys) + if err != nil { + return err + } + return nil + case data.Data_File: + nd, err := unixfsnode.Reify(lctx, lastCidNode, lsys) + if err != nil { + return err + } + + fnd, ok := nd.(datamodel.LargeBytesNode) + if !ok { + return fmt.Errorf("could not process file since it did not present as large bytes") + } + f, err := fnd.AsLargeBytes() + if err != nil { + return err + } + + // Get the entity range. If it's empty, assume the defaults (whole file). + effectiveRange := entityRange + if effectiveRange == nil { + effectiveRange = &gateway.DagByteRange{ + From: 0, + } + } + + from := effectiveRange.From + + // If we're starting to read based on the end of the file, find out where that is. + var fileLength int64 + foundFileLength := false + if effectiveRange.From < 0 { + fileLength, err = f.Seek(0, io.SeekEnd) + if err != nil { + return err + } + from = fileLength + effectiveRange.From + foundFileLength = true + } + + // If we're reading until the end of the file then do it + if effectiveRange.To == nil { + if _, err := f.Seek(from, io.SeekStart); err != nil { + return err + } + _, err = io.Copy(io.Discard, f) + return err + } + + to := *effectiveRange.To + if (*effectiveRange.To) < 0 && !foundFileLength { + fileLength, err = f.Seek(0, io.SeekEnd) + if err != nil { + return err + } + to = fileLength + *effectiveRange.To + foundFileLength = true + } + + numToRead := 1 + to - from + if numToRead < 0 { + return fmt.Errorf("tried to read less than zero bytes") + } + + if _, err := f.Seek(from, io.SeekStart); err != nil { + return err + } + _, err = io.CopyN(io.Discard, f, numToRead) + return err + default: + // Not a supported type, so we're done + return nil + } + } +} diff --git a/lib/gateway_utils.go b/lib/gateway_utils.go new file mode 100644 index 0000000..e7093fb --- /dev/null +++ b/lib/gateway_utils.go @@ -0,0 +1,73 @@ +package lib + +import ( + "context" + "errors" + "fmt" + "net/url" + "strconv" + "strings" + "time" + + "github.com/filecoin-saturn/caboose" + "github.com/ipfs/boxo/gateway" +) + +// contentPathToCarUrl returns an URL that allows retrieval of specified resource +// from a trustless gateway that implements IPIP-402 +func contentPathToCarUrl(path gateway.ImmutablePath, params gateway.CarParams) *url.URL { + return &url.URL{ + Path: path.String(), + RawQuery: carParamsToString(params), + } +} + +// carParamsToString converts CarParams to URL parameters compatible with IPIP-402 +func carParamsToString(params gateway.CarParams) string { + paramsBuilder := strings.Builder{} + paramsBuilder.WriteString("format=car") // always send explicit format in URL, this makes debugging easier, even when Accept header was set + if params.Scope != "" { + paramsBuilder.WriteString("&dag-scope=") + paramsBuilder.WriteString(string(params.Scope)) + } + if params.Range != nil { + paramsBuilder.WriteString("&entity-bytes=") + paramsBuilder.WriteString(strconv.FormatInt(params.Range.From, 10)) + paramsBuilder.WriteString(":") + if params.Range.To != nil { + paramsBuilder.WriteString(strconv.FormatInt(*params.Range.To, 10)) + } else { + paramsBuilder.WriteString("*") + } + } + return paramsBuilder.String() +} + +// GatewayError translates underlying blockstore error into one that gateway code will return as HTTP 502 or 504 +// it also makes sure Retry-After hint from remote blockstore will be passed to HTTP client, if present. +func GatewayError(err error) error { + if errors.Is(err, &gateway.ErrorStatusCode{}) || + errors.Is(err, &gateway.ErrorRetryAfter{}) { + // already correct error + return err + } + + // All timeouts should produce 504 Gateway Timeout + if errors.Is(err, context.DeadlineExceeded) || + errors.Is(err, caboose.ErrSaturnTimeout) || + // Unfortunately this is not an exported type so we have to check for the content. + strings.Contains(err.Error(), "Client.Timeout exceeded") { + return fmt.Errorf("%w: %s", gateway.ErrGatewayTimeout, err.Error()) + } + + // (Saturn) errors that support the RetryAfter interface need to be converted + // to the correct gateway error, such that the HTTP header is set. + for v := err; v != nil; v = errors.Unwrap(v) { + if r, ok := v.(interface{ RetryAfter() time.Duration }); ok { + return gateway.NewErrorRetryAfter(err, r.RetryAfter()) + } + } + + // everything else returns 502 Bad Gateway + return fmt.Errorf("%w: %s", gateway.ErrBadGateway, err.Error()) +} diff --git a/lib/gateway_utils_test.go b/lib/gateway_utils_test.go new file mode 100644 index 0000000..f27d96e --- /dev/null +++ b/lib/gateway_utils_test.go @@ -0,0 +1,94 @@ +package lib + +import ( + "errors" + "fmt" + "github.com/stretchr/testify/assert" + "testing" + "time" + + ifacepath "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/gateway" +) + +func TestContentPathToCarUrl(t *testing.T) { + negativeOffset := int64(-42) + testCases := []struct { + contentPath string // to be turned into gateway.ImmutablePath + carParams gateway.CarParams + expectedUrl string // url.URL.String() + }{ + { + contentPath: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", + carParams: gateway.CarParams{}, + expectedUrl: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi?format=car", + }, + { + contentPath: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", + carParams: gateway.CarParams{Scope: "entity", Range: &gateway.DagByteRange{From: 0, To: nil}}, + expectedUrl: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi?format=car&dag-scope=entity&entity-bytes=0:*", + }, + { + contentPath: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", + carParams: gateway.CarParams{Scope: "block"}, + expectedUrl: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi?format=car&dag-scope=block", + }, + { + contentPath: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", + carParams: gateway.CarParams{Scope: "entity", Range: &gateway.DagByteRange{From: 4, To: &negativeOffset}}, + expectedUrl: "/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi?format=car&dag-scope=entity&entity-bytes=4:-42", + }, + { + // a regression test for case described in https://github.com/ipfs/gateway-conformance/issues/115 + contentPath: "/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze/I/Auditorio_de_Tenerife%2C_Santa_Cruz_de_Tenerife%2C_España%2C_2012-12-15%2C_DD_02.jpg.webp", + carParams: gateway.CarParams{Scope: "entity", Range: &gateway.DagByteRange{From: 0, To: nil}}, + expectedUrl: "/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze/I/Auditorio_de_Tenerife%252C_Santa_Cruz_de_Tenerife%252C_Espa%C3%B1a%252C_2012-12-15%252C_DD_02.jpg.webp?format=car&dag-scope=entity&entity-bytes=0:*", + }, + } + + for _, tc := range testCases { + t.Run("TestContentPathToCarUrl", func(t *testing.T) { + contentPath, err := gateway.NewImmutablePath(ifacepath.New(tc.contentPath)) + if err != nil { + t.Fatal(err) + } + result := contentPathToCarUrl(contentPath, tc.carParams).String() + if result != tc.expectedUrl { + t.Errorf("Expected %q, but got %q", tc.expectedUrl, result) + } + }) + } +} + +type testErr struct { + message string + retryAfter time.Duration +} + +func (e *testErr) Error() string { + return e.message +} + +func (e *testErr) RetryAfter() time.Duration { + return e.retryAfter +} + +func TestGatewayErrorRetryAfter(t *testing.T) { + originalErr := &testErr{message: "test", retryAfter: time.Minute} + var ( + convertedErr error + gatewayErr *gateway.ErrorRetryAfter + ) + + // Test unwrapped + convertedErr = GatewayError(originalErr) + ok := errors.As(convertedErr, &gatewayErr) + assert.True(t, ok) + assert.EqualValues(t, originalErr.retryAfter, gatewayErr.RetryAfter) + + // Test wrapped. + convertedErr = GatewayError(fmt.Errorf("wrapped error: %w", originalErr)) + ok = errors.As(convertedErr, &gatewayErr) + assert.True(t, ok) + assert.EqualValues(t, originalErr.retryAfter, gatewayErr.RetryAfter) +} diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index 21fc287..2d47eae 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -1,59 +1,71 @@ package lib import ( + "bytes" "context" "errors" "fmt" "io" "net/http" - "runtime" - "runtime/debug" + gopath "path" "strconv" "strings" - "sync" "time" "github.com/filecoin-saturn/caboose" - "github.com/ipfs/boxo/blockservice" - blockstore "github.com/ipfs/boxo/blockstore" + "github.com/hashicorp/go-multierror" nsopts "github.com/ipfs/boxo/coreiface/options/namesys" ifacepath "github.com/ipfs/boxo/coreiface/path" - exchange "github.com/ipfs/boxo/exchange" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/gateway" + "github.com/ipfs/boxo/ipld/merkledag" + "github.com/ipfs/boxo/ipld/unixfs" "github.com/ipfs/boxo/namesys" "github.com/ipfs/boxo/namesys/resolve" ipfspath "github.com/ipfs/boxo/path" + "github.com/ipfs/boxo/path/resolver" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" format "github.com/ipfs/go-ipld-format" - golog "github.com/ipfs/go-log/v2" - "github.com/ipld/go-car" + "github.com/ipfs/go-unixfsnode" + ufsData "github.com/ipfs/go-unixfsnode/data" + "github.com/ipfs/go-unixfsnode/hamt" + ufsiter "github.com/ipfs/go-unixfsnode/iter" + carv2 "github.com/ipld/go-car/v2" + "github.com/ipld/go-car/v2/storage" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/node/basicnode" + "github.com/ipld/go-ipld-prime/schema" + "github.com/ipld/go-ipld-prime/traversal" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" "github.com/multiformats/go-multicodec" - "github.com/multiformats/go-multihash" "github.com/prometheus/client_golang/prometheus" - "go.uber.org/multierr" ) -var graphLog = golog.Logger("backend/graph") - const GetBlockTimeout = time.Second * 60 // type DataCallback = func(resource string, reader io.Reader) error // TODO: Don't use a caboose type, perhaps ask them to use a type alias instead of a type type DataCallback = caboose.DataCallback +// TODO: Don't use a caboose type +type ErrPartialResponse = caboose.ErrPartialResponse + +var ErrFetcherUnexpectedEOF = fmt.Errorf("failed to fetch IPLD data") + type CarFetcher interface { Fetch(ctx context.Context, path string, cb DataCallback) error } type gwOptions struct { - ns namesys.NameSystem - vs routing.ValueStore - bs blockstore.Blockstore + ns namesys.NameSystem + vs routing.ValueStore + promRegistry prometheus.Registerer } // WithNameSystem sets the name system to use for the gateway. If not set it will use a default DNSLink resolver @@ -73,51 +85,37 @@ func WithValueStore(vs routing.ValueStore) GraphGatewayOption { } } -// WithBlockstore sets the Blockstore to use for the gateway -func WithBlockstore(bs blockstore.Blockstore) GraphGatewayOption { +// WithPrometheusRegistry sets the registry to use for metrics collection +func WithPrometheusRegistry(reg prometheus.Registerer) GraphGatewayOption { return func(opts *gwOptions) error { - opts.bs = bs + opts.promRegistry = reg return nil } } type GraphGatewayOption func(gwOptions *gwOptions) error -type Notifier interface { - NotifyNewBlocks(ctx context.Context, blocks ...blocks.Block) error -} - -// notifiersForRootCid is used for reducing lock contention by only notifying -// exchanges related to the same content root CID -type notifiersForRootCid struct { - lk sync.RWMutex - deleted int8 - notifiers []Notifier -} - type GraphGateway struct { - fetcher CarFetcher - blockFetcher exchange.Fetcher - routing routing.ValueStore - namesys namesys.NameSystem - bstore blockstore.Blockstore + fetcher CarFetcher + routing routing.ValueStore + namesys namesys.NameSystem + + pc traversal.LinkTargetNodePrototypeChooser - notifiers sync.Map // cid -> notifiersForRootCid - metrics *GraphGatewayMetrics + metrics *GraphGatewayMetrics } type GraphGatewayMetrics struct { contextAlreadyCancelledMetric prometheus.Counter carFetchAttemptMetric prometheus.Counter carBlocksFetchedMetric prometheus.Counter - blockRecoveryAttemptMetric prometheus.Counter carParamsMetric *prometheus.CounterVec bytesRangeStartMetric prometheus.Histogram bytesRangeSizeMetric prometheus.Histogram } -func NewGraphGatewayBackend(f CarFetcher, blockFetcher exchange.Fetcher, opts ...GraphGatewayOption) (*GraphGateway, error) { +func NewGraphGatewayBackend(f CarFetcher, opts ...GraphGatewayOption) (*GraphGateway, error) { var compiledOptions gwOptions for _, o := range opts { if err := o(&compiledOptions); err != nil { @@ -144,32 +142,26 @@ func NewGraphGatewayBackend(f CarFetcher, blockFetcher exchange.Fetcher, opts .. } } - bs := compiledOptions.bs - if compiledOptions.bs == nil { - // Sets up a cache to store blocks in - cbs, err := NewCacheBlockStore(DefaultCacheBlockStoreSize) - if err != nil { - return nil, err - } - - // Set up support for identity hashes (https://github.com/ipfs/bifrost-gateway/issues/38) - cbs = blockstore.NewIdStore(cbs) - bs = cbs + var promReg prometheus.Registerer = prometheus.NewRegistry() + if compiledOptions.promRegistry != nil { + promReg = compiledOptions.promRegistry } return &GraphGateway{ - fetcher: f, - blockFetcher: blockFetcher, - routing: vs, - namesys: ns, - bstore: bs, - notifiers: sync.Map{}, - metrics: registerGraphGatewayMetrics(), + fetcher: f, + routing: vs, + namesys: ns, + metrics: registerGraphGatewayMetrics(promReg), + pc: dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { + if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { + return tlnkNd.LinkTargetNodePrototype(), nil + } + return basicnode.Prototype.Any, nil + }), }, nil } -func registerGraphGatewayMetrics() *GraphGatewayMetrics { - +func registerGraphGatewayMetrics(registerer prometheus.Registerer) *GraphGatewayMetrics { // How many CAR Fetch attempts we had? Need this to calculate % of various graph request types. // We only count attempts here, because success/failure with/without retries are provided by caboose: // - ipfs_caboose_fetch_duration_car_success_count @@ -182,7 +174,7 @@ func registerGraphGatewayMetrics() *GraphGatewayMetrics { Name: "car_fetch_attempts", Help: "The number of times a CAR fetch was attempted by IPFSBackend.", }) - prometheus.MustRegister(carFetchAttemptMetric) + registerer.MustRegister(carFetchAttemptMetric) contextAlreadyCancelledMetric := prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "ipfs", @@ -190,7 +182,7 @@ func registerGraphGatewayMetrics() *GraphGatewayMetrics { Name: "car_fetch_context_already_cancelled", Help: "The number of times context is already cancelled when a CAR fetch was attempted by IPFSBackend.", }) - prometheus.MustRegister(contextAlreadyCancelledMetric) + registerer.MustRegister(contextAlreadyCancelledMetric) // How many blocks were read via CARs? // Need this as a baseline to reason about error ratio vs raw_block_recovery_attempts. @@ -200,21 +192,7 @@ func registerGraphGatewayMetrics() *GraphGatewayMetrics { Name: "car_blocks_fetched", Help: "The number of blocks successfully read via CAR fetch.", }) - prometheus.MustRegister(carBlocksFetchedMetric) - - // How many times CAR response was not enough or just failed, and we had to read a block via ?format=raw - // We only count attempts here, because success/failure with/without retries are provided by caboose: - // - ipfs_caboose_fetch_duration_block_success_count - // - ipfs_caboose_fetch_duration_block_failure_count - // - ipfs_caboose_fetch_duration_block_peer_success_count - // - ipfs_caboose_fetch_duration_block_peer_failure_count - blockRecoveryAttemptMetric := prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: "ipfs", - Subsystem: "gw_graph_backend", - Name: "raw_block_recovery_attempts", - Help: "The number of ?format=raw block fetch attempts due to GraphGateway failure (CAR fetch error, missing block in CAR response, or a block evicted from cache too soon).", - }) - prometheus.MustRegister(blockRecoveryAttemptMetric) + registerer.MustRegister(carBlocksFetchedMetric) carParamsMetric := prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: "ipfs", @@ -222,7 +200,7 @@ func registerGraphGatewayMetrics() *GraphGatewayMetrics { Name: "car_fetch_params", Help: "How many times specific CAR parameter was used during CAR data fetch.", }, []string{"dagScope", "entityRanges"}) // we use 'ranges' instead of 'bytes' here because we only count the number of ranges present - prometheus.MustRegister(carParamsMetric) + registerer.MustRegister(carParamsMetric) bytesRangeStartMetric := prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: "ipfs", @@ -231,7 +209,7 @@ func registerGraphGatewayMetrics() *GraphGatewayMetrics { Help: "Tracks where did the range request start.", Buckets: prometheus.ExponentialBuckets(1024, 2, 24), // 1024 bytes to 8 GiB }) - prometheus.MustRegister(bytesRangeStartMetric) + registerer.MustRegister(bytesRangeStartMetric) bytesRangeSizeMetric := prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: "ipfs", @@ -240,606 +218,1151 @@ func registerGraphGatewayMetrics() *GraphGatewayMetrics { Help: "Tracks the size of range requests.", Buckets: prometheus.ExponentialBuckets(256*1024, 2, 10), // From 256KiB to 100MiB }) - prometheus.MustRegister(bytesRangeSizeMetric) + registerer.MustRegister(bytesRangeSizeMetric) return &GraphGatewayMetrics{ contextAlreadyCancelledMetric, carFetchAttemptMetric, carBlocksFetchedMetric, - blockRecoveryAttemptMetric, carParamsMetric, bytesRangeStartMetric, bytesRangeSizeMetric, } } -func (api *GraphGateway) getRootOfPath(path string) string { - pth, err := ipfspath.ParsePath(path) - if err != nil { - return path - } - if pth.IsJustAKey() { - return pth.Segments()[0] - } else { - return pth.Segments()[1] - } -} +func (api *GraphGateway) fetchCAR(ctx context.Context, path gateway.ImmutablePath, params gateway.CarParams, cb DataCallback) error { + urlWithoutHost := contentPathToCarUrl(path, params).String() -/* -Implementation iteration plan: + api.metrics.carFetchAttemptMetric.Inc() + var ipldError error + fetchErr := api.fetcher.Fetch(ctx, urlWithoutHost, func(resource string, reader io.Reader) error { + return checkRetryableError(&ipldError, func() error { + return cb(resource, reader) + }) + }) + + if ipldError != nil { + fetchErr = ipldError + } else if fetchErr != nil { + fetchErr = GatewayError(fetchErr) + } -1. Fetch CAR into per-request memory blockstore and serve response -2. Fetch CAR into shared memory blockstore and serve response along with a blockservice that does block requests for missing data -3. Start doing the walk locally and then if a path segment is incomplete send a request for blocks and upon every received block try to continue -4. Start doing the walk locally and keep a list of "plausible" blocks, if after issuing a request we get a non-plausible block then report them and attempt to recover by redoing the last segment -5. Don't redo the last segment fully if it's part of a UnixFS file and we can do range requests -*/ + return fetchErr +} -func (api *GraphGateway) loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx context.Context, path string) (gateway.IPFSBackend, func(), error) { - bstore := api.bstore - carFetchingExch := newInboundBlockExchange() - doneWithFetcher := make(chan struct{}, 1) - exch := &handoffExchange{ - startingExchange: carFetchingExch, - followupExchange: &blockFetcherExchWrapper{api.blockFetcher}, - bstore: bstore, - handoffCh: doneWithFetcher, - metrics: api.metrics, +// resolvePathWithRootsAndBlock takes a path and linksystem and returns the set of non-terminal cids, the terminal cid, the remainder, and the block corresponding to the terminal cid +func resolvePathWithRootsAndBlock(ctx context.Context, fpath ipfspath.Path, unixFSLsys *ipld.LinkSystem) ([]cid.Cid, cid.Cid, []string, blocks.Block, error) { + pathRootCids, terminalCid, remainder, terminalBlk, err := resolvePathToLastWithRoots(ctx, fpath, unixFSLsys) + if err != nil { + return nil, cid.Undef, nil, nil, err } - notifierKey := api.getRootOfPath(path) - var notifier *notifiersForRootCid - for { - notifiers, _ := api.notifiers.LoadOrStore(notifierKey, ¬ifiersForRootCid{notifiers: []Notifier{}}) - if n, ok := notifiers.(*notifiersForRootCid); ok { - n.lk.Lock() - // could have been deleted after our load. try again. - if n.deleted != 0 { - n.lk.Unlock() - continue - } - notifier = n - n.notifiers = append(n.notifiers, exch) - n.lk.Unlock() - break - } else { - return nil, nil, errors.New("failed to get notifier") + if terminalBlk == nil { + lctx := ipld.LinkContext{Ctx: ctx} + lnk := cidlink.Link{Cid: terminalCid} + blockData, err := unixFSLsys.LoadRaw(lctx, lnk) + if err != nil { + return nil, cid.Undef, nil, nil, err + } + terminalBlk, err = blocks.NewBlockWithCid(blockData, terminalCid) + if err != nil { + return nil, cid.Undef, nil, nil, err } } - go func(metrics *GraphGatewayMetrics) { - defer func() { - if r := recover(); r != nil { - // TODO: move to Debugw? - graphLog.Errorw("Recovered fetcher error", "path", path, "error", r, "stacktrace", string(debug.Stack())) - } - }() - metrics.carFetchAttemptMetric.Inc() + return pathRootCids, terminalCid, remainder, terminalBlk, err +} - if ce := ctx.Err(); ce != nil && errors.Is(ce, context.Canceled) { - metrics.contextAlreadyCancelledMetric.Inc() - } +// resolvePathToLastWithRoots takes a path and linksystem and returns the set of non-terminal cids, the terminal cid, +// the remainder pathing, the last block loaded, and the last node loaded. +// +// Note: the block returned will be nil if the terminal element is a link or the path is just a CID +func resolvePathToLastWithRoots(ctx context.Context, fpath ipfspath.Path, unixFSLsys *ipld.LinkSystem) ([]cid.Cid, cid.Cid, []string, blocks.Block, error) { + c, p, err := ipfspath.SplitAbsPath(fpath) + if err != nil { + return nil, cid.Undef, nil, nil, err + } - cctx, cncl := context.WithCancel(ctx) - defer cncl() - err := api.fetcher.Fetch(cctx, path, func(resource string, reader io.Reader) error { - cr, err := car.NewCarReader(reader) - if err != nil { - return err - } + if len(p) == 0 { + return nil, c, nil, nil, nil + } - cbCtx, cncl := context.WithCancel(cctx) - defer cncl() + unixFSLsys.NodeReifier = unixfsnode.Reify + defer func() { unixFSLsys.NodeReifier = nil }() - type blockRead struct { - block blocks.Block - err error - } + var cids []cid.Cid + cids = append(cids, c) - blkCh := make(chan blockRead, 1) - go func() { - defer close(blkCh) - for { - blk, rdErr := cr.Next() - select { - case blkCh <- blockRead{blk, rdErr}: - case <-cbCtx.Done(): - return - } - } - }() + pc := dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { + if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { + return tlnkNd.LinkTargetNodePrototype(), nil + } + return basicnode.Prototype.Any, nil + }) - // initially set a higher timeout here so that if there's an initial timeout error we get it from the car reader. - t := time.NewTimer(GetBlockTimeout * 2) - for { - var blkRead blockRead - var ok bool - select { - case blkRead, ok = <-blkCh: - if !t.Stop() { - <-t.C - } - t.Reset(GetBlockTimeout) - case <-t.C: - return gateway.ErrGatewayTimeout - } - if !ok || blkRead.err != nil { - if errors.Is(blkRead.err, io.EOF) { - return nil - } - return blkRead.err - } - if blkRead.block != nil { - if err := bstore.Put(ctx, blkRead.block); err != nil { - return err - } - metrics.carBlocksFetchedMetric.Inc() - api.notifyOngoingRequests(ctx, notifierKey, blkRead.block) - } - } - }) + loadNode := func(ctx context.Context, c cid.Cid) (blocks.Block, ipld.Node, error) { + lctx := ipld.LinkContext{Ctx: ctx} + rootLnk := cidlink.Link{Cid: c} + np, err := pc(rootLnk, lctx) if err != nil { - graphLog.Infow("car Fetch failed", "path", path, "error", err) + return nil, nil, err } - if err := carFetchingExch.Close(); err != nil { - graphLog.Errorw("carFetchingExch.Close()", "error", err) + nd, blockData, err := unixFSLsys.LoadPlusRaw(lctx, rootLnk, np) + if err != nil { + return nil, nil, err + } + blk, err := blocks.NewBlockWithCid(blockData, c) + if err != nil { + return nil, nil, err } - doneWithFetcher <- struct{}{} - close(doneWithFetcher) - }(api.metrics) + return blk, nd, nil + } - bserv := blockservice.New(bstore, exch) - blkgw, err := gateway.NewBlocksBackend(bserv) + nextBlk, nextNd, err := loadNode(ctx, c) if err != nil { - return nil, nil, err + return nil, cid.Undef, nil, nil, err } - return blkgw, func() { - notifier.lk.Lock() - for i, e := range notifier.notifiers { - if e == exch { - notifier.notifiers = append(notifier.notifiers[0:i], notifier.notifiers[i+1:]...) - break - } - } - if len(notifier.notifiers) == 0 { - notifier.deleted = 1 - api.notifiers.Delete(notifierKey) - } - notifier.lk.Unlock() - }, nil -} - -func (api *GraphGateway) notifyOngoingRequests(ctx context.Context, key string, blks ...blocks.Block) { - if notifiers, ok := api.notifiers.Load(key); ok { - notifier, ok := notifiers.(*notifiersForRootCid) - if !ok { - graphLog.Errorw("notifyOngoingRequests failed", "key", key, "error", "could not get notifiersForRootCid") - return + depth := 0 + for i, elem := range p { + nextNd, err = nextNd.LookupBySegment(ipld.ParsePathSegment(elem)) + if err != nil { + return nil, cid.Undef, nil, nil, err } - notifier.lk.RLock() - for _, n := range notifier.notifiers { - err := n.NotifyNewBlocks(ctx, blks...) + if nextNd.Kind() == ipld.Kind_Link { + depth = 0 + lnk, err := nextNd.AsLink() if err != nil { - graphLog.Errorw("notifyOngoingRequests failed", "key", key, "error", err) + return nil, cid.Undef, nil, nil, err + } + cidLnk, ok := lnk.(cidlink.Link) + if !ok { + return nil, cid.Undef, nil, nil, fmt.Errorf("link is not a cidlink: %v", cidLnk) + } + cids = append(cids, cidLnk.Cid) + + if i < len(p)-1 { + nextBlk, nextNd, err = loadNode(ctx, cidLnk.Cid) + if err != nil { + return nil, cid.Undef, nil, nil, err + } } + } else { + depth++ } - notifier.lk.RUnlock() } -} - -type fileCloseWrapper struct { - files.File - closeFn func() -} - -func (w *fileCloseWrapper) Close() error { - w.closeFn() - return w.File.Close() -} -type dirCloseWrapper struct { - files.Directory - closeFn func() -} + // if last node is not a link, just return it's cid, add path to remainder and return + if nextNd.Kind() != ipld.Kind_Link { + // return the cid and the remainder of the path + return cids[:len(cids)-1], cids[len(cids)-1], p[len(p)-depth:], nextBlk, nil + } -func (w *dirCloseWrapper) Close() error { - w.closeFn() - return w.Directory.Close() + return cids[:len(cids)-1], cids[len(cids)-1], nil, nil, nil } -func wrapNodeWithClose[T files.Node](node T, closeFn func()) (T, error) { - var genericNode files.Node = node - switch n := genericNode.(type) { - case *files.Symlink: - closeFn() - return node, nil - case files.File: - var f files.File = &fileCloseWrapper{n, closeFn} - return f.(T), nil - case files.Directory: - var d files.Directory = &dirCloseWrapper{n, closeFn} - return d.(T), nil - default: - closeFn() - var zeroType T - return zeroType, fmt.Errorf("unsupported node type") +func contentMetadataFromRootsAndRemainder(p ipfspath.Path, pathRoots []cid.Cid, terminalCid cid.Cid, remainder []string) gateway.ContentPathMetadata { + var rootCid cid.Cid + if len(pathRoots) > 0 { + rootCid = pathRoots[0] + } else { + rootCid = terminalCid + } + md := gateway.ContentPathMetadata{ + PathSegmentRoots: pathRoots, + LastSegment: ifacepath.NewResolvedPath(p, terminalCid, rootCid, gopath.Join(remainder...)), } + return md } +var errNotUnixFS = fmt.Errorf("data was not unixfs") + func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, byteRanges ...gateway.ByteRange) (gateway.ContentPathMetadata, *gateway.GetResponse, error) { rangeCount := len(byteRanges) api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "entity", "entityRanges": strconv.Itoa(rangeCount)}).Inc() - // IPIP-402 - carParams := "?format=car&dag-scope=entity" + carParams := gateway.CarParams{Scope: gateway.DagScopeEntity} - // If request was HTTP Range Request fetch CAR with entity-bytes=from:to to - // get minimal set of blocks + // fetch CAR with &bytes= to get minimal set of blocks for the request // Note: majority of requests have 0 or max 1 ranges. if there are more ranges than one, // that is a niche edge cache we don't prefetch as CAR and use fallback blockstore instead. if rangeCount > 0 { - bytesBuilder := strings.Builder{} - bytesBuilder.WriteString("&entity-bytes=") r := byteRanges[0] - - bytesBuilder.WriteString(strconv.FormatUint(r.From, 10)) - bytesBuilder.WriteString(":") + carParams.Range = &gateway.DagByteRange{ + From: int64(r.From), + } // TODO: move to boxo or to loadRequestIntoSharedBlockstoreAndBlocksGateway after we pass params in a humane way api.metrics.bytesRangeStartMetric.Observe(float64(r.From)) if r.To != nil { - bytesBuilder.WriteString(strconv.FormatInt(*r.To, 10)) + carParams.Range.To = r.To // TODO: move to boxo or to loadRequestIntoSharedBlockstoreAndBlocksGateway after we pass params in a humane way api.metrics.bytesRangeSizeMetric.Observe(float64(*r.To) - float64(r.From) + 1) - } else { - bytesBuilder.WriteString("*") } - carParams += bytesBuilder.String() } - blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+carParams) + md, terminalElem, err := fetchWithPartialRetries(ctx, path, carParams, loadTerminalEntity, api.metrics, api.fetchCAR) if err != nil { return gateway.ContentPathMetadata{}, nil, err } - md, gr, err := blkgw.Get(ctx, path, byteRanges...) - if err != nil { - return gateway.ContentPathMetadata{}, nil, err + + var resp *gateway.GetResponse + + switch typedTerminalElem := terminalElem.(type) { + case *gateway.GetResponse: + resp = typedTerminalElem + case *backpressuredFile: + resp = gateway.NewGetResponseFromReader(typedTerminalElem, typedTerminalElem.size) + case *backpressuredHAMTDirIterNoRecursion: + ch := make(chan unixfs.LinkResult) + go func() { + defer close(ch) + for typedTerminalElem.Next() { + l := typedTerminalElem.Link() + select { + case ch <- l: + case <-ctx.Done(): + return + } + } + if err := typedTerminalElem.Err(); err != nil { + select { + case ch <- unixfs.LinkResult{Err: err}: + case <-ctx.Done(): + return + } + } + }() + resp = gateway.NewGetResponseFromDirectoryListing(typedTerminalElem.dagSize, ch, nil) + default: + return gateway.ContentPathMetadata{}, nil, fmt.Errorf("invalid data type") } - //TODO: interfaces here aren't good enough so we're getting around the problem this way - runtime.SetFinalizer(gr, func(_ *gateway.GetResponse) { closeFn() }) - return md, gr, nil -} + return md, resp, nil -func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { - api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "all", "entityRanges": "0"}).Inc() - blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=all") - if err != nil { - return gateway.ContentPathMetadata{}, nil, err - } - md, f, err := blkgw.GetAll(ctx, path) - if err != nil { - return gateway.ContentPathMetadata{}, nil, err - } - f, err = wrapNodeWithClose(f, closeFn) - if err != nil { - return gateway.ContentPathMetadata{}, nil, err - } - return md, f, nil } -func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.File, error) { - api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "block", "entityRanges": "0"}).Inc() - // TODO: if path is `/ipfs/cid`, we should use ?format=raw - blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=block") - if err != nil { - return gateway.ContentPathMetadata{}, nil, err - } - md, f, err := blkgw.GetBlock(ctx, path) - if err != nil { - return gateway.ContentPathMetadata{}, nil, err - } - f, err = wrapNodeWithClose(f, closeFn) - if err != nil { - return gateway.ContentPathMetadata{}, nil, err +// loadTerminalEntity returns either a [*gateway.GetResponse], [*backpressuredFile], or [*backpressuredHAMTDirIterNoRecursion] +func loadTerminalEntity(ctx context.Context, c cid.Cid, blk blocks.Block, lsys *ipld.LinkSystem, params gateway.CarParams, getLsys lsysGetter) (interface{}, error) { + var err error + if lsys == nil { + lsys, err = getLsys(ctx, c, params) + if err != nil { + return nil, err + } } - return md, f, nil -} -func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { - api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "entity", "entityRanges": "1"}).Inc() + lctx := ipld.LinkContext{Ctx: ctx} - // TODO: we probably want to move this either to boxo, or at least to loadRequestIntoSharedBlockstoreAndBlocksGateway - api.metrics.bytesRangeStartMetric.Observe(0) - api.metrics.bytesRangeSizeMetric.Observe(1024) + if c.Type() != uint64(multicodec.DagPb) { + var blockData []byte - blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=entity&entity-bytes=0:1023") + if blk != nil { + blockData = blk.RawData() + } else { + blockData, err = lsys.LoadRaw(lctx, cidlink.Link{Cid: c}) + if err != nil { + return nil, err + } + } - if err != nil { - return gateway.ContentPathMetadata{}, nil, err - } - md, f, err := blkgw.Head(ctx, path) - if err != nil { - return gateway.ContentPathMetadata{}, nil, err - } - f, err = wrapNodeWithClose(f, closeFn) - if err != nil { - return gateway.ContentPathMetadata{}, nil, err - } - return md, f, nil -} + f := files.NewBytesFile(blockData) + if params.Range != nil && params.Range.From != 0 { + if _, err := f.Seek(params.Range.From, io.SeekStart); err != nil { + return nil, err + } + } -func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, error) { - api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "block", "entityRanges": "0"}).Inc() - blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=block") - if err != nil { - return gateway.ContentPathMetadata{}, err + return gateway.NewGetResponseFromReader(f, int64(len(blockData))), nil } - defer closeFn() - return blkgw.ResolvePath(ctx, path) -} -func (api *GraphGateway) GetCAR(ctx context.Context, path gateway.ImmutablePath, params gateway.CarParams) (gateway.ContentPathMetadata, io.ReadCloser, error) { - api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "all", "entityRanges": "0"}).Inc() - blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=all") + blockData, pbn, ufsFieldData, fieldNum, err := loadUnixFSBase(ctx, c, blk, lsys) if err != nil { - return gateway.ContentPathMetadata{}, nil, err - } - defer closeFn() - return blkgw.GetCAR(ctx, path, params) -} - -func (api *GraphGateway) IsCached(ctx context.Context, path ifacepath.Path) bool { - return false -} - -// TODO: This is copy-paste from blocks gateway, maybe share code -func (api *GraphGateway) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { - if api.routing == nil { - return nil, gateway.NewErrorStatusCode(errors.New("IPNS Record responses are not supported by this gateway"), http.StatusNotImplemented) + return nil, err } - // Fails fast if the CID is not an encoded Libp2p Key, avoids wasteful - // round trips to the remote routing provider. - if multicodec.Code(c.Type()) != multicodec.Libp2pKey { - return nil, gateway.NewErrorStatusCode(errors.New("cid codec must be libp2p-key"), http.StatusBadRequest) - } + switch fieldNum { + case ufsData.Data_Symlink: + if !ufsFieldData.FieldData().Exists() { + return nil, fmt.Errorf("invalid UnixFS symlink object") + } + lnkTarget := string(ufsFieldData.FieldData().Must().Bytes()) + f := gateway.NewGetResponseFromSymlink(files.NewLinkFile(lnkTarget, nil).(*files.Symlink), int64(len(lnkTarget))) + return f, nil + case ufsData.Data_Metadata: + return nil, fmt.Errorf("UnixFS Metadata unsupported") + case ufsData.Data_HAMTShard, ufsData.Data_Directory: + blk, err := blocks.NewBlockWithCid(blockData, c) + if err != nil { + return nil, fmt.Errorf("could not create block: %w", err) + } + dirRootNd, err := merkledag.ProtoNodeConverter(blk, pbn) + if err != nil { + return nil, fmt.Errorf("could not create dag-pb universal block from UnixFS directory root: %w", err) + } + pn, ok := dirRootNd.(*merkledag.ProtoNode) + if !ok { + return nil, fmt.Errorf("could not create dag-pb node from UnixFS directory root: %w", err) + } - // The value store expects the key itself to be encoded as a multihash. - id, err := peer.FromCid(c) - if err != nil { - return nil, err - } + dirDagSize, err := pn.Size() + if err != nil { + return nil, fmt.Errorf("could not get cumulative size from dag-pb node: %w", err) + } - return api.routing.GetValue(ctx, "/ipns/"+string(id)) -} + switch fieldNum { + case ufsData.Data_Directory: + ch := make(chan unixfs.LinkResult, pbn.Links.Length()) + defer close(ch) + iter := pbn.Links.Iterator() + for !iter.Done() { + _, v := iter.Next() + c := v.Hash.Link().(cidlink.Link).Cid + var name string + var size int64 + if v.Name.Exists() { + name = v.Name.Must().String() + } + if v.Tsize.Exists() { + size = v.Tsize.Must().Int() + } + lnk := unixfs.LinkResult{Link: &format.Link{ + Name: name, + Size: uint64(size), + Cid: c, + }} + ch <- lnk + } + return gateway.NewGetResponseFromDirectoryListing(dirDagSize, ch, nil), nil + case ufsData.Data_HAMTShard: + dirNd, err := unixfsnode.Reify(lctx, pbn, lsys) + if err != nil { + return nil, fmt.Errorf("could not reify sharded directory: %w", err) + } -// TODO: This is copy-paste from blocks gateway, maybe share code -func (api *GraphGateway) ResolveMutable(ctx context.Context, p ifacepath.Path) (gateway.ImmutablePath, error) { - err := p.IsValid() - if err != nil { - return gateway.ImmutablePath{}, err - } + d := &backpressuredHAMTDirIterNoRecursion{ + dagSize: dirDagSize, + linksItr: dirNd.MapIterator(), + dirCid: c, + lsys: lsys, + getLsys: getLsys, + ctx: ctx, + closed: make(chan error), + hasClosed: false, + } + return d, nil + default: + return nil, fmt.Errorf("not a basic or HAMT directory: should be unreachable") + } + case ufsData.Data_Raw, ufsData.Data_File: + nd, err := unixfsnode.Reify(lctx, pbn, lsys) + if err != nil { + return nil, err + } - ipath := ipfspath.Path(p.String()) - switch ipath.Segments()[0] { - case "ipns": - ipath, err = resolve.ResolveIPNS(ctx, api.namesys, ipath) + fnd, ok := nd.(datamodel.LargeBytesNode) + if !ok { + return nil, fmt.Errorf("could not process file since it did not present as large bytes") + } + f, err := fnd.AsLargeBytes() if err != nil { - return gateway.ImmutablePath{}, err + return nil, err } - imPath, err := gateway.NewImmutablePath(ifacepath.New(ipath.String())) + + fileSize, err := f.Seek(0, io.SeekEnd) if err != nil { - return gateway.ImmutablePath{}, err + return nil, fmt.Errorf("unable to get UnixFS file size: %w", err) } - return imPath, nil - case "ipfs": - imPath, err := gateway.NewImmutablePath(ifacepath.New(ipath.String())) + + from := int64(0) + var byteRange gateway.DagByteRange + if params.Range != nil { + from = params.Range.From + byteRange = *params.Range + } + _, err = f.Seek(from, io.SeekStart) if err != nil { - return gateway.ImmutablePath{}, err + return nil, fmt.Errorf("unable to get reset UnixFS file reader: %w", err) } - return imPath, nil + + return &backpressuredFile{ctx: ctx, fileCid: c, byteRange: byteRange, size: fileSize, f: f, getLsys: getLsys, closed: make(chan error)}, nil default: - return gateway.ImmutablePath{}, gateway.NewErrorStatusCode(fmt.Errorf("unsupported path namespace: %s", p.Namespace()), http.StatusNotImplemented) + return nil, fmt.Errorf("unknown UnixFS field type") } } -// TODO: This is copy-paste from blocks gateway, maybe share code -func (api *GraphGateway) GetDNSLinkRecord(ctx context.Context, hostname string) (ifacepath.Path, error) { - if api.namesys != nil { - p, err := api.namesys.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1)) - if err == namesys.ErrResolveRecursion { - err = nil - } - return ifacepath.New(p.String()), err - } +type backpressuredHAMTDirIterNoRecursion struct { + dagSize uint64 + linksItr ipld.MapIterator + dirCid cid.Cid - return nil, gateway.NewErrorStatusCode(errors.New("not implemented"), http.StatusNotImplemented) -} + lsys *ipld.LinkSystem + getLsys lsysGetter + ctx context.Context -var _ gateway.IPFSBackend = (*GraphGateway)(nil) + curLnk unixfs.LinkResult + curProcessed int -type inboundBlockExchange struct { - ps BlockPubSub + closed chan error + hasClosed bool + err error } -func newInboundBlockExchange() *inboundBlockExchange { - return &inboundBlockExchange{ - ps: NewBlockPubSub(), - } +func (it *backpressuredHAMTDirIterNoRecursion) AwaitClose() <-chan error { + return it.closed } -func (i *inboundBlockExchange) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { - blk, more := <-i.ps.Subscribe(ctx, c.Hash()) - if err := ctx.Err(); err != nil { - return nil, err - } - if !more { - return nil, format.ErrNotFound{Cid: c} - } - return blk, nil +func (it *backpressuredHAMTDirIterNoRecursion) Link() unixfs.LinkResult { + return it.curLnk } -func (i *inboundBlockExchange) GetBlocks(ctx context.Context, cids []cid.Cid) (<-chan blocks.Block, error) { - mhMap := make(map[string]struct{}) - for _, c := range cids { - mhMap[string(c.Hash())] = struct{}{} - } - mhs := make([]multihash.Multihash, 0, len(mhMap)) - for k := range mhMap { - mhs = append(mhs, multihash.Multihash(k)) - } - return i.ps.Subscribe(ctx, mhs...), nil -} +func (it *backpressuredHAMTDirIterNoRecursion) Next() bool { + defer func() { + if it.linksItr.Done() || it.err != nil { + if !it.hasClosed { + it.hasClosed = true + close(it.closed) + } + } + }() -func (i *inboundBlockExchange) NotifyNewBlocks(ctx context.Context, blocks ...blocks.Block) error { - // TODO: handle context cancellation and/or blockage here - i.ps.Publish(blocks...) - return nil + if it.err != nil { + return false + } + + iter := it.linksItr + if iter.Done() { + return false + } + + /* + Since there is no way to make a graph request for part of a HAMT during errors we can either fill in the HAMT with + block requests, or we can re-request the HAMT and skip over the parts we already have. + + Here we choose the latter, however in the event of a re-request we request the entity rather than the entire DAG as + a compromise between more requests and over-fetching data. + */ + + var err error + for { + if it.ctx.Err() != nil { + it.err = it.ctx.Err() + return false + } + + retry, processedErr := isRetryableError(err) + if !retry { + it.err = processedErr + return false + } + + var nd ipld.Node + if err != nil { + var lsys *ipld.LinkSystem + lsys, err = it.getLsys(it.ctx, it.dirCid, gateway.CarParams{Scope: gateway.DagScopeEntity}) + if err != nil { + continue + } + + _, pbn, ufsFieldData, _, ufsBaseErr := loadUnixFSBase(it.ctx, it.dirCid, nil, lsys) + if ufsBaseErr != nil { + err = ufsBaseErr + continue + } + + nd, err = hamt.NewUnixFSHAMTShard(it.ctx, pbn, ufsFieldData, lsys) + if err != nil { + err = fmt.Errorf("could not reify sharded directory: %w", err) + continue + } + + iter = nd.MapIterator() + for i := 0; i < it.curProcessed; i++ { + _, _, err = iter.Next() + if err != nil { + continue + } + } + + it.linksItr = iter + } + + var k, v ipld.Node + k, v, err = iter.Next() + if err != nil { + retry, processedErr = isRetryableError(err) + if retry { + err = processedErr + continue + } + it.err = processedErr + return false + } + + var name string + name, err = k.AsString() + if err != nil { + it.err = err + return false + } + + var lnk ipld.Link + lnk, err = v.AsLink() + if err != nil { + it.err = err + return false + } + + cl, ok := lnk.(cidlink.Link) + if !ok { + it.err = fmt.Errorf("link not a cidlink") + return false + } + + c := cl.Cid + + pbLnk, ok := v.(*ufsiter.IterLink) + if !ok { + it.err = fmt.Errorf("HAMT value is not a dag-pb link") + return false + } + + cumulativeDagSize := uint64(0) + if pbLnk.Substrate.Tsize.Exists() { + cumulativeDagSize = uint64(pbLnk.Substrate.Tsize.Must().Int()) + } + + it.curLnk = unixfs.LinkResult{ + Link: &format.Link{ + Name: name, + Size: cumulativeDagSize, + Cid: c, + }, + } + it.curProcessed++ + break + } + + return true } -func (i *inboundBlockExchange) Close() error { - i.ps.Shutdown() - return nil +func (it *backpressuredHAMTDirIterNoRecursion) Err() error { + return it.err } -var _ exchange.Interface = (*inboundBlockExchange)(nil) +var _ AwaitCloser = (*backpressuredHAMTDirIterNoRecursion)(nil) -type handoffExchange struct { - startingExchange, followupExchange exchange.Interface - bstore blockstore.Blockstore - handoffCh <-chan struct{} - metrics *GraphGatewayMetrics +func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { + api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "all", "entityRanges": "0"}).Inc() + return fetchWithPartialRetries(ctx, path, gateway.CarParams{Scope: gateway.DagScopeAll}, loadTerminalUnixFSElementWithRecursiveDirectories, api.metrics, api.fetchCAR) } -func (f *handoffExchange) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { - blkCh, err := f.startingExchange.GetBlocks(ctx, []cid.Cid{c}) - if err != nil { - return nil, err - } - blk, ok := <-blkCh - if ok { - return blk, nil - } +type loadTerminalElement[T any] func(ctx context.Context, c cid.Cid, blk blocks.Block, lsys *ipld.LinkSystem, params gateway.CarParams, getLsys lsysGetter) (T, error) +type fetchCarFn = func(ctx context.Context, path gateway.ImmutablePath, params gateway.CarParams, cb DataCallback) error - select { - case <-f.handoffCh: - graphLog.Debugw("switching to backup block fetcher", "cid", c) - f.metrics.blockRecoveryAttemptMetric.Inc() - return f.followupExchange.GetBlock(ctx, c) - case <-ctx.Done(): - return nil, ctx.Err() - } +type terminalPathType[T any] struct { + resp T + err error + md gateway.ContentPathMetadata } -func (f *handoffExchange) GetBlocks(ctx context.Context, cids []cid.Cid) (<-chan blocks.Block, error) { - blkCh, err := f.startingExchange.GetBlocks(ctx, cids) - if err != nil { - return nil, err - } +type nextReq struct { + c cid.Cid + params gateway.CarParams +} - retCh := make(chan blocks.Block) +func fetchWithPartialRetries[T any](ctx context.Context, path gateway.ImmutablePath, initialParams gateway.CarParams, resolveTerminalElementFn loadTerminalElement[T], metrics *GraphGatewayMetrics, fetchCAR fetchCarFn) (gateway.ContentPathMetadata, T, error) { + var zeroReturnType T + + terminalPathElementCh := make(chan terminalPathType[T], 1) go func() { - cs := cid.NewSet() - for cs.Len() < len(cids) { - blk, ok := <-blkCh - if !ok { - break - } + cctx, cancel := context.WithCancel(ctx) + defer cancel() + + hasSentAsyncData := false + var closeCh <-chan error + + sendRequest := make(chan nextReq, 1) + sendResponse := make(chan *ipld.LinkSystem, 1) + getLsys := func(ctx context.Context, c cid.Cid, params gateway.CarParams) (*ipld.LinkSystem, error) { select { - case retCh <- blk: - cs.Add(blk.Cid()) + case sendRequest <- nextReq{c: c, params: params}: case <-ctx.Done(): + return nil, ctx.Err() } - } - for cs.Len() < len(cids) { select { + case lsys := <-sendResponse: + return lsys, nil case <-ctx.Done(): - return - case <-f.handoffCh: - var newCidArr []cid.Cid - for _, c := range cids { - if !cs.Has(c) { - blk, _ := f.bstore.Get(ctx, c) - if blk != nil { - select { - case retCh <- blk: - cs.Add(blk.Cid()) - case <-ctx.Done(): - return - } - } else { - newCidArr = append(newCidArr, c) - } - } - } + return nil, ctx.Err() + } + } - if len(newCidArr) == 0 { - return + p := ipfspath.FromString(path.String()) + params := initialParams + + err := fetchCAR(cctx, path, params, func(resource string, reader io.Reader) error { + gb, err := carToLinearBlockGetter(cctx, reader, metrics) + if err != nil { + return err + } + + lsys := getLinksystem(gb) + + if hasSentAsyncData { + _, _, _, _, err = resolvePathToLastWithRoots(cctx, p, lsys) + if err != nil { + return err } - graphLog.Debugw("needed to use use a backup fetcher for cids", "cids", newCidArr) - f.metrics.blockRecoveryAttemptMetric.Add(float64(len(newCidArr))) - fch, err := f.followupExchange.GetBlocks(ctx, newCidArr) + select { + case sendResponse <- lsys: + case <-cctx.Done(): + return cctx.Err() + } + } else { + // First resolve the path since we always need to. + pathRootCids, terminalCid, remainder, terminalBlk, err := resolvePathWithRootsAndBlock(cctx, p, lsys) if err != nil { - graphLog.Errorw("error getting blocks from followupExchange", "error", err) - return + return err } - for cs.Len() < len(cids) { - blk, ok := <-fch - if !ok { - return - } + md := contentMetadataFromRootsAndRemainder(p, pathRootCids, terminalCid, remainder) + + if len(remainder) > 0 { + terminalPathElementCh <- terminalPathType[T]{err: errNotUnixFS} + return nil + } + + if hasSentAsyncData { select { - case retCh <- blk: - cs.Add(blk.Cid()) + case sendResponse <- lsys: case <-ctx.Done(): - return + return ctx.Err() + } + } + + nd, err := resolveTerminalElementFn(cctx, terminalCid, terminalBlk, lsys, params, getLsys) + if err != nil { + return err + } + + ndAc, ok := any(nd).(AwaitCloser) + if !ok { + terminalPathElementCh <- terminalPathType[T]{ + resp: nd, + md: md, } + return nil + } + + hasSentAsyncData = true + terminalPathElementCh <- terminalPathType[T]{ + resp: nd, + md: md, + } + + closeCh = ndAc.AwaitClose() + } + + select { + case closeErr := <-closeCh: + return closeErr + case req := <-sendRequest: + // set path and params for next iteration + p = ipfspath.FromCid(req.c) + imPath, err := gateway.NewImmutablePath(ifacepath.New(p.String())) + if err != nil { + return err + } + params = req.params + remainderUrl := contentPathToCarUrl(imPath, params).String() + return ErrPartialResponse{StillNeed: []string{remainderUrl}} + case <-cctx.Done(): + return cctx.Err() + } + }) + + if !hasSentAsyncData && err != nil { + terminalPathElementCh <- terminalPathType[T]{err: err} + return + } + + if err != nil { + lsys := getLinksystem(func(ctx context.Context, cid cid.Cid) (blocks.Block, error) { + return nil, multierror.Append(ErrFetcherUnexpectedEOF, format.ErrNotFound{Cid: cid}) + }) + for { + select { + case <-closeCh: + return + case <-sendRequest: + case sendResponse <- lsys: + case <-cctx.Done(): + return } } } }() - return retCh, nil + + select { + case t := <-terminalPathElementCh: + if t.err != nil { + return gateway.ContentPathMetadata{}, zeroReturnType, t.err + } + return t.md, t.resp, nil + case <-ctx.Done(): + return gateway.ContentPathMetadata{}, zeroReturnType, ctx.Err() + } +} + +func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.File, error) { + api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "block", "entityRanges": "0"}).Inc() + p := ipfspath.FromString(path.String()) + + var md gateway.ContentPathMetadata + var f files.File + // TODO: if path is `/ipfs/cid`, we should use ?format=raw + err := api.fetchCAR(ctx, path, gateway.CarParams{Scope: gateway.DagScopeBlock}, func(resource string, reader io.Reader) error { + gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) + if err != nil { + return err + } + lsys := getLinksystem(gb) + + // First resolve the path since we always need to. + pathRoots, terminalCid, remainder, terminalBlk, err := resolvePathToLastWithRoots(ctx, p, lsys) + if err != nil { + return err + } + + var blockData []byte + if terminalBlk != nil { + blockData = terminalBlk.RawData() + } else { + lctx := ipld.LinkContext{Ctx: ctx} + lnk := cidlink.Link{Cid: terminalCid} + blockData, err = lsys.LoadRaw(lctx, lnk) + if err != nil { + return err + } + } + + md = contentMetadataFromRootsAndRemainder(p, pathRoots, terminalCid, remainder) + + f = files.NewBytesFile(blockData) + return nil + }) + + if err != nil { + return gateway.ContentPathMetadata{}, nil, err + } + + return md, f, nil +} + +func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, *gateway.HeadResponse, error) { + api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "entity", "entityRanges": "1"}).Inc() + + // TODO: we probably want to move this either to boxo, or at least to loadRequestIntoSharedBlockstoreAndBlocksGateway + api.metrics.bytesRangeStartMetric.Observe(0) + api.metrics.bytesRangeSizeMetric.Observe(3071) + + p := ipfspath.FromString(path.String()) + + var md gateway.ContentPathMetadata + var n *gateway.HeadResponse + // TODO: fallback to dynamic fetches in case we haven't requested enough data + rangeTo := int64(3071) + err := api.fetchCAR(ctx, path, gateway.CarParams{Scope: gateway.DagScopeEntity, Range: &gateway.DagByteRange{From: 0, To: &rangeTo}}, func(resource string, reader io.Reader) error { + gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) + if err != nil { + return err + } + lsys := getLinksystem(gb) + + // First resolve the path since we always need to. + pathRoots, terminalCid, remainder, terminalBlk, err := resolvePathWithRootsAndBlock(ctx, p, lsys) + if err != nil { + return err + } + + md = contentMetadataFromRootsAndRemainder(p, pathRoots, terminalCid, remainder) + + lctx := ipld.LinkContext{Ctx: ctx} + pathTerminalCidLink := cidlink.Link{Cid: terminalCid} + + // Load the block at the root of the terminal path element + dataBytes := terminalBlk.RawData() + + // It's not UnixFS if there is a remainder or it's not dag-pb + if len(remainder) > 0 || terminalCid.Type() != uint64(multicodec.DagPb) { + n = gateway.NewHeadResponseForFile(files.NewBytesFile(dataBytes), int64(len(dataBytes))) + return nil + } + + // Let's figure out if the terminal element is valid UnixFS and if so what kind + np, err := api.pc(pathTerminalCidLink, lctx) + if err != nil { + return err + } + + nodeDecoder, err := lsys.DecoderChooser(pathTerminalCidLink) + if err != nil { + return err + } + + nb := np.NewBuilder() + err = nodeDecoder(nb, bytes.NewReader(dataBytes)) + if err != nil { + return err + } + lastCidNode := nb.Build() + + if pbn, ok := lastCidNode.(dagpb.PBNode); !ok { + // This shouldn't be possible since we already checked for dag-pb usage + return fmt.Errorf("node was not go-codec-dagpb node") + } else if !pbn.FieldData().Exists() { + // If it's not valid UnixFS then just return the block bytes + n = gateway.NewHeadResponseForFile(files.NewBytesFile(dataBytes), int64(len(dataBytes))) + return nil + } else if unixfsFieldData, decodeErr := ufsData.DecodeUnixFSData(pbn.Data.Must().Bytes()); decodeErr != nil { + // If it's not valid UnixFS then just return the block bytes + n = gateway.NewHeadResponseForFile(files.NewBytesFile(dataBytes), int64(len(dataBytes))) + return nil + } else { + switch fieldNum := unixfsFieldData.FieldDataType().Int(); fieldNum { + case ufsData.Data_Directory, ufsData.Data_HAMTShard: + dirRootNd, err := merkledag.ProtoNodeConverter(terminalBlk, lastCidNode) + if err != nil { + return fmt.Errorf("could not create dag-pb universal block from UnixFS directory root: %w", err) + } + pn, ok := dirRootNd.(*merkledag.ProtoNode) + if !ok { + return fmt.Errorf("could not create dag-pb node from UnixFS directory root: %w", err) + } + + sz, err := pn.Size() + if err != nil { + return fmt.Errorf("could not get cumulative size from dag-pb node: %w", err) + } + + n = gateway.NewHeadResponseForDirectory(int64(sz)) + return nil + case ufsData.Data_Symlink: + fd := unixfsFieldData.FieldData() + if fd.Exists() { + n = gateway.NewHeadResponseForSymlink(int64(len(fd.Must().Bytes()))) + return nil + } + // If there is no target then it's invalid so just return the block + gateway.NewHeadResponseForFile(files.NewBytesFile(dataBytes), int64(len(dataBytes))) + return nil + case ufsData.Data_Metadata: + n = gateway.NewHeadResponseForFile(files.NewBytesFile(dataBytes), int64(len(dataBytes))) + return nil + case ufsData.Data_Raw, ufsData.Data_File: + ufsNode, err := unixfsnode.Reify(lctx, pbn, lsys) + if err != nil { + return err + } + fileNode, ok := ufsNode.(datamodel.LargeBytesNode) + if !ok { + return fmt.Errorf("data not a large bytes node despite being UnixFS bytes") + } + f, err := fileNode.AsLargeBytes() + if err != nil { + return err + } + + fileSize, err := f.Seek(0, io.SeekEnd) + if err != nil { + return fmt.Errorf("unable to get UnixFS file size: %w", err) + } + _, err = f.Seek(0, io.SeekStart) + if err != nil { + return fmt.Errorf("unable to get reset UnixFS file reader: %w", err) + } + + out, err := io.ReadAll(io.LimitReader(f, 3072)) + if errors.Is(err, io.EOF) { + n = gateway.NewHeadResponseForFile(files.NewBytesFile(out), fileSize) + return nil + } + return err + } + } + return nil + }) + + if err != nil { + return gateway.ContentPathMetadata{}, nil, err + } + + return md, n, nil } -func (f *handoffExchange) NotifyNewBlocks(ctx context.Context, blocks ...blocks.Block) error { - err1 := f.startingExchange.NotifyNewBlocks(ctx, blocks...) - err2 := f.followupExchange.NotifyNewBlocks(ctx, blocks...) - return multierr.Combine(err1, err2) +func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, error) { + api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "block", "entityRanges": "0"}).Inc() + + var md gateway.ContentPathMetadata + err := api.fetchCAR(ctx, path, gateway.CarParams{Scope: gateway.DagScopeBlock}, func(resource string, reader io.Reader) error { + gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) + if err != nil { + return err + } + lsys := getLinksystem(gb) + + // First resolve the path since we always need to. + p := ipfspath.FromString(path.String()) + pathRoots, terminalCid, remainder, _, err := resolvePathToLastWithRoots(ctx, p, lsys) + if err != nil { + return err + } + + md = contentMetadataFromRootsAndRemainder(p, pathRoots, terminalCid, remainder) + + return nil + }) + + if err != nil { + return gateway.ContentPathMetadata{}, err + } + + return md, nil } -func (f *handoffExchange) Close() error { - err1 := f.startingExchange.Close() - err2 := f.followupExchange.Close() - return multierr.Combine(err1, err2) +func (api *GraphGateway) GetCAR(ctx context.Context, path gateway.ImmutablePath, params gateway.CarParams) (gateway.ContentPathMetadata, io.ReadCloser, error) { + numRanges := "0" + if params.Range != nil { + numRanges = "1" + } + api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": string(params.Scope), "entityRanges": numRanges}).Inc() + rootCid, err := getRootCid(path) + if err != nil { + return gateway.ContentPathMetadata{}, nil, err + } + p := ipfspath.FromString(path.String()) + + switch params.Order { + case gateway.DagOrderUnspecified, gateway.DagOrderUnknown, gateway.DagOrderDFS: + default: + return gateway.ContentPathMetadata{}, nil, fmt.Errorf("unsupported dag order %q", params.Order) + } + + r, w := io.Pipe() + go func() { + numBlocksSent := 0 + var cw storage.WritableCar + var blockBuffer []blocks.Block + err = api.fetchCAR(ctx, path, params, func(resource string, reader io.Reader) error { + numBlocksThisCall := 0 + gb, err := carToLinearBlockGetter(ctx, reader, api.metrics) + if err != nil { + return err + } + teeBlock := func(ctx context.Context, c cid.Cid) (blocks.Block, error) { + blk, err := gb(ctx, c) + if err != nil { + return nil, err + } + if numBlocksThisCall >= numBlocksSent { + if cw == nil { + blockBuffer = append(blockBuffer, blk) + } else { + err = cw.Put(ctx, blk.Cid().KeyString(), blk.RawData()) + if err != nil { + return nil, fmt.Errorf("error writing car block: %w", err) + } + } + numBlocksSent++ + } + numBlocksThisCall++ + return blk, nil + } + l := getLinksystem(teeBlock) + + // First resolve the path since we always need to. + _, terminalCid, remainder, terminalBlk, err := resolvePathWithRootsAndBlock(ctx, p, l) + if err != nil { + return err + } + if len(remainder) > 0 { + return nil + } + + if cw == nil { + cw, err = storage.NewWritable(w, []cid.Cid{terminalCid}, carv2.WriteAsCarV1(true), carv2.AllowDuplicatePuts(params.Duplicates.Bool())) + if err != nil { + // io.PipeWriter.CloseWithError always returns nil. + _ = w.CloseWithError(err) + return nil + } + for _, blk := range blockBuffer { + err = cw.Put(ctx, blk.Cid().KeyString(), blk.RawData()) + if err != nil { + _ = w.CloseWithError(fmt.Errorf("error writing car block: %w", err)) + return nil + } + } + blockBuffer = nil + } + + err = walkGatewaySimpleSelector(ctx, terminalBlk, params.Scope, params.Range, l) + if err != nil { + return err + } + return nil + }) + + _ = w.CloseWithError(err) + }() + + return gateway.ContentPathMetadata{ + PathSegmentRoots: []cid.Cid{rootCid}, + LastSegment: ifacepath.NewResolvedPath(p, rootCid, rootCid, ""), + ContentType: "", + }, r, nil } -var _ exchange.Interface = (*handoffExchange)(nil) +func getRootCid(imPath gateway.ImmutablePath) (cid.Cid, error) { + imPathStr := imPath.String() + if !strings.HasPrefix(imPathStr, "/ipfs/") { + return cid.Undef, fmt.Errorf("path does not have /ipfs/ prefix") + } + + firstSegment, _, _ := strings.Cut(imPathStr[6:], "/") + rootCid, err := cid.Decode(firstSegment) + if err != nil { + return cid.Undef, err + } -type blockFetcherExchWrapper struct { - f exchange.Fetcher + return rootCid, nil } -func (b *blockFetcherExchWrapper) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { - return b.f.GetBlock(ctx, c) +func (api *GraphGateway) IsCached(ctx context.Context, path ifacepath.Path) bool { + return false } -func (b *blockFetcherExchWrapper) GetBlocks(ctx context.Context, cids []cid.Cid) (<-chan blocks.Block, error) { - return b.f.GetBlocks(ctx, cids) +// TODO: This is copy-paste from blocks gateway, maybe share code +func (api *GraphGateway) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { + if api.routing == nil { + return nil, gateway.NewErrorStatusCode(errors.New("IPNS Record responses are not supported by this gateway"), http.StatusNotImplemented) + } + + // Fails fast if the CID is not an encoded Libp2p Key, avoids wasteful + // round trips to the remote routing provider. + if multicodec.Code(c.Type()) != multicodec.Libp2pKey { + return nil, gateway.NewErrorStatusCode(errors.New("cid codec must be libp2p-key"), http.StatusBadRequest) + } + + // The value store expects the key itself to be encoded as a multihash. + id, err := peer.FromCid(c) + if err != nil { + return nil, err + } + + return api.routing.GetValue(ctx, "/ipns/"+string(id)) } -func (b *blockFetcherExchWrapper) NotifyNewBlocks(ctx context.Context, blocks ...blocks.Block) error { - return nil +// TODO: This is copy-paste from blocks gateway, maybe share code +func (api *GraphGateway) ResolveMutable(ctx context.Context, p ifacepath.Path) (gateway.ImmutablePath, error) { + err := p.IsValid() + if err != nil { + return gateway.ImmutablePath{}, err + } + + ipath := ipfspath.Path(p.String()) + switch ipath.Segments()[0] { + case "ipns": + ipath, err = resolve.ResolveIPNS(ctx, api.namesys, ipath) + if err != nil { + return gateway.ImmutablePath{}, err + } + imPath, err := gateway.NewImmutablePath(ifacepath.New(ipath.String())) + if err != nil { + return gateway.ImmutablePath{}, err + } + return imPath, nil + case "ipfs": + imPath, err := gateway.NewImmutablePath(ifacepath.New(ipath.String())) + if err != nil { + return gateway.ImmutablePath{}, err + } + return imPath, nil + default: + return gateway.ImmutablePath{}, gateway.NewErrorStatusCode(fmt.Errorf("unsupported path namespace: %s", p.Namespace()), http.StatusNotImplemented) + } +} + +// TODO: This is copy-paste from blocks gateway, maybe share code +func (api *GraphGateway) GetDNSLinkRecord(ctx context.Context, hostname string) (ifacepath.Path, error) { + if api.namesys != nil { + p, err := api.namesys.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1)) + if err == namesys.ErrResolveRecursion { + err = nil + } + return ifacepath.New(p.String()), err + } + + return nil, gateway.NewErrorStatusCode(errors.New("not implemented"), http.StatusNotImplemented) } -func (b *blockFetcherExchWrapper) Close() error { +var _ gateway.IPFSBackend = (*GraphGateway)(nil) + +func checkRetryableError(e *error, fn func() error) error { + err := fn() + retry, processedErr := isRetryableError(err) + if retry { + return processedErr + } + *e = processedErr return nil } -var _ exchange.Interface = (*blockFetcherExchWrapper)(nil) +func isRetryableError(err error) (bool, error) { + if errors.Is(err, ErrFetcherUnexpectedEOF) { + return false, err + } + + if format.IsNotFound(err) { + return true, err + } + initialErr := err + + // Checks if err is of a type that does not implement the .Is interface and + // cannot be directly compared to. Therefore, errors.Is cannot be used. + for { + _, ok := err.(resolver.ErrNoLink) + if ok { + return false, err + } + + _, ok = err.(datamodel.ErrWrongKind) + if ok { + return false, err + } + + _, ok = err.(datamodel.ErrNotExists) + if ok { + return false, err + } + + errNoSuchField, ok := err.(schema.ErrNoSuchField) + if ok { + // Convert into a more general error type so the gateway code can know what this means + // TODO: Have either a more generally usable error type system for IPLD errors (e.g. a base type indicating that data cannot exist) + // or at least have one that is specific to the gateway consumer and part of the Backend contract instead of this being implicit + err = datamodel.ErrNotExists{Segment: errNoSuchField.Field} + return false, err + } + + err = errors.Unwrap(err) + if err == nil { + return true, initialErr + } + } +} diff --git a/lib/pubsub.go b/lib/pubsub.go deleted file mode 100644 index 0d1d553..0000000 --- a/lib/pubsub.go +++ /dev/null @@ -1,146 +0,0 @@ -package lib - -import ( - "context" - "sync" - - "github.com/multiformats/go-multihash" - - pubsub "github.com/cskr/pubsub" - blocks "github.com/ipfs/go-block-format" -) - -// Note: blantantly copied from boxo/bitswap internals https://github.com/ipfs/boxo/blob/664b3e5ee4a997c67909da9b017a28efa40cc8ae/bitswap/client/internal/notifications/notifications.go -// some minor changes were made - -const bufferSize = 16 - -// BlockPubSub is a simple interface for publishing blocks and being able to subscribe -// for multihashes. It's used internally by an exchange to decouple receiving blocks -// and actually providing them back to the GetBlocks caller. -// Note: because multihashes are being requested and blocks returned the codecs could be anything -type BlockPubSub interface { - Publish(blocks ...blocks.Block) - Subscribe(ctx context.Context, keys ...multihash.Multihash) <-chan blocks.Block - Shutdown() -} - -// NewBlockPubSub generates a new BlockPubSub interface. -func NewBlockPubSub() BlockPubSub { - return &impl{ - wrapped: *pubsub.New(bufferSize), - closed: make(chan struct{}), - } -} - -type impl struct { - lk sync.RWMutex - wrapped pubsub.PubSub - - closed chan struct{} -} - -func (ps *impl) Publish(blocks ...blocks.Block) { - ps.lk.RLock() - defer ps.lk.RUnlock() - select { - case <-ps.closed: - return - default: - } - - for _, block := range blocks { - ps.wrapped.Pub(block, string(block.Cid().Hash())) - } -} - -func (ps *impl) Shutdown() { - ps.lk.Lock() - defer ps.lk.Unlock() - select { - case <-ps.closed: - return - default: - } - close(ps.closed) - ps.wrapped.Shutdown() -} - -// Subscribe returns a channel of blocks for the given |keys|. |blockChannel| -// is closed if the |ctx| times out or is cancelled, or after receiving the blocks -// corresponding to |keys|. -func (ps *impl) Subscribe(ctx context.Context, keys ...multihash.Multihash) <-chan blocks.Block { - - blocksCh := make(chan blocks.Block, len(keys)) - valuesCh := make(chan interface{}, len(keys)) // provide our own channel to control buffer, prevent blocking - if len(keys) == 0 { - close(blocksCh) - return blocksCh - } - - // prevent shutdown - ps.lk.RLock() - defer ps.lk.RUnlock() - - select { - case <-ps.closed: - close(blocksCh) - return blocksCh - default: - } - - // AddSubOnceEach listens for each key in the list, and closes the channel - // once all keys have been received - ps.wrapped.AddSubOnceEach(valuesCh, toStrings(keys)...) - go func() { - defer func() { - close(blocksCh) - - ps.lk.RLock() - defer ps.lk.RUnlock() - // Don't touch the pubsub instance if we're - // already closed. - select { - case <-ps.closed: - return - default: - } - - ps.wrapped.Unsub(valuesCh) - }() - - for { - select { - case <-ctx.Done(): - return - case <-ps.closed: - return - case val, ok := <-valuesCh: - if !ok { - return - } - block, ok := val.(blocks.Block) - if !ok { - return - } - select { - case <-ctx.Done(): - return - case blocksCh <- block: // continue - case <-ps.closed: - return - } - } - } - }() - - return blocksCh -} - -func toStrings(keys []multihash.Multihash) []string { - strs := make([]string, 0, len(keys)) - for _, key := range keys { - strs = append(strs, string(key)) - } - return strs -} diff --git a/test-backend/handlers.go b/test-backend/handlers.go deleted file mode 100644 index 231583e..0000000 --- a/test-backend/handlers.go +++ /dev/null @@ -1,627 +0,0 @@ -package main - -import ( - "bytes" - "context" - "fmt" - "io" - "net/http" - "net/url" - "runtime/debug" - "strconv" - "strings" - "sync" - "time" - - "github.com/ipfs/boxo/fetcher" - "github.com/ipfs/boxo/files" - "github.com/ipfs/boxo/gateway" - "github.com/ipfs/boxo/ipld/merkledag" - unixfile "github.com/ipfs/boxo/ipld/unixfs/file" - "github.com/ipfs/boxo/path" - "github.com/ipfs/boxo/path/resolver" - blocks "github.com/ipfs/go-block-format" - format "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-unixfsnode" - "github.com/ipld/go-car" - "github.com/ipld/go-car/util" - dagpb "github.com/ipld/go-codec-dagpb" - "github.com/ipld/go-ipld-prime" - cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "github.com/ipld/go-ipld-prime/node/basicnode" - "github.com/ipld/go-ipld-prime/schema" - "github.com/ipld/go-ipld-prime/traversal" - "github.com/ipld/go-ipld-prime/traversal/selector" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" - - "github.com/ipfs/boxo/blockservice" - ipath "github.com/ipfs/boxo/coreiface/path" - "github.com/ipfs/go-cid" -) - -func makeMetricsHandler(port int) (*http.Server, error) { - mux := http.NewServeMux() - - gatherers := prometheus.Gatherers{ - prometheus.DefaultGatherer, - } - options := promhttp.HandlerOpts{} - mux.Handle("/debug/metrics/prometheus", promhttp.HandlerFor(gatherers, options)) - - return &http.Server{ - Handler: mux, - Addr: ":" + strconv.Itoa(port), - }, nil -} - -func withRequestLogger(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - goLog.Infow(r.Method, "url", r.URL, "host", r.Host) - // TODO: if debug is enabled, show more? goLog.Infow("request received", "url", r.URL, "host", r.Host, "method", r.Method, "ua", r.UserAgent(), "referer", r.Referer()) - next.ServeHTTP(w, r) - }) -} - -var noModtime = time.Unix(0, 0) - -func makeGatewayCARHandler(bsrv blockservice.BlockService, port int) (*http.Server, error) { - mux := http.NewServeMux() - mux.HandleFunc("/ipfs/", func(w http.ResponseWriter, r *http.Request) { - // the hour is a hard fallback, we don't expect it to happen, but just in case - ctx, cancel := context.WithTimeout(r.Context(), time.Hour) - defer cancel() - r = r.WithContext(ctx) - - defer func() { - if r := recover(); r != nil { - goLog.Error("A panic occurred in the gateway handler!") - goLog.Error(r) - debug.PrintStack() - } - }() - - if r.Method != http.MethodGet { - w.Header().Add("Allow", http.MethodGet) - - errmsg := "Method " + r.Method + " not allowed" - http.Error(w, errmsg, http.StatusMethodNotAllowed) - return - } - - isCar := false - if formatParam := r.URL.Query().Get("format"); formatParam != "" { - isCar = formatParam == "car" - if !isCar { - http.Error(w, "only car format supported", http.StatusBadRequest) - return - } - } else { - for _, header := range r.Header.Values("Accept") { - for _, value := range strings.Split(header, ",") { - accept := strings.TrimSpace(value) - if strings.HasPrefix(accept, "application/vnd.ipld.car") { - isCar = true - break - } - } - } - } - if !isCar { - http.Error(w, "only car format supported", http.StatusBadRequest) - return - } - - contentPath := ipath.New(r.URL.Path) - if contentPath.Mutable() { - http.Error(w, "only immutable block requests supported", http.StatusBadRequest) - return - } else if contentPath.Namespace() != "ipfs" { - http.Error(w, "only the ipfs names is supported", http.StatusBadRequest) - return - } - - carStream, err := simpleSelectorToCar(ctx, bsrv, contentPath.String(), r.URL.Query()) - if err != nil { - http.Error(w, "only the ipfs names is supported", http.StatusBadRequest) - return - } - - const immutableCacheControl = "public, max-age=29030400, immutable" - // immutable! CACHE ALL THE THINGS, FOREVER! wolololol - w.Header().Set("Cache-Control", immutableCacheControl) - w.Header().Set("Server", userAgent) - - // Set modtime to 'zero time' to disable Last-Modified header (superseded by Cache-Control) - - io.Copy(w, carStream) - }) - - // Creates metrics handler for total response size. Matches the same metrics - // from Kubo: - // https://github.com/ipfs/kubo/blob/e550d9e4761ea394357c413c02ade142c0dea88c/core/corehttp/metrics.go#L79-L152 - sum := prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Namespace: "ipfs", - Subsystem: "http", - Name: "response_size_bytes", - Help: "The HTTP response sizes in bytes.", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - }, nil) - err := prometheus.Register(sum) - if err != nil { - return nil, err - } - - // Construct the HTTP handler for the gateway. - handler := promhttp.InstrumentHandlerResponseSize(sum, mux) - - // Add logging - handler = withRequestLogger(handler) - - return &http.Server{ - Handler: handler, - Addr: ":" + strconv.Itoa(port), - }, nil -} - -func simpleSelectorToCar(ctx context.Context, bsrv blockservice.BlockService, p string, params url.Values) (io.ReadCloser, error) { - pathSegs := strings.Split(p, "/") - if len(pathSegs) < 3 || !(pathSegs[0] == "" && pathSegs[1] == "ipfs") { - return nil, fmt.Errorf("invalid path") - } - pathSegs = pathSegs[2:] - rootCidStr := pathSegs[0] - rootCid, err := cid.Decode(rootCidStr) - if err != nil { - return nil, err - } - - ipfspath, err := path.ParsePath(p) - if err != nil { - return nil, err - } - - r, w := io.Pipe() - - rangeStr, hasRange := params.Get("bytes"), params.Has("bytes") - depthStr, hasDepth := params.Get("depth"), params.Has("depth") - - if hasDepth && !(depthStr == "0" || depthStr == "1" || depthStr == "all") { - return nil, fmt.Errorf("depth type: %s not supported", depthStr) - } - var getRange *gateway.ByteRange - if hasRange { - getRange, err = rangeStrToByteRange(rangeStr) - if err != nil { - return nil, err - } - } - - go func() { - defer w.Close() - - // Setup header for the output car - err = car.WriteHeader(&car.CarHeader{ - Roots: []cid.Cid{rootCid}, - Version: 1, - }, w) - if err != nil { - goLog.Error(fmt.Errorf("writing car header: %w", err)) - } - - blockGetter := merkledag.NewDAGService(bsrv).Session(ctx) - blockGetter = &nodeGetterToCarExporer{ - ng: blockGetter, - w: w, - mhSet: make(map[string]struct{}), - } - dsrv := merkledag.NewReadOnlyDagService(blockGetter) - - // Setup the UnixFS resolver. - f := newNodeGetterFetcherSingleUseFactory(ctx, blockGetter) - pathResolver := resolver.NewBasicResolver(f) - - lastCid, remainder, err := pathResolver.ResolveToLastNode(ctx, ipfspath) - if err != nil { - goLog.Error(err) - return - } - - if hasDepth && depthStr == "0" { - return - } - - lastCidNode, err := dsrv.Get(ctx, lastCid) - if err != nil { - goLog.Error(err) - return - } - - ufsNode, err := unixfile.NewUnixfsFile(ctx, dsrv, lastCidNode) - if err != nil { - // It's not UnixFS - - // If it's all fetch the graph recursively - if depthStr == "all" { - if err := merkledag.FetchGraph(ctx, lastCid, dsrv); err != nil { - goLog.Error(err) - } - return - } - - //if not then either this is an error (which we can't report) or this is the last block for us to return - return - } - if f, ok := ufsNode.(files.File); ok { - if len(remainder) > 0 { - // this is an error, so we're done - return - } - - if !hasRange { - nw, err := io.Copy(io.Discard, f) - goLog.Debugf("nwritten %d", nw) - if err != nil { - goLog.Error(err) - } - return - } - - // TODO: testing + check off by one errors - var numToRead int64 - if *getRange.To < 0 { - size, err := f.Seek(0, io.SeekEnd) - if err != nil { - return - } - numToRead = (size - *getRange.To) - int64(getRange.From) - } else { - numToRead = int64(getRange.From) - *getRange.To - } - - if _, err := f.Seek(int64(getRange.From), io.SeekStart); err != nil { - return - } - _, _ = io.CopyN(io.Discard, f, numToRead) - return - } else if d, ok := ufsNode.(files.Directory); ok { - if depthStr == "1" { - iter := d.Entries() - for iter.Next() { - } - return - } - if depthStr == "all" { - // TODO: being lazy here - w, err := files.NewTarWriter(io.Discard) - if err != nil { - goLog.Error(fmt.Errorf("could not create tar write %w", err)) - return - } - if err := w.WriteFile(d, "tmp"); err != nil { - goLog.Error(err) - return - } - return - } - } else { - return - } - }() - return r, nil -} - -type nodeGetterToCarExporer struct { - ng format.NodeGetter - w io.Writer - - lk sync.RWMutex - mhSet map[string]struct{} -} - -func (n *nodeGetterToCarExporer) Get(ctx context.Context, c cid.Cid) (format.Node, error) { - nd, err := n.ng.Get(ctx, c) - if err != nil { - return nil, err - } - - if err := n.trySendBlock(nd); err != nil { - return nil, err - } - - return nd, nil -} - -func (n *nodeGetterToCarExporer) GetMany(ctx context.Context, cids []cid.Cid) <-chan *format.NodeOption { - ndCh := n.ng.GetMany(ctx, cids) - outCh := make(chan *format.NodeOption) - go func() { - defer close(outCh) - for nd := range ndCh { - if nd.Err == nil { - if err := n.trySendBlock(nd.Node); err != nil { - select { - case outCh <- &format.NodeOption{Err: err}: - case <-ctx.Done(): - } - return - } - select { - case outCh <- nd: - case <-ctx.Done(): - } - } - } - }() - return outCh -} - -func (n *nodeGetterToCarExporer) trySendBlock(block blocks.Block) error { - h := string(block.Cid().Hash()) - n.lk.RLock() - _, found := n.mhSet[h] - n.lk.RUnlock() - if !found { - doSend := false - n.lk.Lock() - _, found := n.mhSet[h] - if !found { - doSend = true - n.mhSet[h] = struct{}{} - } - n.lk.Unlock() - if doSend { - err := util.LdWrite(n.w, block.Cid().Bytes(), block.RawData()) // write to the output car - if err != nil { - return fmt.Errorf("writing to output car: %w", err) - } - } - } - return nil -} - -var _ format.NodeGetter = (*nodeGetterToCarExporer)(nil) - -type nodeGetterFetcherSingleUseFactory struct { - linkSystem ipld.LinkSystem - protoChooser traversal.LinkTargetNodePrototypeChooser -} - -func newNodeGetterFetcherSingleUseFactory(ctx context.Context, ng format.NodeGetter) *nodeGetterFetcherSingleUseFactory { - ls := cidlink.DefaultLinkSystem() - ls.TrustedStorage = true - ls.StorageReadOpener = blockOpener(ctx, ng) - ls.NodeReifier = unixfsnode.Reify - - pc := dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { - if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { - return tlnkNd.LinkTargetNodePrototype(), nil - } - return basicnode.Prototype.Any, nil - }) - - return &nodeGetterFetcherSingleUseFactory{ls, pc} -} - -func (n *nodeGetterFetcherSingleUseFactory) NewSession(ctx context.Context) fetcher.Fetcher { - return n -} - -func (n *nodeGetterFetcherSingleUseFactory) NodeMatching(ctx context.Context, root ipld.Node, selector ipld.Node, cb fetcher.FetchCallback) error { - return n.nodeMatching(ctx, n.blankProgress(ctx), root, selector, cb) -} - -func (n *nodeGetterFetcherSingleUseFactory) BlockOfType(ctx context.Context, link ipld.Link, nodePrototype ipld.NodePrototype) (ipld.Node, error) { - return n.linkSystem.Load(ipld.LinkContext{}, link, nodePrototype) -} - -func (n *nodeGetterFetcherSingleUseFactory) BlockMatchingOfType(ctx context.Context, root ipld.Link, selector ipld.Node, nodePrototype ipld.NodePrototype, cb fetcher.FetchCallback) error { - // retrieve first node - prototype, err := n.PrototypeFromLink(root) - if err != nil { - return err - } - node, err := n.BlockOfType(ctx, root, prototype) - if err != nil { - return err - } - - progress := n.blankProgress(ctx) - progress.LastBlock.Link = root - return n.nodeMatching(ctx, progress, node, selector, cb) -} - -func (n *nodeGetterFetcherSingleUseFactory) PrototypeFromLink(lnk ipld.Link) (ipld.NodePrototype, error) { - return n.protoChooser(lnk, ipld.LinkContext{}) -} - -func (n *nodeGetterFetcherSingleUseFactory) nodeMatching(ctx context.Context, initialProgress traversal.Progress, node ipld.Node, match ipld.Node, cb fetcher.FetchCallback) error { - matchSelector, err := selector.ParseSelector(match) - if err != nil { - return err - } - return initialProgress.WalkMatching(node, matchSelector, func(prog traversal.Progress, n ipld.Node) error { - return cb(fetcher.FetchResult{ - Node: n, - Path: prog.Path, - LastBlockPath: prog.LastBlock.Path, - LastBlockLink: prog.LastBlock.Link, - }) - }) -} - -func (n *nodeGetterFetcherSingleUseFactory) blankProgress(ctx context.Context) traversal.Progress { - return traversal.Progress{ - Cfg: &traversal.Config{ - LinkSystem: n.linkSystem, - LinkTargetNodePrototypeChooser: n.protoChooser, - }, - } -} - -func blockOpener(ctx context.Context, ng format.NodeGetter) ipld.BlockReadOpener { - return func(_ ipld.LinkContext, lnk ipld.Link) (io.Reader, error) { - cidLink, ok := lnk.(cidlink.Link) - if !ok { - return nil, fmt.Errorf("invalid link type for loading: %v", lnk) - } - - blk, err := ng.Get(ctx, cidLink.Cid) - if err != nil { - return nil, err - } - - return bytes.NewReader(blk.RawData()), nil - } -} - -var _ fetcher.Fetcher = (*nodeGetterFetcherSingleUseFactory)(nil) -var _ fetcher.Factory = (*nodeGetterFetcherSingleUseFactory)(nil) - -func rangeStrToByteRange(rangeStr string) (*gateway.ByteRange, error) { - rangeElems := strings.Split(rangeStr, ":") - if len(rangeElems) > 2 { - return nil, fmt.Errorf("invalid range") - } - first, err := strconv.ParseUint(rangeElems[0], 10, 64) - if err != nil { - return nil, err - } - - if rangeElems[1] == "*" { - return &gateway.ByteRange{ - From: first, - To: nil, - }, nil - } - - second, err := strconv.ParseInt(rangeElems[1], 10, 64) - if err != nil { - return nil, err - } - - if second < 0 { - // TODO: fix, might also require a fix in boxo/gateway - return nil, fmt.Errorf("unsupported") - } - - if uint64(second) < first { - return nil, fmt.Errorf("invalid range") - } - - return &gateway.ByteRange{ - From: first, - To: &second, - }, nil -} - -func makeGatewayBlockHandler(bsrv blockservice.BlockService, port int) (*http.Server, error) { - mux := http.NewServeMux() - mux.HandleFunc("/ipfs/", func(w http.ResponseWriter, r *http.Request) { - // the hour is a hard fallback, we don't expect it to happen, but just in case - ctx, cancel := context.WithTimeout(r.Context(), time.Hour) - defer cancel() - r = r.WithContext(ctx) - - defer func() { - if r := recover(); r != nil { - goLog.Error("A panic occurred in the gateway handler!") - goLog.Error(r) - debug.PrintStack() - } - }() - - if r.Method != http.MethodGet { - w.Header().Add("Allow", http.MethodGet) - - errmsg := "Method " + r.Method + " not allowed" - http.Error(w, errmsg, http.StatusMethodNotAllowed) - return - } - - isBlock := false - if formatParam := r.URL.Query().Get("format"); formatParam != "" { - isBlock = formatParam == "raw" - if !isBlock { - http.Error(w, "only raw format supported", http.StatusBadRequest) - return - } - } else { - for _, header := range r.Header.Values("Accept") { - for _, value := range strings.Split(header, ",") { - accept := strings.TrimSpace(value) - if accept == "application/vnd.ipld.raw" { - isBlock = true - break - } - } - } - } - if !isBlock { - http.Error(w, "only raw format supported", http.StatusBadRequest) - return - } - - contentPath := ipath.New(r.URL.Path) - if contentPath.Mutable() { - http.Error(w, "only immutable block requests supported", http.StatusBadRequest) - return - } else if contentPath.Namespace() != "ipfs" { - http.Error(w, "only the ipfs names is supported", http.StatusBadRequest) - return - } - - strComps := strings.Split(strings.TrimRight(contentPath.String(), "/"), "/") - if len(strComps) != 3 { - http.Error(w, "requests must be for single raw blocks", http.StatusBadRequest) - return - } - c, err := cid.Decode(strComps[2]) - if err != nil { - http.Error(w, fmt.Sprintf("not a valid cid %s", strComps[2]), http.StatusBadRequest) - return - } - - blk, err := bsrv.GetBlock(r.Context(), c) - if err != nil { - http.Error(w, fmt.Sprintf("could not get cid %s", c), http.StatusInternalServerError) - return - } - - const immutableCacheControl = "public, max-age=29030400, immutable" - // immutable! CACHE ALL THE THINGS, FOREVER! wolololol - w.Header().Set("Cache-Control", immutableCacheControl) - w.Header().Set("Server", userAgent) - - // Set modtime to 'zero time' to disable Last-Modified header (superseded by Cache-Control) - - http.ServeContent(w, r, c.String()+".bin", noModtime, bytes.NewReader(blk.RawData())) - }) - - // Creates metrics handler for total response size. Matches the same metrics - // from Kubo: - // https://github.com/ipfs/kubo/blob/e550d9e4761ea394357c413c02ade142c0dea88c/core/corehttp/metrics.go#L79-L152 - sum := prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Namespace: "ipfs", - Subsystem: "http", - Name: "response_size_bytes", - Help: "The HTTP response sizes in bytes.", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - }, nil) - err := prometheus.Register(sum) - if err != nil { - return nil, err - } - - // Construct the HTTP handler for the gateway. - handler := promhttp.InstrumentHandlerResponseSize(sum, mux) - - // Add logging - handler = withRequestLogger(handler) - - return &http.Server{ - Handler: handler, - Addr: ":" + strconv.Itoa(port), - }, nil -} diff --git a/test-backend/main.go b/test-backend/main.go deleted file mode 100644 index a61b27f..0000000 --- a/test-backend/main.go +++ /dev/null @@ -1,147 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "log" - "net/http" - "os" - "os/signal" - "sync" - - "github.com/ipfs/bifrost-gateway/lib" - "github.com/ipfs/boxo/bitswap/client" - "github.com/ipfs/boxo/bitswap/network" - "github.com/ipfs/boxo/blockservice" - offline "github.com/ipfs/boxo/exchange/offline" - golog "github.com/ipfs/go-log/v2" - carbs "github.com/ipld/go-car/v2/blockstore" - "github.com/libp2p/go-libp2p" - dht "github.com/libp2p/go-libp2p-kad-dht" - "github.com/libp2p/go-libp2p/core/host" - "github.com/libp2p/go-libp2p/core/routing" - "github.com/spf13/cobra" -) - -var goLog = golog.Logger("test-backend") - -func main() { - if err := rootCmd.Execute(); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } -} - -func init() { - rootCmd.Flags().Int("gateway-port", 8082, "gateway port") - rootCmd.Flags().Int("metrics-port", 8042, "metrics port") - rootCmd.Flags().String("car-blockstore", "", "a CAR file to use for serving data instead of network requests") - golog.SetLogLevel("test-backend", "debug") -} - -var rootCmd = &cobra.Command{ - Use: name, - Version: version, - CompletionOptions: cobra.CompletionOptions{DisableDefaultCmd: true}, - Short: "Test Gateway backend for https://github.com/ipfs/bifrost-gateway (EXPERIMENTAL)", - RunE: func(cmd *cobra.Command, args []string) error { - // Get flags. - gatewayPort, _ := cmd.Flags().GetInt("gateway-port") - metricsPort, _ := cmd.Flags().GetInt("metrics-port") - carbsLocation, _ := cmd.Flags().GetString("car-blockstore") - - var bsrv blockservice.BlockService - if carbsLocation != "" { - bs, err := carbs.OpenReadOnly(carbsLocation) - if err != nil { - return err - } - bsrv = blockservice.New(bs, offline.Exchange(bs)) - } else { - //blockCacheSize, err := getEnvInt(EnvBlockCacheSize, lib.DefaultCacheBlockStoreSize) - //if err != nil { - // return err - //} - blockCacheSize := lib.DefaultCacheBlockStoreSize - - bs, err := lib.NewCacheBlockStore(blockCacheSize) - if err != nil { - return err - } - - var r routing.Routing - h, err := libp2p.New(libp2p.Routing(func(host host.Host) (routing.PeerRouting, error) { - r, err = dht.New(cmd.Context(), host, dht.BootstrapPeersFunc(dht.GetDefaultBootstrapPeerAddrInfos)) - return r, err - })) - if err != nil { - return err - } - n := network.NewFromIpfsHost(h, r) - bsc := client.New(cmd.Context(), n, bs) - n.Start(bsc) - defer n.Stop() - - bsrv = blockservice.New(bs, bsc) - } - - log.Printf("Starting %s %s", name, version) - - var gatewaySrv *http.Server - var err error - - if true { - gatewaySrv, err = makeGatewayCARHandler(bsrv, gatewayPort) - if err != nil { - return err - } - } else { - gatewaySrv, err = makeGatewayBlockHandler(bsrv, gatewayPort) - if err != nil { - return err - } - } - - metricsSrv, err := makeMetricsHandler(metricsPort) - if err != nil { - return err - } - - quit := make(chan os.Signal, 1) - var wg sync.WaitGroup - wg.Add(2) - - go func() { - defer wg.Done() - - //log.Printf("%s: %d", EnvBlockCacheSize, blockCacheSize) - log.Printf("Path gateway listening on http://127.0.0.1:%d", gatewayPort) - log.Printf(" Smoke test (JPG): http://127.0.0.1:%d/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi?format=raw", gatewayPort) - log.Printf("Subdomain gateway configured on dweb.link and http://localhost:%d", gatewayPort) - log.Printf(" Smoke test (Subdomain+DNSLink+UnixFS+HAMT): http://localhost:%d/ipns/en.wikipedia-on-ipfs.org/wiki/", gatewayPort) - err := gatewaySrv.ListenAndServe() - if err != nil && !errors.Is(err, http.ErrServerClosed) { - log.Printf("Failed to start gateway: %s", err) - quit <- os.Interrupt - } - }() - - go func() { - defer wg.Done() - log.Printf("Metrics exposed at http://127.0.0.1:%d/debug/metrics/prometheus", metricsPort) - err := metricsSrv.ListenAndServe() - if err != nil && !errors.Is(err, http.ErrServerClosed) { - log.Printf("Failed to start metrics: %s", err) - quit <- os.Interrupt - } - }() - - signal.Notify(quit, os.Interrupt) - <-quit - log.Printf("Closing servers...") - go gatewaySrv.Close() - go metricsSrv.Close() - wg.Wait() - return nil - }, -} diff --git a/test-backend/version.go b/test-backend/version.go deleted file mode 100644 index 318d7f5..0000000 --- a/test-backend/version.go +++ /dev/null @@ -1,39 +0,0 @@ -package main - -import ( - "runtime/debug" - "time" -) - -var name = "test-backend" -var version = buildVersion() -var userAgent = name + "/" + version - -func buildVersion() string { - var revision string - var day string - var dirty bool - - info, ok := debug.ReadBuildInfo() - if !ok { - return "dev-build" - } - for _, kv := range info.Settings { - switch kv.Key { - case "vcs.revision": - revision = kv.Value[:7] - case "vcs.time": - t, _ := time.Parse(time.RFC3339, kv.Value) - day = t.UTC().Format("2006-01-02") - case "vcs.modified": - dirty = kv.Value == "true" - } - } - if dirty { - revision += "-dirty" - } - if revision != "" { - return day + "-" + revision - } - return "dev-build" -}