diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index 73ee255557b..2967c999721 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -70,27 +70,9 @@ jobs: restore-keys: ${{ runner.os }}-${{ github.job }}-helia- - run: sudo apt update - run: sudo apt install -y libxkbcommon0 libxdamage1 libgbm1 libpango-1.0-0 libcairo2 # dependencies for playwright - - # TODO: for now run against version from https://github.com/ipfs/helia/pull/584 - - name: Checkout helia-interop with provisional fix - uses: actions/checkout@v4 - with: - repository: ipfs/helia - ref: ab6b385787075ad9932f24362293b3bb82ff1d96 - path: helia - - name: Run provisional build of helia - run: npm i && npm run build - working-directory: helia - - name: Run provisional build of helia-interop - run: npm i && npm run build && npx . - working-directory: helia/packages/interop + - run: npx --package @helia/interop helia-interop env: KUBO_BINARY: ${{ github.workspace }}/cmd/ipfs/ipfs - - # TODO: switch back to release once https://github.com/ipfs/helia/pull/584 ships - #- run: npx --package @helia/interop helia-interop - # env: - # KUBO_BINARY: ${{ github.workspace }}/cmd/ipfs/ipfs ipfs-webui: needs: [interop-prep] runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 93901ba1efb..fa40e162580 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ # Kubo Changelogs +- [v0.31](docs/changelogs/v0.31.md) - [v0.30](docs/changelogs/v0.30.md) - [v0.29](docs/changelogs/v0.29.md) - [v0.28](docs/changelogs/v0.28.md) diff --git a/cmd/ipfs/kubo/daemon.go b/cmd/ipfs/kubo/daemon.go index eb9fff8e448..0e5637cdb3b 100644 --- a/cmd/ipfs/kubo/daemon.go +++ b/cmd/ipfs/kubo/daemon.go @@ -586,7 +586,7 @@ take effect. prometheus.MustRegister(&corehttp.IpfsNodeCollector{Node: node}) // start MFS pinning thread - startPinMFS(daemonConfigPollInterval, cctx, &ipfsPinMFSNode{node}) + startPinMFS(cctx, daemonConfigPollInterval, &ipfsPinMFSNode{node}) // The daemon is *finally* ready. fmt.Printf("Daemon is ready\n") diff --git a/cmd/ipfs/kubo/pinmfs.go b/cmd/ipfs/kubo/pinmfs.go index 846ee8a77af..c9187145ce3 100644 --- a/cmd/ipfs/kubo/pinmfs.go +++ b/cmd/ipfs/kubo/pinmfs.go @@ -12,7 +12,7 @@ import ( pinclient "github.com/ipfs/boxo/pinning/remote/client" cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" - logging "github.com/ipfs/go-log" + logging "github.com/ipfs/go-log/v2" config "github.com/ipfs/kubo/config" "github.com/ipfs/kubo/core" @@ -40,6 +40,7 @@ func init() { d, err := time.ParseDuration(pollDurStr) if err != nil { mfslog.Error("error parsing MFS_PIN_POLL_INTERVAL, using default:", err) + return } daemonConfigPollInterval = d } @@ -74,56 +75,28 @@ func (x *ipfsPinMFSNode) PeerHost() host.Host { return x.node.PeerHost } -func startPinMFS(configPollInterval time.Duration, cctx pinMFSContext, node pinMFSNode) { - errCh := make(chan error) - go pinMFSOnChange(configPollInterval, cctx, node, errCh) - go func() { - for { - select { - case err, isOpen := <-errCh: - if !isOpen { - return - } - mfslog.Errorf("%v", err) - case <-cctx.Context().Done(): - return - } - } - }() +func startPinMFS(cctx pinMFSContext, configPollInterval time.Duration, node pinMFSNode) { + go pinMFSOnChange(cctx, configPollInterval, node) } -func pinMFSOnChange(configPollInterval time.Duration, cctx pinMFSContext, node pinMFSNode, errCh chan<- error) { - defer close(errCh) - - var tmo *time.Timer - defer func() { - if tmo != nil { - tmo.Stop() - } - }() +func pinMFSOnChange(cctx pinMFSContext, configPollInterval time.Duration, node pinMFSNode) { + tmo := time.NewTimer(configPollInterval) + defer tmo.Stop() lastPins := map[string]lastPin{} for { // polling sleep - if tmo == nil { - tmo = time.NewTimer(configPollInterval) - } else { - tmo.Reset(configPollInterval) - } select { case <-cctx.Context().Done(): return case <-tmo.C: + tmo.Reset(configPollInterval) } // reread the config, which may have changed in the meantime cfg, err := cctx.GetConfig() if err != nil { - select { - case errCh <- fmt.Errorf("pinning reading config (%v)", err): - case <-cctx.Context().Done(): - return - } + mfslog.Errorf("pinning reading config (%v)", err) continue } mfslog.Debugf("pinning loop is awake, %d remote services", len(cfg.Pinning.RemoteServices)) @@ -131,30 +104,29 @@ func pinMFSOnChange(configPollInterval time.Duration, cctx pinMFSContext, node p // get the most recent MFS root cid rootNode, err := node.RootNode() if err != nil { - select { - case errCh <- fmt.Errorf("pinning reading MFS root (%v)", err): - case <-cctx.Context().Done(): - return - } + mfslog.Errorf("pinning reading MFS root (%v)", err) continue } - rootCid := rootNode.Cid() // pin to all remote services in parallel - pinAllMFS(cctx.Context(), node, cfg, rootCid, lastPins, errCh) + pinAllMFS(cctx.Context(), node, cfg, rootNode.Cid(), lastPins) } } // pinAllMFS pins on all remote services in parallel to overcome DoS attacks. -func pinAllMFS(ctx context.Context, node pinMFSNode, cfg *config.Config, rootCid cid.Cid, lastPins map[string]lastPin, errCh chan<- error) { - ch := make(chan lastPin, len(cfg.Pinning.RemoteServices)) - for svcName_, svcConfig_ := range cfg.Pinning.RemoteServices { +func pinAllMFS(ctx context.Context, node pinMFSNode, cfg *config.Config, rootCid cid.Cid, lastPins map[string]lastPin) { + ch := make(chan lastPin) + var started int + + for svcName, svcConfig := range cfg.Pinning.RemoteServices { + if ctx.Err() != nil { + break + } + // skip services where MFS is not enabled - svcName, svcConfig := svcName_, svcConfig_ mfslog.Debugf("pinning MFS root considering service %q", svcName) if !svcConfig.Policies.MFS.Enable { mfslog.Debugf("pinning service %q is not enabled", svcName) - ch <- lastPin{} continue } // read mfs pin interval for this service @@ -165,11 +137,7 @@ func pinAllMFS(ctx context.Context, node pinMFSNode, cfg *config.Config, rootCid var err error repinInterval, err = time.ParseDuration(svcConfig.Policies.MFS.RepinInterval) if err != nil { - select { - case errCh <- fmt.Errorf("remote pinning service %q has invalid MFS.RepinInterval (%v)", svcName, err): - case <-ctx.Done(): - } - ch <- lastPin{} + mfslog.Errorf("remote pinning service %q has invalid MFS.RepinInterval (%v)", svcName, err) continue } } @@ -182,38 +150,30 @@ func pinAllMFS(ctx context.Context, node pinMFSNode, cfg *config.Config, rootCid } else { mfslog.Debugf("pinning MFS root to %q: skipped due to MFS.RepinInterval=%s (remaining: %s)", svcName, repinInterval.String(), (repinInterval - time.Since(last.Time)).String()) } - ch <- lastPin{} continue } } mfslog.Debugf("pinning MFS root %q to %q", rootCid, svcName) - go func() { - if r, err := pinMFS(ctx, node, rootCid, svcName, svcConfig); err != nil { - select { - case errCh <- fmt.Errorf("pinning MFS root %q to %q (%v)", rootCid, svcName, err): - case <-ctx.Done(): - } - ch <- lastPin{} - } else { - ch <- r + go func(svcName string, svcConfig config.RemotePinningService) { + r, err := pinMFS(ctx, node, rootCid, svcName, svcConfig) + if err != nil { + mfslog.Errorf("pinning MFS root %q to %q (%v)", rootCid, svcName, err) } - }() + ch <- r + }(svcName, svcConfig) + started++ } - for i := 0; i < len(cfg.Pinning.RemoteServices); i++ { + + // Collect results from all started goroutines. + for i := 0; i < started; i++ { if x := <-ch; x.IsValid() { lastPins[x.ServiceName] = x } } } -func pinMFS( - ctx context.Context, - node pinMFSNode, - cid cid.Cid, - svcName string, - svcConfig config.RemotePinningService, -) (lastPin, error) { +func pinMFS(ctx context.Context, node pinMFSNode, cid cid.Cid, svcName string, svcConfig config.RemotePinningService) (lastPin, error) { c := pinclient.NewClient(svcConfig.API.Endpoint, svcConfig.API.Key) pinName := svcConfig.Policies.MFS.PinName @@ -243,43 +203,46 @@ func pinMFS( } for range lsPinCh { // in case the prior loop exits early } - if err := <-lsErrCh; err != nil { + err := <-lsErrCh + if err != nil { return lastPin{}, fmt.Errorf("error while listing remote pins: %v", err) } - // CID of the current MFS root is already being pinned, nothing to do - if pinning { - mfslog.Debugf("pinning MFS to %q: pin for %q exists since %s, skipping", svcName, cid, pinTime.String()) - return lastPin{Time: pinTime, ServiceName: svcName, ServiceConfig: svcConfig, CID: cid}, nil - } - - // Prepare Pin.name - addOpts := []pinclient.AddOption{pinclient.PinOpts.WithName(pinName)} + if !pinning { + // Prepare Pin.name + addOpts := []pinclient.AddOption{pinclient.PinOpts.WithName(pinName)} - // Prepare Pin.origins - // Add own multiaddrs to the 'origins' array, so Pinning Service can - // use that as a hint and connect back to us (if possible) - if node.PeerHost() != nil { - addrs, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(node.PeerHost())) - if err != nil { - return lastPin{}, err + // Prepare Pin.origins + // Add own multiaddrs to the 'origins' array, so Pinning Service can + // use that as a hint and connect back to us (if possible) + if node.PeerHost() != nil { + addrs, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(node.PeerHost())) + if err != nil { + return lastPin{}, err + } + addOpts = append(addOpts, pinclient.PinOpts.WithOrigins(addrs...)) } - addOpts = append(addOpts, pinclient.PinOpts.WithOrigins(addrs...)) - } - // Create or replace pin for MFS root - if existingRequestID != "" { - mfslog.Debugf("pinning to %q: replacing existing MFS root pin with %q", svcName, cid) - _, err := c.Replace(ctx, existingRequestID, cid, addOpts...) - if err != nil { - return lastPin{}, err + // Create or replace pin for MFS root + if existingRequestID != "" { + mfslog.Debugf("pinning to %q: replacing existing MFS root pin with %q", svcName, cid) + if _, err = c.Replace(ctx, existingRequestID, cid, addOpts...); err != nil { + return lastPin{}, err + } + } else { + mfslog.Debugf("pinning to %q: creating a new MFS root pin for %q", svcName, cid) + if _, err = c.Add(ctx, cid, addOpts...); err != nil { + return lastPin{}, err + } } } else { - mfslog.Debugf("pinning to %q: creating a new MFS root pin for %q", svcName, cid) - _, err := c.Add(ctx, cid, addOpts...) - if err != nil { - return lastPin{}, err - } + mfslog.Debugf("pinning MFS to %q: pin for %q exists since %s, skipping", svcName, cid, pinTime.String()) } - return lastPin{Time: pinTime, ServiceName: svcName, ServiceConfig: svcConfig, CID: cid}, nil + + return lastPin{ + Time: pinTime, + ServiceName: svcName, + ServiceConfig: svcConfig, + CID: cid, + }, nil } diff --git a/cmd/ipfs/kubo/pinmfs_test.go b/cmd/ipfs/kubo/pinmfs_test.go index da71d362cdb..750be9c987d 100644 --- a/cmd/ipfs/kubo/pinmfs_test.go +++ b/cmd/ipfs/kubo/pinmfs_test.go @@ -1,14 +1,19 @@ package kubo import ( + "bufio" "context" + "encoding/json" + "errors" "fmt" + "io" "strings" "testing" "time" merkledag "github.com/ipfs/boxo/ipld/merkledag" ipld "github.com/ipfs/go-ipld-format" + logging "github.com/ipfs/go-log/v2" config "github.com/ipfs/kubo/config" "github.com/libp2p/go-libp2p/core/host" peer "github.com/libp2p/go-libp2p/core/peer" @@ -60,25 +65,37 @@ func isErrorSimilar(e1, e2 error) bool { } func TestPinMFSConfigError(t *testing.T) { - ctx := &testPinMFSContext{ - ctx: context.Background(), + ctx, cancel := context.WithTimeout(context.Background(), 2*testConfigPollInterval) + defer cancel() + + cctx := &testPinMFSContext{ + ctx: ctx, cfg: nil, err: fmt.Errorf("couldn't read config"), } node := &testPinMFSNode{} - errCh := make(chan error) - go pinMFSOnChange(testConfigPollInterval, ctx, node, errCh) - if !isErrorSimilar(<-errCh, ctx.err) { - t.Errorf("error did not propagate") + + logReader := logging.NewPipeReader() + go func() { + pinMFSOnChange(cctx, testConfigPollInterval, node) + logReader.Close() + }() + + level, msg := readLogLine(t, logReader) + if level != "error" { + t.Error("expected error to be logged") } - if !isErrorSimilar(<-errCh, ctx.err) { + if !isErrorSimilar(errors.New(msg), cctx.err) { t.Errorf("error did not propagate") } } func TestPinMFSRootNodeError(t *testing.T) { - ctx := &testPinMFSContext{ - ctx: context.Background(), + ctx, cancel := context.WithTimeout(context.Background(), 2*testConfigPollInterval) + defer cancel() + + cctx := &testPinMFSContext{ + ctx: ctx, cfg: &config.Config{ Pinning: config.Pinning{}, }, @@ -87,12 +104,16 @@ func TestPinMFSRootNodeError(t *testing.T) { node := &testPinMFSNode{ err: fmt.Errorf("cannot create root node"), } - errCh := make(chan error) - go pinMFSOnChange(testConfigPollInterval, ctx, node, errCh) - if !isErrorSimilar(<-errCh, node.err) { - t.Errorf("error did not propagate") + logReader := logging.NewPipeReader() + go func() { + pinMFSOnChange(cctx, testConfigPollInterval, node) + logReader.Close() + }() + level, msg := readLogLine(t, logReader) + if level != "error" { + t.Error("expected error to be logged") } - if !isErrorSimilar(<-errCh, node.err) { + if !isErrorSimilar(errors.New(msg), node.err) { t.Errorf("error did not propagate") } } @@ -155,7 +176,8 @@ func TestPinMFSService(t *testing.T) { } func testPinMFSServiceWithError(t *testing.T, cfg *config.Config, expectedErrorPrefix string) { - goctx, cancel := context.WithCancel(context.Background()) + goctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() ctx := &testPinMFSContext{ ctx: goctx, cfg: cfg, @@ -164,16 +186,36 @@ func testPinMFSServiceWithError(t *testing.T, cfg *config.Config, expectedErrorP node := &testPinMFSNode{ err: nil, } - errCh := make(chan error) - go pinMFSOnChange(testConfigPollInterval, ctx, node, errCh) - defer cancel() - // first pass through the pinning loop - err := <-errCh - if !strings.Contains((err).Error(), expectedErrorPrefix) { - t.Errorf("expecting error containing %q", expectedErrorPrefix) + logReader := logging.NewPipeReader() + go func() { + pinMFSOnChange(ctx, testConfigPollInterval, node) + logReader.Close() + }() + level, msg := readLogLine(t, logReader) + if level != "error" { + t.Error("expected error to be logged") } - // second pass through the pinning loop - if !strings.Contains((err).Error(), expectedErrorPrefix) { + if !strings.Contains(msg, expectedErrorPrefix) { t.Errorf("expecting error containing %q", expectedErrorPrefix) } } + +func readLogLine(t *testing.T, logReader io.Reader) (string, string) { + t.Helper() + + r := bufio.NewReader(logReader) + data, err := r.ReadBytes('\n') + if err != nil { + t.Fatal(err) + } + + logInfo := struct { + Level string `json:"level"` + Msg string `json:"msg"` + }{} + err = json.Unmarshal(data, &logInfo) + if err != nil { + t.Fatal(err) + } + return logInfo.Level, logInfo.Msg +} diff --git a/config/profile.go b/config/profile.go index c73b05af27c..63e76d8ec1b 100644 --- a/config/profile.go +++ b/config/profile.go @@ -124,7 +124,7 @@ This profile may only be applied when first initializing the node. "flatfs": { Description: `Configures the node to use the flatfs datastore. -This is the most battle-tested and reliable datastore. +This is the most battle-tested and reliable datastore. You should use this datastore if: * You need a very simple and very reliable datastore, and you trust your @@ -145,21 +145,20 @@ This profile may only be applied when first initializing the node. }, }, "badgerds": { - Description: `Configures the node to use the experimental badger datastore. + Description: `Configures the node to use the legacy badgerv1 datastore. -Use this datastore if some aspects of performance, -especially the speed of adding many gigabytes of files, are critical. -However, be aware that: +NOTE: this is badger 1.x, which has known bugs and is no longer supported by the upstream team. +It is provided here only for pre-existing users, allowing them to migrate away to more modern datastore. + +Other caveats: * This datastore will not properly reclaim space when your datastore is smaller than several gigabytes. If you run IPFS with --enable-gc, you plan on storing very little data in your IPFS node, and disk usage is more critical than performance, consider using flatfs. -* This datastore uses up to several gigabytes of memory. +* This datastore uses up to several gigabytes of memory. * Good for medium-size datastores, but may run into performance issues if your dataset is bigger than a terabyte. -* The current implementation is based on old badger 1.x - which is no longer supported by the upstream team. This profile may only be applied when first initializing the node.`, diff --git a/core/commands/pin/remotepin.go b/core/commands/pin/remotepin.go index 132532554cc..3721913e77c 100644 --- a/core/commands/pin/remotepin.go +++ b/core/commands/pin/remotepin.go @@ -221,6 +221,8 @@ NOTE: a comma-separated notation is supported in CLI for convenience: // Block unless --background=true is passed if !req.Options[pinBackgroundOptionName].(bool) { + const pinWaitTime = 500 * time.Millisecond + var timer *time.Timer requestID := ps.GetRequestId() for { ps, err = c.GetStatusByID(ctx, requestID) @@ -237,10 +239,15 @@ NOTE: a comma-separated notation is supported in CLI for convenience: if s == pinclient.StatusFailed { return fmt.Errorf("remote service failed to pin requestid=%q", requestID) } - tmr := time.NewTimer(time.Second / 2) + if timer == nil { + timer = time.NewTimer(pinWaitTime) + } else { + timer.Reset(pinWaitTime) + } select { - case <-tmr.C: + case <-timer.C: case <-ctx.Done(): + timer.Stop() return fmt.Errorf("waiting for pin interrupted, requestid=%q remains on remote service", requestID) } } diff --git a/core/corehttp/webui.go b/core/corehttp/webui.go index 82520e30da2..2117d78afb5 100644 --- a/core/corehttp/webui.go +++ b/core/corehttp/webui.go @@ -1,11 +1,12 @@ package corehttp // WebUI version confirmed to work with this Kubo version -const WebUIPath = "/ipfs/bafybeihatzsgposbr3hrngo42yckdyqcc56yean2rynnwpzxstvdlphxf4" // v4.3.0 +const WebUIPath = "/ipfs/bafybeif6abowqcavbkz243biyh7pde7ick5kkwwytrh7pd2hkbtuqysjxy" // v4.3.2 // WebUIPaths is a list of all past webUI paths. var WebUIPaths = []string{ WebUIPath, + "/ipfs/bafybeihatzsgposbr3hrngo42yckdyqcc56yean2rynnwpzxstvdlphxf4", "/ipfs/bafybeigggyffcf6yfhx5irtwzx3cgnk6n3dwylkvcpckzhqqrigsxowjwe", "/ipfs/bafybeidf7cpkwsjkq6xs3r6fbbxghbugilx3jtezbza7gua3k5wjixpmba", "/ipfs/bafybeiamycmd52xvg6k3nzr6z3n33de6a2teyhquhj4kspdtnvetnkrfim", diff --git a/docs/EARLY_TESTERS.md b/docs/EARLY_TESTERS.md index 6c5b09b1585..e3280b0eb16 100644 --- a/docs/EARLY_TESTERS.md +++ b/docs/EARLY_TESTERS.md @@ -27,7 +27,7 @@ We will ask early testers to participate at two points in the process: - [ ] Infura (@MichaelMure) - [ ] OrbitDB (@haydenyoung) - [ ] Pinata (@obo20) -- [ ] PL EngRes bifrost (@cewood ns4plabs) +- [ ] Shipyard (@cewood, @ns4plabs) - [ ] Siderus (@koalalorenzo) - [ ] Textile (@sanderpick) - [ ] @RubenKelevra diff --git a/docs/RELEASE_CHECKLIST.md b/docs/RELEASE_CHECKLIST.md index 655329e1a7f..476c77b15f3 100644 --- a/docs/RELEASE_CHECKLIST.md +++ b/docs/RELEASE_CHECKLIST.md @@ -1,4 +1,4 @@ - + # ✅ Release Checklist (vX.Y.Z[-rcN]) @@ -62,7 +62,7 @@ This section covers tasks to be done during each release. - [example](https://github.com/ipfs/kubo/pull/9306) - [ ] Cherry-pick commits from `master` to the `release-vX.Y.Z` using `git cherry-pick -x ` - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) Add full changelog and contributors to the [changelog](docs/changelogs/vX.Y.md) - - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) Replace the `Changelog` and `Contributors` sections of the [changelog](docs/changelogs/vX.Y.md) with the stdout of `./bin/mkreleaselog` + - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) Replace the `Changelog` and `Contributors` sections of the [changelog](docs/changelogs/vX.Y.md) with the stdout of `./bin/mkreleaselog`. Note that the command expects your `$GOPATH/src/github.com/ipfs/kubo` to include latest commits from `release-vX.Y` - do **NOT** copy the stderr - [ ] verify all CI checks on the PR from `release-vX.Y` to `release` are passing - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) Merge the PR from `release-vX.Y` to `release` using the `Create a merge commit` @@ -77,40 +77,42 @@ This section covers tasks to be done during each release. - [ ] ⚠️ push the tag to GitHub using `git push origin vX.Y.Z(-RCN)` - do **NOT** use `git push --tags` because it pushes all your local tags -- [ ] Publish the release to [DockerHub](https://hub.docker.com/r/ipfs/kubo/)
using `./kuboreleaser --skip-check-before --skip-run release --version vX.Y.Z(-rcN) publish-to-dockerhub` or ... - - [ ] Wait for [Publish docker image](https://github.com/ipfs/kubo/actions/workflows/docker-image.yml) workflow run initiated by the tag push to finish - - [ ] verify the image is available on [Docker Hub](https://hub.docker.com/r/ipfs/kubo/tags) - [ ] Verify [ipfs/distributions](https://github.com/ipfs/distributions)'s `.tool-versions`'s `golang` entry is set to the [latest go release](https://go.dev/doc/devel/release) on the major go branch [Kubo is being tested on](https://github.com/ipfs/kubo/blob/master/.github/workflows/gotest.yml) (see `go-version:`). -- [ ] Publish the release to [dist.ipfs.tech](https://dist.ipfs.tech)
using `./kuboreleaser release --version vX.Y.Z(-rcN) publish-to-distributions` or ... - - [ ] check out [ipfs/distributions](https://github.com/ipfs/distributions) - - [ ] run `./dist.sh add-version kubo vX.Y.Z(-RCN)` to add the new version to the `versions` file - - [usage](https://github.com/ipfs/distributions#usage) - - [ ] create and merge the PR which updates `dists/kubo/versions` and `dists/go-ipfs/versions` (![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) and `dists/kubo/current_version` and `dists/go-ipfs/current_version`) - - [example](https://github.com/ipfs/distributions/pull/760) - - [ ] wait for the [CI](https://github.com/ipfs/distributions/actions/workflows/main.yml) workflow run initiated by the merge to master to finish - - [ ] verify the release is available on [dist.ipfs.tech](https://dist.ipfs.tech/#kubo) -
-- [ ] Publish the release to [NPM](https://www.npmjs.com/package/go-ipfs?activeTab=versions)
using `./kuboreleaser release --version vX.Y.Z(-rcN) publish-to-npm` (⚠️ you might need to run the command a couple of times because GHA might not be able to see the new distribution straight away due to caching) or ... - - [ ] run the [Release to npm](https://github.com/ipfs/npm-go-ipfs/actions/workflows/main.yml) workflow - - [ ] check [Release to npm](https://github.com/ipfs/npm-go-ipfs/actions/workflows/main.yml) workflow run logs to verify it discovered the new release - - [ ] verify the release is available on [NPM](https://www.npmjs.com/package/go-ipfs?activeTab=versions) -
-- [ ] Publish the release to [GitHub](https://github.com/ipfs/kubo/releases)
using `./kuboreleaser release --version vX.Y.Z(-rcN) publish-to-github` or ... - - [ ] create a new release on [GitHub](https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository#creating-a-release) - - [RC example](https://github.com/ipfs/kubo/releases/tag/v0.17.0-rc1) - - [FINAL example](https://github.com/ipfs/kubo/releases/tag/v0.17.0) - - [ ] use the `vX.Y.Z(-RCN)` tag - - [ ] link to the release issue - - [ ] ![](https://img.shields.io/badge/only-RC-blue?style=flat-square) link to the changelog in the description - - [ ] ![](https://img.shields.io/badge/only-RC-blue?style=flat-square) check the `This is a pre-release` checkbox - - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) copy the changelog (without the header) in the description - - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) do **NOT** check the `This is a pre-release` checkbox - - [ ] run the [sync-release-assets](https://github.com/ipfs/kubo/actions/workflows/sync-release-assets.yml) workflow - - [ ] wait for the [sync-release-assets](https://github.com/ipfs/kubo/actions/workflows/sync-release-assets.yml) workflow run to finish - - [ ] verify the release assets are present in the [GitHub release](https://github.com/ipfs/kubo/releases/tag/vX.Y.Z(-RCN)) -
-- [ ] Run Thunderdome testing, see the [Thunderdome release docs](./releases_thunderdome.md) for details - - [ ] create a PR and merge the experiment config into Thunderdome +- [ ] Publish to Dockerhub, NPM, and dist.ipfs.tech and GitHub using `./kuboreleaser --skip-check-before --skip-run release --version vX.Y.Z(-rcN) publish-to-all` or follow each step below: + - [ ] Publish the release to [DockerHub](https://hub.docker.com/r/ipfs/kubo/)
using `./kuboreleaser --skip-check-before --skip-run release --version vX.Y.Z(-rcN) publish-to-dockerhub` or ... + - [ ] Wait for [Publish docker image](https://github.com/ipfs/kubo/actions/workflows/docker-image.yml) workflow run initiated by the tag push to finish + - [ ] verify the image is available on [Docker Hub](https://hub.docker.com/r/ipfs/kubo/tags) + - [ ] Publish the release to [dist.ipfs.tech](https://dist.ipfs.tech)
using `./kuboreleaser release --version vX.Y.Z(-rcN) publish-to-distributions` or ... + - [ ] check out [ipfs/distributions](https://github.com/ipfs/distributions) + - [ ] run `./dist.sh add-version kubo vX.Y.Z(-RCN)` to add the new version to the `versions` file + - [usage](https://github.com/ipfs/distributions#usage) + - [ ] create and merge the PR which updates `dists/kubo/versions` and `dists/go-ipfs/versions` (![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) and `dists/kubo/current_version` and `dists/go-ipfs/current_version`) + - [example](https://github.com/ipfs/distributions/pull/760) + - [ ] wait for the [CI](https://github.com/ipfs/distributions/actions/workflows/main.yml) workflow run initiated by the merge to master to finish + - [ ] verify the release is available on [dist.ipfs.tech](https://dist.ipfs.tech/#kubo) +
+ - [ ] Publish the release to [NPM](https://www.npmjs.com/package/go-ipfs?activeTab=versions)
using `./kuboreleaser release --version vX.Y.Z(-rcN) publish-to-npm` (⚠️ you might need to run the command a couple of times because GHA might not be able to see the new distribution straight away due to caching) or ... + - [ ] run the [Release to npm](https://github.com/ipfs/npm-go-ipfs/actions/workflows/main.yml) workflow + - [ ] check [Release to npm](https://github.com/ipfs/npm-go-ipfs/actions/workflows/main.yml) workflow run logs to verify it discovered the new release + - [ ] verify the release is available on [NPM](https://www.npmjs.com/package/go-ipfs?activeTab=versions) +
+ - [ ] Publish the release to [GitHub](https://github.com/ipfs/kubo/releases)
using `./kuboreleaser release --version vX.Y.Z(-rcN) publish-to-github` or ... + - [ ] create a new release on [GitHub](https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository#creating-a-release) + - [RC example](https://github.com/ipfs/kubo/releases/tag/v0.17.0-rc1) + - [FINAL example](https://github.com/ipfs/kubo/releases/tag/v0.17.0) + - [ ] use the `vX.Y.Z(-RCN)` tag + - [ ] link to the release issue + - [ ] ![](https://img.shields.io/badge/only-RC-blue?style=flat-square) link to the changelog in the description + - [ ] ![](https://img.shields.io/badge/only-RC-blue?style=flat-square) check the `This is a pre-release` checkbox + - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) copy the changelog (without the header) in the description + - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) do **NOT** check the `This is a pre-release` checkbox + - [ ] run the [sync-release-assets](https://github.com/ipfs/kubo/actions/workflows/sync-release-assets.yml) workflow + - [ ] wait for the [sync-release-assets](https://github.com/ipfs/kubo/actions/workflows/sync-release-assets.yml) workflow run to finish + - [ ] verify the release assets are present in the [GitHub release](https://github.com/ipfs/kubo/releases/tag/vX.Y.Z(-RCN)) +
+- [ ] Update Kubo staging environment, see the [Running Kubo tests on staging](https://www.notion.so/Running-Kubo-tests-on-staging-488578bb46154f9bad982e4205621af8) for details. + - [ ] ![](https://img.shields.io/badge/only-RC-blue?style=flat-square) Test last release against the current RC + - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) Test last release against the current one - [ ] Promote the release
using `./kuboreleaser release --version vX.Y.Z(-rcN) promote` or ... - [ ] create an [IPFS Discourse](https://discuss.ipfs.tech) topic - [prerelease example](https://discuss.ipfs.tech/t/kubo-v0-16-0-rc1-release-candidate-is-out/15248) @@ -151,10 +153,6 @@ This section covers tasks to be done during each release. - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) run the [update-on-new-ipfs-tag.yml](https://github.com/ipfs/ipfs-docs/actions/workflows/update-on-new-ipfs-tag.yml) workflow - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) merge the PR created by the [update-on-new-ipfs-tag.yml](https://github.com/ipfs/ipfs-docs/actions/workflows/update-on-new-ipfs-tag.yml) workflow run
-- [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) Ask Brave to update Kubo in Brave Desktop - - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) use [this link](https://github.com/brave/brave-browser/issues/new?assignees=&labels=OS%2FDesktop&projects=&template=desktop.md&title=) to create an issue for the new Kubo version - - [basic example](https://github.com/brave/brave-browser/issues/31453), [example with additional notes](https://github.com/brave/brave-browser/issues/27965) - - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) post link to the issue in `#shared-pl-brave` for visibility - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) Create a blog entry on [blog.ipfs.tech](https://blog.ipfs.tech)
using `./kuboreleaser release --version vX.Y.Z(-rcN) update-ipfs-blog --date YYYY-MM-DD` or ... - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) create a PR which adds a release note for the new Kubo version - [example](https://github.com/ipfs/ipfs-blog/pull/529) @@ -172,7 +170,7 @@ This section covers tasks to be done during each release.
- [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) ![](https://img.shields.io/badge/not-PATCH-yellow?style=flat-square) Create a dependency update PR - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) ![](https://img.shields.io/badge/not-PATCH-yellow?style=flat-square) check out [ipfs/kubo](https://github.com/ipfs/kubo) - - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) ![](https://img.shields.io/badge/not-PATCH-yellow?style=flat-square) run `go get -u` in root directory + - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) ![](https://img.shields.io/badge/not-PATCH-yellow?style=flat-square) go over direct dependencies from `go.mod` in the root directory (NOTE: do not run `go get -u` as it will upgrade indirect dependencies which may cause problems) - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) ![](https://img.shields.io/badge/not-PATCH-yellow?style=flat-square) run `make mod_tidy` - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) ![](https://img.shields.io/badge/not-PATCH-yellow?style=flat-square) create a PR which updates `go.mod` and `go.sum` - [ ] ![](https://img.shields.io/badge/only-FINAL-green?style=flat-square) ![](https://img.shields.io/badge/not-PATCH-yellow?style=flat-square) add the PR to the next release milestone diff --git a/docs/RELEASE_ISSUE_TEMPLATE.md b/docs/RELEASE_ISSUE_TEMPLATE.md index cfa97594325..321026ea6c0 100644 --- a/docs/RELEASE_ISSUE_TEMPLATE.md +++ b/docs/RELEASE_ISSUE_TEMPLATE.md @@ -1,4 +1,4 @@ - + # Items to do upon creating the release issue @@ -22,7 +22,6 @@ * Expected RC date: week of YYYY-MM-DD * 🚢 Expected final release date: YYYY-MM-DD * Release PR: -* Thunderdome PR: * Accompanying PR for improving the release process: ([example](https://github.com/ipfs/kubo/pull/9391)) * Changelog: https://github.com/ipfs/kubo/blob/master/docs/changelogs/vX.Y.md diff --git a/docs/changelogs/v0.30.md b/docs/changelogs/v0.30.md index c5073049c86..36c3a5c754f 100644 --- a/docs/changelogs/v0.30.md +++ b/docs/changelogs/v0.30.md @@ -9,12 +9,12 @@ - [Improved P2P connectivity](#improved-p2p-connectivity) - [Refactored Bitswap and dag-pb chunker](#refactored-bitswap-and-dag-pb-chunker) - [WebRTC-Direct Transport enabled by default](#webrtc-direct-transport-enabled-by-default) + - [UnixFS 1.5: Mode and Modification Time Support](#unixfs-15-mode-and-modification-time-support) - [AutoNAT V2 Service Introduced Alongside V1](#autonat-v2-service-introduced-alongside-v1) - [Automated `ipfs version check`](#automated-ipfs-version-check) - [Version Suffix Configuration](#version-suffix-configuration) - [`/unix/` socket support in `Addresses.API`](#unix-socket-support-in-addressesapi) - [Cleaned Up `ipfs daemon` Startup Log](#cleaned-up-ipfs-daemon-startup-log) - - [UnixFS 1.5: Mode and Modification Time Support](#unixfs-15-mode-and-modification-time-support) - [Commands Preserve Specified Hostname](#commands-preserve-specified-hostname) - [📝 Changelog](#-changelog) - [👨‍👩‍👧‍👦 Contributors](#-contributors) @@ -30,9 +30,9 @@ This release took longer and is more packed with fixes and features than usual. #### Improved P2P connectivity -This release comes with significant go-libp2p update from v0.34.1 to v0.36.3 ([release notes](https://github.com/ipfs/boxo/releases/)). +This release comes with significant go-libp2p update from v0.34.1 to v0.36.3 ([release notes](https://github.com/libp2p/go-libp2p/releases/)). -It includes multiple fixes to key protocols: QUIC/Webtransport/WebRTC, Connection Upgrades through Relay ([DCUtR](https://github.com/libp2p/specs/blob/master/relay/DCUtR.md)), and Secure WebSockets. +It includes multiple fixes to key protocols: [QUIC](https://github.com/libp2p/specs/tree/master/quic)/[Webtransport](https://github.com/libp2p/specs/tree/master/webtransport)/[WebRTC](https://github.com/libp2p/specs/tree/master/webrtc), Connection Upgrades through Relay ([DCUtR](https://github.com/libp2p/specs/blob/master/relay/DCUtR.md)), and [Secure WebSockets](https://github.com/libp2p/specs/pull/624). Also, peers that are behind certain types of NAT will now be more reachable. For this alone, Kubo users are highly encouraged to upgrade. @@ -45,7 +45,7 @@ Some workloads may experience improved memory profile thanks to optimizations fr #### WebRTC-Direct Transport enabled by default -Kubo now ships with `/udp/4001/webrtc-direct` listener enabled by default. +Kubo now ships with [WebRTC Direct](https://github.com/libp2p/specs/blob/master/webrtc/webrtc-direct.md) listener enabled by default: `/udp/4001/webrtc-direct`. WebRTC Direct complements existing `/wss` (Secure WebSockets) and `/webtransport` transports. Unlike `/wss`, which requires a domain name and a CA-issued TLS certificate, WebRTC Direct works with IPs and can be enabled by default on all Kubo nodes. @@ -54,9 +54,41 @@ Learn more: [`Swarm.Transports.Network.WebRTCDirect`](https://github.com/ipfs/ku > [!NOTE] > Kubo 0.30 includes a migration for existing users that adds `/webrtc-direct` listener on the same UDP port as `/udp/{port}/quic-v1`. This supports the WebRTC-Direct rollout by reusing preexisting UDP firewall settings and port mappings created for QUIC. +#### UnixFS 1.5: Mode and Modification Time Support + +Kubo now allows users to opt-in to store mode and modification time for files, directories, and symbolic links. +By default, if you do not opt-in, the old behavior remains unchanged, and the same CIDs will be generated as before. + +The `ipfs add` CLI options `--preserve-mode` and `--preserve-mtime` can be used to store the original mode and last modified time of the file being added, and `ipfs files stat /ipfs/CID` can be used for inspecting these optional attributes: + +```console +$ touch ./file +$ chmod 654 ./file +$ ipfs add --preserve-mode --preserve-mtime -Q ./file +QmczQr4XS1rRnWVopyg5Chr9EQ7JKpbhgnrjpb5kTQ1DKQ + +$ ipfs files stat /ipfs/QmczQr4XS1rRnWVopyg5Chr9EQ7JKpbhgnrjpb5kTQ1DKQ +QmczQr4XS1rRnWVopyg5Chr9EQ7JKpbhgnrjpb5kTQ1DKQ +Size: 0 +CumulativeSize: 22 +ChildBlocks: 0 +Type: file +Mode: -rw-r-xr-- (0654) +Mtime: 13 Aug 2024, 21:15:31 UTC +``` + +The CLI and HTTP RPC options `--mode`, `--mtime` and `--mtime-nsecs` can be used to set them to arbitrary values. + +Opt-in support for `mode` and `mtime` was also added to MFS (`ipfs files --help`). For more information see `--help` text of `ipfs files touch|stat|chmod` commands. + +Modification time support was also added to the Gateway. If present, value from file's dag-pb is returned in `Last-Modified` HTTP header and requests made with `If-Modified-Since` can produce HTTP 304 not modified response. + +> [!NOTE] +> Storing `mode` and `mtime` requires root block to be `dag-pb` and disabled `raw-leaves` setting to create envelope for storing the metadata. + #### AutoNAT V2 Service Introduced Alongside V1 -The AutoNAT service enables nodes to determine their public reachability on the internet. AutoNAT V2 enhances this protocol with improved features. In this release, Kubo will offer both V1 and V2 services to other peers, although it will continue to use only V1 when acting as a client. Future releases will phase out V1, transitioning clients to utilize V2 exclusively. +The AutoNAT service enables nodes to determine their public reachability on the internet. [AutoNAT V2](https://github.com/libp2p/specs/pull/538) enhances this protocol with improved features. In this release, Kubo will offer both V1 and V2 services to other peers, although it will continue to use only V1 when acting as a client. Future releases will phase out V1, transitioning clients to utilize V2 exclusively. For more details, see the [Deployment Plan for AutoNAT V2](https://github.com/ipfs/kubo/issues/10091) and [`AutoNAT`](https://github.com/ipfs/kubo/blob/master/docs/config.md#autonat) configuration options. @@ -120,37 +152,9 @@ The previous lengthy listing of all listener and announced multiaddrs has been r The output now features a simplified list of swarm listeners, displayed in the format `host:port (TCP+UDP)`, which provides essential information for debugging connectivity issues, particularly related to port forwarding. Announced libp2p addresses are no longer printed on startup, because libp2p may change or augument them based on AutoNAT, relay, and UPnP state. Instead, users are prompted to run `ipfs id` to obtain up-to-date list of listeners and announced multiaddrs in libp2p format. -#### UnixFS 1.5: Mode and Modification Time Support - -Kubo now allows users to opt-in to store mode and modification time for files, directories, and symbolic links. -By default, if you do not opt-in, the old behavior remains unchanged, and the same CIDs will be generated as before. - -The `ipfs add` CLI options `--preserve-mode` and `--preserve-mtime` can be used to store the original mode and last modified time of the file being added, and `ipfs files stat /ipfs/CID` can be used for inspecting these optional attributes: - -```console -$ touch ./file -$ chmod 654 ./file -$ ipfs add --preserve-mode --preserve-mtime -Q ./file -QmczQr4XS1rRnWVopyg5Chr9EQ7JKpbhgnrjpb5kTQ1DKQ - -$ ipfs files stat /ipfs/QmczQr4XS1rRnWVopyg5Chr9EQ7JKpbhgnrjpb5kTQ1DKQ -QmczQr4XS1rRnWVopyg5Chr9EQ7JKpbhgnrjpb5kTQ1DKQ -Size: 0 -CumulativeSize: 22 -ChildBlocks: 0 -Type: file -Mode: -rw-r-xr-- (0654) -Mtime: 13 Aug 2024, 21:15:31 UTC -``` - -The CLI and HTTP RPC options `--mode`, `--mtime` and `--mtime-nsecs` can be used to set them to arbitrary values. - -Opt-in support for `mode` and `mtime` was also added to MFS (`ipfs files --help`). For more information see `--help` text of `ipfs files touch|stat|chmod` commands. - -Modification time support was also added to the Gateway. If present, value from file's dag-pb is returned in `Last-Modified` HTTP header and requests made with `If-Modified-Since` can produce HTTP 304 not modified response. +#### Commands Preserve Specified Hostname -> [!NOTE] -> Storing `mode` and `mtime` requires root block to be `dag-pb` and disabled `raw-leaves` setting to create envelope for storing the metadata. +When executing a [CLI command](https://docs.ipfs.tech/reference/kubo/cli/) over [Kubo RPC API](https://docs.ipfs.tech/reference/kubo/rpc/), if a hostname is specified by `--api=/dns4//` the resulting HTTP request now contains the hostname, instead of the the IP address that the hostname resolved to, as was the previous behavior. This makes it easier for those trying to run Kubo behind a reverse proxy using hostname-based rules. #### Commands Preserve Specified Hostname @@ -158,4 +162,180 @@ When executing a [CLI command](https://docs.ipfs.tech/reference/kubo/cli/) over ### 📝 Changelog +
Full Changelog + +- github.com/ipfs/kubo: + - chore: set version to 0.30.0 + - chore: bump CurrentVersionNumber + - chore: boxo v0.23.0 and go-libp2p v0.36.3 (#10507) ([ipfs/kubo#10507](https://github.com/ipfs/kubo/pull/10507)) + - fix: switch back to go 1.22 (#10502) ([ipfs/kubo#10502](https://github.com/ipfs/kubo/pull/10502)) + - chore: update go-unixfsnode, cmds, and boxo (#10494) ([ipfs/kubo#10494](https://github.com/ipfs/kubo/pull/10494)) + - fix(cli): preserve hostname specified with --api in http request headers (#10497) ([ipfs/kubo#10497](https://github.com/ipfs/kubo/pull/10497)) + - chore: upgrade to go 1.23 (#10486) ([ipfs/kubo#10486](https://github.com/ipfs/kubo/pull/10486)) + - fix: error during config when running benchmarks (#10495) ([ipfs/kubo#10495](https://github.com/ipfs/kubo/pull/10495)) + - chore: update version to rc-2 + - chore: update version + - chore: fix function name (#10481) ([ipfs/kubo#10481](https://github.com/ipfs/kubo/pull/10481)) + - feat: Support storing UnixFS 1.5 Mode and ModTime (#10478) ([ipfs/kubo#10478](https://github.com/ipfs/kubo/pull/10478)) + - fix(rpc): cross-platform support for /unix/ socket maddrs in Addresses.API ([ipfs/kubo#10019](https://github.com/ipfs/kubo/pull/10019)) + - chore(daemon): sort listeners (#10480) ([ipfs/kubo#10480](https://github.com/ipfs/kubo/pull/10480)) + - feat(daemon): improve stdout on startup (#10472) ([ipfs/kubo#10472](https://github.com/ipfs/kubo/pull/10472)) + - fix(daemon): panic in kubo/daemon.go:595 (#10473) ([ipfs/kubo#10473](https://github.com/ipfs/kubo/pull/10473)) + - feat: webui v4.3.0 (#10477) ([ipfs/kubo#10477](https://github.com/ipfs/kubo/pull/10477)) + - docs(readme): add Gentoo Linux (#10474) ([ipfs/kubo#10474](https://github.com/ipfs/kubo/pull/10474)) + - libp2p: default to prefering TLS ([ipfs/kubo#10227](https://github.com/ipfs/kubo/pull/10227)) + - docs: document unofficial Ubuntu PPA ([ipfs/kubo#10467](https://github.com/ipfs/kubo/pull/10467)) + - feat: run AutoNAT V2 service in addition to V1 (#10468) ([ipfs/kubo#10468](https://github.com/ipfs/kubo/pull/10468)) + - feat: go-libp2p 0.36 and /webrtc-direct listener (#10463) ([ipfs/kubo#10463](https://github.com/ipfs/kubo/pull/10463)) + - chore: update dependencies (#10462)(#10466) ([ipfs/kubo#10466](https://github.com/ipfs/kubo/pull/10466)) + - feat: periodic version check and json config (#10438) ([ipfs/kubo#10438](https://github.com/ipfs/kubo/pull/10438)) + - docs: clarify pnet limitations + - docs: "error mounting: could not resolve name" (#10449) ([ipfs/kubo#10449](https://github.com/ipfs/kubo/pull/10449)) + - docs: update ipfs-swarm-key-gen example (#10453) ([ipfs/kubo#10453](https://github.com/ipfs/kubo/pull/10453)) + - chore: update deps incl. boxo v0.21.0 (#10444) ([ipfs/kubo#10444](https://github.com/ipfs/kubo/pull/10444)) + - chore: go-libp2p 0.35.1 (#10430) ([ipfs/kubo#10430](https://github.com/ipfs/kubo/pull/10430)) + - docsa: update RELEASE_CHECKLIST.md + - chore: create next changelog (#10443) ([ipfs/kubo#10443](https://github.com/ipfs/kubo/pull/10443)) + - Merge Release: v0.29.0 [skip changelog] ([ipfs/kubo#10442](https://github.com/ipfs/kubo/pull/10442)) + - fix(cli): unify --name param in ls and add (#10439) ([ipfs/kubo#10439](https://github.com/ipfs/kubo/pull/10439)) + - fix(libp2p): streams config validation in resource manager (#10435) ([ipfs/kubo#10435](https://github.com/ipfs/kubo/pull/10435)) + - chore: fix some typos (#10396) ([ipfs/kubo#10396](https://github.com/ipfs/kubo/pull/10396)) + - chore: update version +- github.com/ipfs/boxo (v0.20.0 -> v0.23.0): + - Release v0.23.0 ([ipfs/boxo#669](https://github.com/ipfs/boxo/pull/669)) + - docs(changelog): move entry to correct release + - Release v0.22.0 ([ipfs/boxo#654](https://github.com/ipfs/boxo/pull/654)) + - Release v0.21.0 ([ipfs/boxo#622](https://github.com/ipfs/boxo/pull/622)) +- github.com/ipfs/go-ipfs-cmds (v0.11.0 -> v0.13.0): + - chore: release v0.13.0 (#261) ([ipfs/go-ipfs-cmds#261](https://github.com/ipfs/go-ipfs-cmds/pull/261)) + - chore: release v0.12.0 (#259) ([ipfs/go-ipfs-cmds#259](https://github.com/ipfs/go-ipfs-cmds/pull/259)) +- github.com/ipfs/go-unixfsnode (v1.9.0 -> v1.9.1): + - Update release version ([ipfs/go-unixfsnode#76](https://github.com/ipfs/go-unixfsnode/pull/76)) + - chore: update dependencies ([ipfs/go-unixfsnode#75](https://github.com/ipfs/go-unixfsnode/pull/75)) +- github.com/libp2p/go-libp2p (v0.34.1 -> v0.36.3): + - Release v0.36.3 + - Fix: WebSocket: Clone TLS config before creating a new listener + - fix: enable dctur when interface address is public (#2931) ([libp2p/go-libp2p#2931](https://github.com/libp2p/go-libp2p/pull/2931)) + - fix: QUIC/Webtransport Transports now will prefer their owned listeners for dialing out (#2936) ([libp2p/go-libp2p#2936](https://github.com/libp2p/go-libp2p/pull/2936)) + - ci: uci/update-go (#2937) ([libp2p/go-libp2p#2937](https://github.com/libp2p/go-libp2p/pull/2937)) + - fix: slice append value (#2938) ([libp2p/go-libp2p#2938](https://github.com/libp2p/go-libp2p/pull/2938)) + - webrtc: wait for listener context before dropping connection (#2932) ([libp2p/go-libp2p#2932](https://github.com/libp2p/go-libp2p/pull/2932)) + - ci: use go1.23, drop go1.21 (#2933) ([libp2p/go-libp2p#2933](https://github.com/libp2p/go-libp2p/pull/2933)) + - Fail on any test timeout (#2929) ([libp2p/go-libp2p#2929](https://github.com/libp2p/go-libp2p/pull/2929)) + - test: Try to fix test timeout (#2930) ([libp2p/go-libp2p#2930](https://github.com/libp2p/go-libp2p/pull/2930)) + - ci: Out of the tarpit (#2923) ([libp2p/go-libp2p#2923](https://github.com/libp2p/go-libp2p/pull/2923)) + - Fix proto import paths (#2920) ([libp2p/go-libp2p#2920](https://github.com/libp2p/go-libp2p/pull/2920)) + - Release v0.36.2 + - webrtc: reduce loglevel for pion logs (#2915) ([libp2p/go-libp2p#2915](https://github.com/libp2p/go-libp2p/pull/2915)) + - webrtc: close connection when remote closes (#2914) ([libp2p/go-libp2p#2914](https://github.com/libp2p/go-libp2p/pull/2914)) + - basic_host: close swarm on Close (#2916) ([libp2p/go-libp2p#2916](https://github.com/libp2p/go-libp2p/pull/2916)) + - Revert "Create funding.json" (#2919) ([libp2p/go-libp2p#2919](https://github.com/libp2p/go-libp2p/pull/2919)) + - Create funding.json + - Release v0.36.1 + - Release v0.36.0 (#2905) ([libp2p/go-libp2p#2905](https://github.com/libp2p/go-libp2p/pull/2905)) + - swarm: add a default timeout to conn.NewStream (#2907) ([libp2p/go-libp2p#2907](https://github.com/libp2p/go-libp2p/pull/2907)) + - udpmux: Don't log an error if canceled because of shutdown (#2903) ([libp2p/go-libp2p#2903](https://github.com/libp2p/go-libp2p/pull/2903)) + - ObsAddrManager: Infer external addresses for transports that share the same listening address. (#2892) ([libp2p/go-libp2p#2892](https://github.com/libp2p/go-libp2p/pull/2892)) + - feat: WebRTC reuse QUIC conn (#2889) ([libp2p/go-libp2p#2889](https://github.com/libp2p/go-libp2p/pull/2889)) + - examples/chat-with-mdns: default to a random port (#2896) ([libp2p/go-libp2p#2896](https://github.com/libp2p/go-libp2p/pull/2896)) + - allow findpeers limit to be 0 (#2894) ([libp2p/go-libp2p#2894](https://github.com/libp2p/go-libp2p/pull/2894)) + - quic: add support for quic-go metrics (#2823) ([libp2p/go-libp2p#2823](https://github.com/libp2p/go-libp2p/pull/2823)) + - webrtc: remove experimental tag, enable by default (#2887) ([libp2p/go-libp2p#2887](https://github.com/libp2p/go-libp2p/pull/2887)) + - config: fix AddrFactory for AutoNAT (#2868) ([libp2p/go-libp2p#2868](https://github.com/libp2p/go-libp2p/pull/2868)) + - chore: /quic → /quic-v1 (#2888) ([libp2p/go-libp2p#2888](https://github.com/libp2p/go-libp2p/pull/2888)) + - basichost: reset stream if SetProtocol fails (#2875) ([libp2p/go-libp2p#2875](https://github.com/libp2p/go-libp2p/pull/2875)) + - feat: libp2phttp `/http-path` (#2850) ([libp2p/go-libp2p#2850](https://github.com/libp2p/go-libp2p/pull/2850)) + - readme: update per latest multiversx rename (#2874) ([libp2p/go-libp2p#2874](https://github.com/libp2p/go-libp2p/pull/2874)) + - websocket: don't return transport.ErrListenerClosed on closing listener (#2867) ([libp2p/go-libp2p#2867](https://github.com/libp2p/go-libp2p/pull/2867)) + - Added tau to README.md (#2870) ([libp2p/go-libp2p#2870](https://github.com/libp2p/go-libp2p/pull/2870)) + - basichost: reset new stream if rcmgr blocks (#2869) ([libp2p/go-libp2p#2869](https://github.com/libp2p/go-libp2p/pull/2869)) + - transport integration tests: test conn attempt is dropped when the rcmgr blocks for WebRTC (#2856) ([libp2p/go-libp2p#2856](https://github.com/libp2p/go-libp2p/pull/2856)) + - webtransport: close underlying h3 connection (#2862) ([libp2p/go-libp2p#2862](https://github.com/libp2p/go-libp2p/pull/2862)) + - peerstore: don't intern protocols (#2860) ([libp2p/go-libp2p#2860](https://github.com/libp2p/go-libp2p/pull/2860)) + - autonatv2: add server metrics for dial requests (#2848) ([libp2p/go-libp2p#2848](https://github.com/libp2p/go-libp2p/pull/2848)) + - PR Comments + - Add a transport level test to ensure we close conns after rejecting them by the rcmgr + - Close quic conns when wrapping conn fails + - libp2p: use rcmgr for autonat dials (#2842) ([libp2p/go-libp2p#2842](https://github.com/libp2p/go-libp2p/pull/2842)) + - metricshelper: improve checks for ip and transport (#2849) ([libp2p/go-libp2p#2849](https://github.com/libp2p/go-libp2p/pull/2849)) + - Don't reuse the URL, make a new one + - Use default transport to make using the Host cheaper + - cleanup + - Add future test + - HTTP Host implements RoundTripper + - swarm: improve dial worker performance for common case + - pstoremanager: fix connectedness check + - autonatv2: implement autonatv2 spec (#2469) ([libp2p/go-libp2p#2469](https://github.com/libp2p/go-libp2p/pull/2469)) + - webrtc: add a test for establishing many connections (#2801) ([libp2p/go-libp2p#2801](https://github.com/libp2p/go-libp2p/pull/2801)) + - webrtc: fix ufrag prefix for dialing (#2832) ([libp2p/go-libp2p#2832](https://github.com/libp2p/go-libp2p/pull/2832)) + - circuitv2: improve voucher validation (#2826) ([libp2p/go-libp2p#2826](https://github.com/libp2p/go-libp2p/pull/2826)) + - libp2phttp: workaround for ResponseWriter's CloseNotifier (#2821) ([libp2p/go-libp2p#2821](https://github.com/libp2p/go-libp2p/pull/2821)) + - Update README.md (#2830) ([libp2p/go-libp2p#2830](https://github.com/libp2p/go-libp2p/pull/2830)) + - identify: add test for observed address handling (#2828) ([libp2p/go-libp2p#2828](https://github.com/libp2p/go-libp2p/pull/2828)) + - identify: fix bug in observed address handling (#2825) ([libp2p/go-libp2p#2825](https://github.com/libp2p/go-libp2p/pull/2825)) + - identify: Don't filter addr if remote is neither public nor private (#2820) ([libp2p/go-libp2p#2820](https://github.com/libp2p/go-libp2p/pull/2820)) + - limit ping duration to 30s (#1358) ([libp2p/go-libp2p#1358](https://github.com/libp2p/go-libp2p/pull/1358)) + - Remove out-dated code in example readme (#2818) ([libp2p/go-libp2p#2818](https://github.com/libp2p/go-libp2p/pull/2818)) + - v0.35.0 (#2812) ([libp2p/go-libp2p#2812](https://github.com/libp2p/go-libp2p/pull/2812)) + - rcmgr: Support specific network prefix in conn limiter (#2807) ([libp2p/go-libp2p#2807](https://github.com/libp2p/go-libp2p/pull/2807)) +- github.com/libp2p/go-libp2p-kad-dht (v0.25.2 -> v0.26.1): + - Release v0.26.1 ([libp2p/go-libp2p-kad-dht#983](https://github.com/libp2p/go-libp2p-kad-dht/pull/983)) + - fix: Unexport hasValidConnectedness to make a patch release ([libp2p/go-libp2p-kad-dht#982](https://github.com/libp2p/go-libp2p-kad-dht/pull/982)) + - correctly merging fix from https://github.com/libp2p/go-libp2p-kad-dht/pull/976 ([libp2p/go-libp2p-kad-dht#980](https://github.com/libp2p/go-libp2p-kad-dht/pull/980)) + - Release v0.26.0 ([libp2p/go-libp2p-kad-dht#979](https://github.com/libp2p/go-libp2p-kad-dht/pull/979)) + - chore: update deps ([libp2p/go-libp2p-kad-dht#974](https://github.com/libp2p/go-libp2p-kad-dht/pull/974)) + - Upgrade to go-log v2.5.1 ([libp2p/go-libp2p-kad-dht#971](https://github.com/libp2p/go-libp2p-kad-dht/pull/971)) + - Fix: don't perform lookupCheck if not enough peers in routing table ([libp2p/go-libp2p-kad-dht#970](https://github.com/libp2p/go-libp2p-kad-dht/pull/970)) + - findnode(self) should return multiple peers ([libp2p/go-libp2p-kad-dht#968](https://github.com/libp2p/go-libp2p-kad-dht/pull/968)) +- github.com/libp2p/go-libp2p-routing-helpers (v0.7.3 -> v0.7.4): + - chore: release v0.7.4 (#85) ([libp2p/go-libp2p-routing-helpers#85](https://github.com/libp2p/go-libp2p-routing-helpers/pull/85)) + - fix: composable parallel router tracing by index (#84) ([libp2p/go-libp2p-routing-helpers#84](https://github.com/libp2p/go-libp2p-routing-helpers/pull/84)) +- github.com/multiformats/go-multiaddr (v0.12.4 -> v0.13.0): + - Release v0.13.0 ([multiformats/go-multiaddr#248](https://github.com/multiformats/go-multiaddr/pull/248)) + - Add support for http-path ([multiformats/go-multiaddr#246](https://github.com/multiformats/go-multiaddr/pull/246)) +- github.com/whyrusleeping/cbor-gen (v0.1.1 -> v0.1.2): + - properly extend strings (#95) ([whyrusleeping/cbor-gen#95](https://github.com/whyrusleeping/cbor-gen/pull/95)) + - ioutil to io (#98) ([whyrusleeping/cbor-gen#98](https://github.com/whyrusleeping/cbor-gen/pull/98)) + +
+ ### 👨‍👩‍👧‍👦 Contributors + +| Contributor | Commits | Lines ± | Files Changed | +|-------------|---------|---------|---------------| +| Andrew Gillis | 14 | +4920/-1714 | 145 | +| sukun | 26 | +4402/-448 | 79 | +| Marco Munizaga | 32 | +2287/-536 | 73 | +| Marcin Rataj | 41 | +685/-193 | 86 | +| Patryk | 1 | +312/-10 | 8 | +| guillaumemichel | 7 | +134/-105 | 14 | +| Adin Schmahmann | 5 | +145/-80 | 9 | +| Henrique Dias | 2 | +190/-1 | 6 | +| Josh Klopfenstein | 1 | +90/-35 | 27 | +| gammazero | 5 | +90/-28 | 11 | +| Jeromy Johnson | 1 | +116/-0 | 5 | +| Daniel N | 3 | +27/-25 | 9 | +| Daniel Norman | 2 | +28/-19 | 4 | +| Ivan Shvedunov | 2 | +25/-10 | 2 | +| Michael Muré | 2 | +22/-9 | 4 | +| Dominic Della Valle | 1 | +23/-4 | 1 | +| Andrei Vukolov | 1 | +27/-0 | 1 | +| chris erway | 1 | +9/-9 | 9 | +| Vitaly Zdanevich | 1 | +12/-0 | 1 | +| Guillaume Michel | 1 | +4/-7 | 1 | +| swedneck | 1 | +10/-0 | 1 | +| Jorropo | 2 | +5/-5 | 3 | +| omahs | 1 | +4/-4 | 4 | +| THAT ONE GUY | 1 | +3/-5 | 2 | +| vyzo | 1 | +5/-2 | 1 | +| looklose | 1 | +3/-3 | 2 | +| web3-bot | 2 | +2/-3 | 4 | +| Dave Huseby | 1 | +5/-0 | 1 | +| shenpengfeng | 1 | +1/-1 | 1 | +| bytetigers | 1 | +1/-1 | 1 | +| Sorin Stanculeanu | 1 | +1/-1 | 1 | +| Lukáš Lukáč | 1 | +1/-1 | 1 | +| Gabe | 1 | +1/-1 | 1 | +| Bryan Stenson | 1 | +1/-1 | 1 | +| Samy Fodil | 1 | +1/-0 | 1 | +| Lane Rettig | 1 | +1/-0 | 1 | diff --git a/docs/changelogs/v0.31.md b/docs/changelogs/v0.31.md new file mode 100644 index 00000000000..80823816cc8 --- /dev/null +++ b/docs/changelogs/v0.31.md @@ -0,0 +1,18 @@ +# Kubo changelog v0.31 + +- [v0.31.0](#v0310) + +## v0.31.0 + +- [Overview](#overview) +- [🔦 Highlights](#-highlights) +- [📝 Changelog](#-changelog) +- [👨‍👩‍👧‍👦 Contributors](#-contributors) + +### Overview + +### 🔦 Highlights + +### 📝 Changelog + +### 👨‍👩‍👧‍👦 Contributors diff --git a/docs/config.md b/docs/config.md index 107050927b7..5500128abf1 100644 --- a/docs/config.md +++ b/docs/config.md @@ -9,16 +9,6 @@ config file at runtime. - [The Kubo config file](#the-kubo-config-file) - [Table of Contents](#table-of-contents) - - [Profiles](#profiles) - - [Types](#types) - - [`flag`](#flag) - - [`priority`](#priority) - - [`strings`](#strings) - - [`duration`](#duration) - - [`optionalInteger`](#optionalinteger) - - [`optionalBytes`](#optionalbytes) - - [`optionalString`](#optionalstring) - - [`optionalDuration`](#optionalduration) - [`Addresses`](#addresses) - [`Addresses.API`](#addressesapi) - [`Addresses.Gateway`](#addressesgateway) @@ -184,184 +174,26 @@ config file at runtime. - [`Version.AgentSuffix`](#versionagentsuffix) - [`Version.SwarmCheckEnabled`](#versionswarmcheckenabled) - [`Version.SwarmCheckPercentThreshold`](#versionswarmcheckpercentthreshold) - -## Profiles - -Configuration profiles allow to tweak configuration quickly. Profiles can be -applied with the `--profile` flag to `ipfs init` or with the `ipfs config profile -apply` command. When a profile is applied a backup of the configuration file -will be created in `$IPFS_PATH`. - -The available configuration profiles are listed below. You can also find them -documented in `ipfs config profile --help`. - -- `server` - - Disables local host discovery, recommended when - running IPFS on machines with public IPv4 addresses. - -- `randomports` - - Use a random port number for the incoming swarm connections. - -- `default-datastore` - - Configures the node to use the default datastore (flatfs). - - Read the "flatfs" profile description for more information on this datastore. - - This profile may only be applied when first initializing the node. - -- `local-discovery` - - Enables local discovery (enabled by default). Useful to re-enable local discovery after it's - disabled by another profile (e.g., the server profile). - -- `test` - - Reduces external interference of IPFS daemon, this - is useful when using the daemon in test environments. - -- `default-networking` - - Restores default network settings. - Inverse profile of the test profile. - -- `flatfs` - - Configures the node to use the flatfs datastore. Flatfs is the default datastore. - - This is the most battle-tested and reliable datastore. - You should use this datastore if: - - - You need a very simple and very reliable datastore, and you trust your - filesystem. This datastore stores each block as a separate file in the - underlying filesystem so it's unlikely to lose data unless there's an issue - with the underlying file system. - - You need to run garbage collection in a way that reclaims free space as soon as possible. - - You want to minimize memory usage. - - You are ok with the default speed of data import, or prefer to use `--nocopy`. - - This profile may only be applied when first initializing the node. - - -- `badgerds` - - Configures the node to use the experimental badger datastore. Keep in mind that this **uses an outdated badger 1.x**. - - Use this datastore if some aspects of performance, - especially the speed of adding many gigabytes of files, are critical. However, be aware that: - - - This datastore will not properly reclaim space when your datastore is - smaller than several gigabytes. If you run IPFS with `--enable-gc`, you plan on storing very little data in - your IPFS node, and disk usage is more critical than performance, consider using - `flatfs`. - - This datastore uses up to several gigabytes of memory. - - Good for medium-size datastores, but may run into performance issues if your dataset is bigger than a terabyte. - - The current implementation is based on old badger 1.x which is no longer supported by the upstream team. - - This profile may only be applied when first initializing the node. - -- `lowpower` - - Reduces daemon overhead on the system. Affects node - functionality - performance of content discovery and data - fetching may be degraded. Local data won't be announced on routing systems like Amino DHT. - - - `Swarm.ConnMgr` set to maintain minimum number of p2p connections at a time. - - Disables [`Reprovider`](#reprovider) service → no CID will be announced on Amino DHT and other routing systems(!) - - Disables AutoNAT. - - Use this profile with caution. - -- `legacy-cid-v0` - - Makes UnixFS import (`ipfs add`) produce legacy CIDv0 with no raw leaves, sha2-256 and 256 KiB chunks. - - > [!WARNING] - > This profile is provided for legacy users and should not be used for new projects. - -- `test-cid-v1` - - Makes UnixFS import (`ipfs add`) produce modern CIDv1 with raw leaves, sha2-256 and 1 MiB chunks. - - > [!NOTE] - > This profile will become the new implicit default, provided for testing purposes. - > Follow [kubo#4143](https://github.com/ipfs/kubo/issues/4143) for more details. - -## Types - -This document refers to the standard JSON types (e.g., `null`, `string`, -`number`, etc.), as well as a few custom types, described below. - -### `flag` - -Flags allow enabling and disabling features. However, unlike simple booleans, -they can also be `null` (or omitted) to indicate that the default value should -be chosen. This makes it easier for Kubo to change the defaults in the -future unless the user _explicitly_ sets the flag to either `true` (enabled) or -`false` (disabled). Flags have three possible states: - -- `null` or missing (apply the default value). -- `true` (enabled) -- `false` (disabled) - -### `priority` - -Priorities allow specifying the priority of a feature/protocol and disabling the -feature/protocol. Priorities can take one of the following values: - -- `null`/missing (apply the default priority, same as with flags) -- `false` (disabled) -- `1 - 2^63` (priority, lower is preferred) - -### `strings` - -Strings is a special type for conveniently specifying a single string, an array -of strings, or null: - -- `null` -- `"a single string"` -- `["an", "array", "of", "strings"]` - -### `duration` - -Duration is a type for describing lengths of time, using the same format go -does (e.g, `"1d2h4m40.01s"`). - -### `optionalInteger` - -Optional integers allow specifying some numerical value which has -an implicit default when missing from the config file: - -- `null`/missing will apply the default value defined in Kubo sources (`.WithDefault(value)`) -- an integer between `-2^63` and `2^63-1` (i.e. `-9223372036854775808` to `9223372036854775807`) - -### `optionalBytes` - -Optional Bytes allow specifying some number of bytes which has -an implicit default when missing from the config file: - -- `null`/missing (apply the default value defined in Kubo sources) -- a string value indicating the number of bytes, including human readable representations: - - [SI sizes](https://en.wikipedia.org/wiki/Metric_prefix#List_of_SI_prefixes) (metric units, powers of 1000), e.g. `1B`, `2kB`, `3MB`, `4GB`, `5TB`, …) - - [IEC sizes](https://en.wikipedia.org/wiki/Binary_prefix#IEC_prefixes) (binary units, powers of 1024), e.g. `1B`, `2KiB`, `3MiB`, `4GiB`, `5TiB`, …) - -### `optionalString` - -Optional strings allow specifying some string value which has -an implicit default when missing from the config file: - -- `null`/missing will apply the default value defined in Kubo sources (`.WithDefault("value")`) -- a string - -### `optionalDuration` - -Optional durations allow specifying some duration value which has -an implicit default when missing from the config file: - -- `null`/missing will apply the default value defined in Kubo sources (`.WithDefault("1h2m3s")`) -- a string with a valid [go duration](#duration) (e.g, `"1d2h4m40.01s"`). + - [Profiles](#profiles) + - [`server` profile](#server-profile) + - [`randomports` profile](#randomports-profile) + - [`default-datastore` profile](#default-datastore-profile) + - [`local-discovery` profile](#local-discovery-profile) + - [`default-networking` profile](#default-networking-profile) + - [`flatfs` profile](#flatfs-profile) + - [`badgerds` profile](#badgerds-profile) + - [`lowpower` profile](#lowpower-profile) + - [`legacy-cid-v0` profile](#legacy-cid-v0-profile) + - [`test-cid-v1` profile](#test-cid-v1-profile) + - [Types](#types) + - [`flag`](#flag) + - [`priority`](#priority) + - [`strings`](#strings) + - [`duration`](#duration) + - [`optionalInteger`](#optionalinteger) + - [`optionalBytes`](#optionalbytes) + - [`optionalString`](#optionalstring) + - [`optionalDuration`](#optionalduration) ## `Addresses` @@ -446,6 +278,12 @@ Type: `array[string]` (multiaddrs) An array of swarm addresses not to announce to the network. Takes precedence over `Addresses.Announce` and `Addresses.AppendAnnounce`. +> [!TIP] +> The [`server` configuration profile](#server-profile) fills up this list with sensible defaults, +> preventing announcement of non-routable IP addresses (e.g., `/ip4/192.168.0.0/ipcidr/16`, +> which is the multiaddress representation of `192.168.0.0/16`) but you should always +> check settings against your own network and/or hosting provider. + Default: `[]` Type: `array[string]` (multiaddrs) @@ -844,7 +682,18 @@ We are working on developing a modern replacement. To support our efforts, pleas ### `Gateway.PublicGateways` -`PublicGateways` is a dictionary for defining gateway behavior on specified hostnames. +> [!IMPORTANT] +> This configuration is **NOT** for HTTP Client, it is for HTTP Server – use this ONLY if you want to run your own IPFS gateway. + +`PublicGateways` is a configuration map used for dictionary for customizing gateway behavior +on specified hostnames that point at your Kubo instance. + +It is useful when you want to run [Path gateway](https://specs.ipfs.tech/http-gateways/path-gateway/) on `example.com/ipfs/cid`, +and [Subdomain gateway](https://specs.ipfs.tech/http-gateways/subdomain-gateway/) on `cid.ipfs.example.org`, +or limit `verifiable.example.net` to response types defined in [Trustless Gateway](https://specs.ipfs.tech/http-gateways/trustless-gateway/) specification. + +> [!CAUTION] +> Keys (Hostnames) MUST be unique. Do not use the same parent domain for multiple gateway types, it will break origin isolation. Hostnames can optionally be defined with one or more wildcards. @@ -877,7 +726,9 @@ Type: `array[string]` #### `Gateway.PublicGateways: UseSubdomains` -A boolean to configure whether the gateway at the hostname provides [Origin isolation](https://developer.mozilla.org/en-US/docs/Web/Security/Same-origin_policy) +A boolean to configure whether the gateway at the hostname should be +a [Subdomain Gateway](https://specs.ipfs.tech/http-gateways/subdomain-gateway/) +and provide [Origin isolation](https://developer.mozilla.org/en-US/docs/Web/Security/Same-origin_policy) between content roots. - `true` - enables [subdomain gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway) at `http://*.{hostname}/` @@ -926,7 +777,7 @@ Type: `bool` An optional flag to explicitly configure whether subdomain gateway's redirects (enabled by `UseSubdomains: true`) should always inline a DNSLink name (FQDN) -into a single DNS label: +into a single DNS label ([specification](https://specs.ipfs.tech/http-gateways/subdomain-gateway/#host-request-header)): ``` //example.com/ipns/example.net → HTTP 301 → //example-net.ipns.example.com @@ -945,8 +796,14 @@ Type: `flag` #### `Gateway.PublicGateways: DeserializedResponses` An optional flag to explicitly configure whether this gateway responds to deserialized -requests, or not. By default, it is enabled. When disabling this option, the gateway -operates as a Trustless Gateway only: https://specs.ipfs.tech/http-gateways/trustless-gateway/. +requests, or not. By default, it is enabled. + +When disabled, the gateway operates strictly as a [Trustless Gateway](https://specs.ipfs.tech/http-gateways/trustless-gateway/). + +> [!TIP] +> Disabling deserialized responses will protect you from acting as a free web hosting, +> while still allowing trustless clients like [@helia/verified-fetch](https://www.npmjs.com/package/@helia/verified-fetch) +> to utilize it for [trustless, verifiable data retrieval](https://docs.ipfs.tech/reference/http/gateway/#trustless-verifiable-retrieval). Default: same as global `Gateway.DeserializedResponses` @@ -1813,10 +1670,11 @@ node will try to connect to one or more private IP addresses whenever dialing another node, even if this other node is on a different network. This may trigger netscan alerts on some hosting providers or cause strain in some setups. -The `server` configuration profile fills up this list with sensible defaults, -preventing dials to all non-routable IP addresses (e.g., `/ip4/192.168.0.0/ipcidr/16`, -which is the multiaddress representation of `192.168.0.0/16`) but you should always -check settings against your own network and/or hosting provider. +> [!TIP] +> The [`server` configuration profile](#server-profile) fills up this list with sensible defaults, +> preventing dials to all non-routable IP addresses (e.g., `/ip4/192.168.0.0/ipcidr/16`, +> which is the multiaddress representation of `192.168.0.0/16`) but you should always +> check settings against your own network and/or hosting provider. Default: `[]` @@ -1834,7 +1692,7 @@ Type: `bool` ### `Swarm.DisableNatPortMap` -Disable automatic NAT port forwarding. +Disable automatic NAT port forwarding (turn off [UPnP](https://en.wikipedia.org/wiki/Universal_Plug_and_Play)). When not disabled (default), Kubo asks NAT devices (e.g., routers), to open up an external port and forward it to the port Kubo is running on. When this @@ -2491,3 +2349,194 @@ trigger update warning. Default: `5` Type: `optionalInteger` (1-100) + +## Profiles + +Configuration profiles allow to tweak configuration quickly. Profiles can be +applied with the `--profile` flag to `ipfs init` or with the `ipfs config profile +apply` command. When a profile is applied a backup of the configuration file +will be created in `$IPFS_PATH`. + +Configuration profiles can be applied additively. For example, both the `test-cid-v1` and `lowpower` profiles can be applied one after the other. +The available configuration profiles are listed below. You can also find them +documented in `ipfs config profile --help`. + +### `server` profile + +Disables local [`Discovery.MDNS`](#discoverymdns), [turns off uPnP NAT port mapping](#swarmdisablenatportmap), and blocks connections to +IPv4 and IPv6 prefixes that are [private, local only, or unrouteable](https://github.com/ipfs/kubo/blob/b71cf0d15904bdef21fe2eee5f1118a274309a4d/config/profile.go#L24-L43). + +Recommended when running IPFS on machines with public IPv4 addresses (no NAT, no uPnP) +at providers that interpret local IPFS discovery and traffic as netscan abuse ([example](https://github.com/ipfs/kubo/issues/10327)). + +### `randomports` profile + +Use a random port number for the incoming swarm connections. +Used for testing. + +### `default-datastore` profile + +Configures the node to use the default datastore (flatfs). + +Read the "flatfs" profile description for more information on this datastore. + +This profile may only be applied when first initializing the node. + +### `local-discovery` profile + +Enables local [`Discovery.MDNS`](#discoverymdns) (enabled by default). + +Useful to re-enable local discovery after it's disabled by another profile +(e.g., the server profile). + +`test` profile + +Reduces external interference of IPFS daemon, this +is useful when using the daemon in test environments. + +### `default-networking` profile + +Restores default network settings. +Inverse profile of the test profile. + +### `flatfs` profile + +Configures the node to use the flatfs datastore. Flatfs is the default datastore. + +This is the most battle-tested and reliable datastore. +You should use this datastore if: + +- You need a very simple and very reliable datastore, and you trust your + filesystem. This datastore stores each block as a separate file in the + underlying filesystem so it's unlikely to lose data unless there's an issue + with the underlying file system. +- You need to run garbage collection in a way that reclaims free space as soon as possible. +- You want to minimize memory usage. +- You are ok with the default speed of data import, or prefer to use `--nocopy`. + +This profile may only be applied when first initializing the node. + +### `badgerds` profile + +Configures the node to use the legacy badgerv1 datastore. + +> [!CAUTION] +> This is based on very old badger 1.x, which has known bugs and is no longer supported by the upstream team. +> It is provided here only for pre-existing users, allowing them to migrate away to more modern datastore. +> Do not use it for new deployments, unless you really, really know what you are doing. + +Also, be aware that: + +- This datastore will not properly reclaim space when your datastore is + smaller than several gigabytes. If you run IPFS with `--enable-gc`, you plan on storing very little data in + your IPFS node, and disk usage is more critical than performance, consider using + `flatfs`. +- This datastore uses up to several gigabytes of memory. +- Good for medium-size datastores, but may run into performance issues if your dataset is bigger than a terabyte. +- The current implementation is based on old badger 1.x which is no longer supported by the upstream team. + +This profile may only be applied when first initializing the node. + +### `lowpower` profile + +Reduces daemon overhead on the system. Affects node +functionality - performance of content discovery and data +fetching may be degraded. + +> [!CAUTION] +> Local data won't be announced on routing systems like Amino DHT. + +- `Swarm.ConnMgr` set to maintain minimum number of p2p connections at a time. +- Disables [`Reprovider`](#reprovider) service → no CID will be announced on Amino DHT and other routing systems(!) +- Disables [`AutoNAT`](#autonat). + +Use this profile with caution. + +### `legacy-cid-v0` profile + +Makes UnixFS import (`ipfs add`) produce legacy CIDv0 with no raw leaves, sha2-256 and 256 KiB chunks. + +> [!NOTE] +> This profile is provided for legacy users and should not be used for new projects. + +### `test-cid-v1` profile + +Makes UnixFS import (`ipfs add`) produce modern CIDv1 with raw leaves, sha2-256 and 1 MiB chunks. + +> [!NOTE] +> This profile will become the new implicit default, provided for testing purposes. +> Follow [kubo#4143](https://github.com/ipfs/kubo/issues/4143) for more details. + +## Types + +This document refers to the standard JSON types (e.g., `null`, `string`, +`number`, etc.), as well as a few custom types, described below. + +### `flag` + +Flags allow enabling and disabling features. However, unlike simple booleans, +they can also be `null` (or omitted) to indicate that the default value should +be chosen. This makes it easier for Kubo to change the defaults in the +future unless the user _explicitly_ sets the flag to either `true` (enabled) or +`false` (disabled). Flags have three possible states: + +- `null` or missing (apply the default value). +- `true` (enabled) +- `false` (disabled) + +### `priority` + +Priorities allow specifying the priority of a feature/protocol and disabling the +feature/protocol. Priorities can take one of the following values: + +- `null`/missing (apply the default priority, same as with flags) +- `false` (disabled) +- `1 - 2^63` (priority, lower is preferred) + +### `strings` + +Strings is a special type for conveniently specifying a single string, an array +of strings, or null: + +- `null` +- `"a single string"` +- `["an", "array", "of", "strings"]` + +### `duration` + +Duration is a type for describing lengths of time, using the same format go +does (e.g, `"1d2h4m40.01s"`). + +### `optionalInteger` + +Optional integers allow specifying some numerical value which has +an implicit default when missing from the config file: + +- `null`/missing will apply the default value defined in Kubo sources (`.WithDefault(value)`) +- an integer between `-2^63` and `2^63-1` (i.e. `-9223372036854775808` to `9223372036854775807`) + +### `optionalBytes` + +Optional Bytes allow specifying some number of bytes which has +an implicit default when missing from the config file: + +- `null`/missing (apply the default value defined in Kubo sources) +- a string value indicating the number of bytes, including human readable representations: + - [SI sizes](https://en.wikipedia.org/wiki/Metric_prefix#List_of_SI_prefixes) (metric units, powers of 1000), e.g. `1B`, `2kB`, `3MB`, `4GB`, `5TB`, …) + - [IEC sizes](https://en.wikipedia.org/wiki/Binary_prefix#IEC_prefixes) (binary units, powers of 1024), e.g. `1B`, `2KiB`, `3MiB`, `4GiB`, `5TiB`, …) + +### `optionalString` + +Optional strings allow specifying some string value which has +an implicit default when missing from the config file: + +- `null`/missing will apply the default value defined in Kubo sources (`.WithDefault("value")`) +- a string + +### `optionalDuration` + +Optional durations allow specifying some duration value which has +an implicit default when missing from the config file: + +- `null`/missing will apply the default value defined in Kubo sources (`.WithDefault("1h2m3s")`) +- a string with a valid [go duration](#duration) (e.g, `"1d2h4m40.01s"`). diff --git a/docs/datastores.md b/docs/datastores.md index ccedce4c715..c18ecb0c7c5 100644 --- a/docs/datastores.md +++ b/docs/datastores.md @@ -39,6 +39,12 @@ Uses a leveldb database to store key value pairs. Uses [badger](https://github.com/dgraph-io/badger) as a key value store. +> [!CAUTION] +> This is based on very old badger 1.x, which has known bugs and is no longer supported by the upstream team. +> It is provided here only for pre-existing users, allowing them to migrate away to more modern datastore. +> Do not use it for new deployments, unless you really, really know what you are doing. + + * `syncWrites`: Flush every write to disk before continuing. Setting this to false is safe as kubo will automatically flush writes to disk before and after performing critical operations like pinning. However, you can set this to true to be extra-safe (at the cost of a 2-3x slowdown when adding files). * `truncate`: Truncate the DB if a partially written sector is found (defaults to true). There is no good reason to set this to false unless you want to manually recover partially written (and unpinned) blocks if kubo crashes half-way through adding a file. diff --git a/docs/releases_thunderdome.md b/docs/releases_thunderdome.md index 11057e26a30..53034b3bbda 100644 --- a/docs/releases_thunderdome.md +++ b/docs/releases_thunderdome.md @@ -50,7 +50,7 @@ This will build the Docker images, upload them to ECR, and then launch the exper ## Analyze Results -Add a log entry in https://www.notion.so/pl-strflt/ce2d1bd56f3541028d960d3711465659 and link to it from the release issue, so that experiment results are publicly visible. +Add a log entry in https://www.notion.so/ceb2047e79f2498494077a2739a6c493 and link to it from the release issue, so that experiment results are publicly visible. The `deploy` command will output a link to the Grafana dashboard for the experiment. We don't currently have rigorous acceptance criteria, so you should look for anomalies or changes in the metrics and make sure they are tolerable and explainable. Unexplainable anomalies should be noted in the log with a screenshot, and then root caused.