support piececid calculation for single car file filedrive-team#11

.gitignore

@@ -1,4 +1,5 @@
 /graphsplit
+/commp
 
 # Binaries for programs and plugins
 *.exe

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "extern/filecoin-ffi"]
+	path = extern/filecoin-ffi
+	url = https://github.com/filecoin-project/filecoin-ffi.git

Makefile

@@ -1,3 +1,16 @@
 .PHONY: build
 build:
 	go build -ldflags "-s -w" -o graphsplit ./cmd/graphsplit/main.go
+
+
+## FFI
+
+ffi: 
+	./extern/filecoin-ffi/install-filcrypto
+.PHONY: ffi
+
+
+commp: 
+	go build -ldflags "-s -w" -o commp ./cmd/commp/main.go
+.PHONY: commp
+

cmd/commp/main.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/filedrive-team/go-graphsplit/piece"
+	logging "github.com/ipfs/go-log/v2"
+	"github.com/urfave/cli/v2"
+)
+
+var log = logging.Logger("commp")
+
+func main() {
+	logging.SetLogLevel("*", "INFO")
+	local := []*cli.Command{
+		singleCmd,
+	}
+
+	app := &cli.App{
+		Name:     "commp",
+		Flags:    []cli.Flag{},
+		Commands: local,
+	}
+
+	if err := app.Run(os.Args); err != nil {
+		fmt.Println("Error: ", err)
+		os.Exit(1)
+	}
+}
+
+var singleCmd = &cli.Command{
+	Name:  "single",
+	Usage: "PieceCID and PieceSize calculation",
+	Flags: []cli.Flag{},
+	Action: func(c *cli.Context) error {
+		ctx := context.Background()
+		targetPath := c.Args().First()
+
+		res, err := piece.CalcCommP(ctx, targetPath)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("PieceCID: %s, PieceSize: %d\n", res.Root, res.Size)
+		return nil
+	},
+}

go.mod

@@ -3,36 +3,26 @@ module github.com/filedrive-team/go-graphsplit
 go 1.15
 
 require (
-	github.com/davidlazar/go-crypto v0.0.0-20190912175916-7055855a373f // indirect
-	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
-	github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f // indirect
+	github.com/filecoin-project/go-commp-utils v0.1.0
+	github.com/filecoin-project/go-padreader v0.0.0-20201016201355-9c5eb1faedb5
+	github.com/filecoin-project/go-state-types v0.1.0
 	github.com/ipfs/go-blockservice v0.1.4
 	github.com/ipfs/go-cid v0.0.7
 	github.com/ipfs/go-datastore v0.4.5
 	github.com/ipfs/go-ipfs-blockstore v1.0.3
 	github.com/ipfs/go-ipfs-chunker v0.0.5
 	github.com/ipfs/go-ipfs-exchange-offline v0.0.1
 	github.com/ipfs/go-ipfs-files v0.0.8
-	github.com/ipfs/go-ipld-cbor v0.0.5-0.20200204214505-252690b78669 // indirect
 	github.com/ipfs/go-ipld-format v0.2.0
 	github.com/ipfs/go-log/v2 v2.1.2
 	github.com/ipfs/go-merkledag v0.3.2
-	github.com/ipfs/go-peertaskqueue v0.2.0 // indirect
 	github.com/ipfs/go-unixfs v0.2.4
 	github.com/ipld/go-car v0.1.1-0.20201119040415-11b6074b6d4d
 	github.com/ipld/go-ipld-prime v0.5.1-0.20201021195245-109253e8a018
-	github.com/ipld/go-ipld-prime-proto v0.1.0 // indirect
-	github.com/jinzhu/gorm v1.9.16
 	github.com/libp2p/go-libp2p v0.12.0 // indirect
-	github.com/libp2p/go-libp2p-record v0.1.1 // indirect
-	github.com/libp2p/go-sockaddr v0.1.0 // indirect
-	github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a // indirect
-	github.com/smartystreets/assertions v1.0.1 // indirect
 	github.com/urfave/cli/v2 v2.3.0
 	github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2 // indirect
-	golang.org/x/lint v0.0.0-20200130185559-910be7a94367 // indirect
-	golang.org/x/tools v0.0.0-20200827010519-17fd2f27a9e3 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
-	google.golang.org/protobuf v1.25.0 // indirect
-	gopkg.in/yaml.v2 v2.3.0 // indirect
 )
+
+replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi

piece/commp.go

@@ -0,0 +1,67 @@
+package piece
+
+import (
+	"bufio"
+	"context"
+	"io"
+	"os"
+
+	"github.com/filecoin-project/go-commp-utils/ffiwrapper"
+	"github.com/filecoin-project/go-padreader"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/ipfs/go-cid"
+	"github.com/ipld/go-car"
+	"golang.org/x/xerrors"
+)
+
+type CommPRet struct {
+	Root cid.Cid
+	Size abi.UnpaddedPieceSize
+}
+
+// almost copy paste from https://github.com/filecoin-project/lotus/node/impl/client/client.go#L749-L770
+func CalcCommP(ctx context.Context, inpath string) (*CommPRet, error) {
+
+	// Hard-code the sector type to 32GiBV1_1, because:
+	// - ffiwrapper.GeneratePieceCIDFromFile requires a RegisteredSealProof
+	// - commP itself is sector-size independent, with rather low probability of that changing
+	//   ( note how the final rust call is identical for every RegSP type )
+	//   https://github.com/filecoin-project/rust-filecoin-proofs-api/blob/v5.0.0/src/seal.rs#L1040-L1050
+	//
+	// IF/WHEN this changes in the future we will have to be able to calculate
+	// "old style" commP, and thus will need to introduce a version switch or similar
+	arbitraryProofType := abi.RegisteredSealProof_StackedDrg32GiBV1_1
+
+	rdr, err := os.Open(inpath)
+	if err != nil {
+		return nil, err
+	}
+	defer rdr.Close() //nolint:errcheck
+
+	stat, err := rdr.Stat()
+	if err != nil {
+		return nil, err
+	}
+
+	// check that the data is a car file; if it's not, retrieval won't work
+	_, _, err = car.ReadHeader(bufio.NewReader(rdr))
+	if err != nil {
+		return nil, xerrors.Errorf("not a car file: %w", err)
+	}
+
+	if _, err := rdr.Seek(0, io.SeekStart); err != nil {
+		return nil, xerrors.Errorf("seek to start: %w", err)
+	}
+
+	pieceReader, pieceSize := padreader.New(rdr, uint64(stat.Size()))
+	commP, err := ffiwrapper.GeneratePieceCIDFromFile(arbitraryProofType, pieceReader, pieceSize)
+
+	if err != nil {
+		return nil, xerrors.Errorf("computing commP failed: %w", err)
+	}
+
+	return &CommPRet{
+		Root: commP,
+		Size: pieceSize,
+	}, nil
+}