diff --git a/cmd/filestatus.go b/cmd/filestatus.go new file mode 100644 index 0000000..d55b36b --- /dev/null +++ b/cmd/filestatus.go @@ -0,0 +1,261 @@ +package cmd + +import ( + "crypto/md5" //nolint:gosec + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sync" + + "github.com/dustin/go-humanize" //nolint:misspell + "github.com/spf13/cobra" + "github.com/wtsi-hgi/ibackup/put" + "github.com/wtsi-hgi/ibackup/set" + "github.com/wtsi-npg/extendo/v2" +) + +// options for this cmd. +var filestatusIrods bool +var filestatusDB string + +// statusCmd represents the status command. +var filestatusCmd = &cobra.Command{ + Use: "filestatus", + Short: "Get the status of a file in the database", + Long: `Get the status of a file in the database. + +Prints out a summary of the given file for each set the file appears in. + +The --database option should be the path to the local backup of the iBackup +database, defaulting to the value of the IBACKUP_LOCAL_DB_BACKUP_PATH +environmental variable. + +The --irods options will gather additional information about the file, such as +the local and remote checksums. +`, + Run: func(cmd *cobra.Command, args []string) { + if len(args) < 1 { + die("you must supply the file to be checked") + } + + err := fileSummary(filestatusDB, args[0], filestatusIrods) + if err != nil { + die(err.Error()) + } + }, +} + +func init() { + RootCmd.AddCommand(filestatusCmd) + + filestatusCmd.Flags().StringVarP(&filestatusDB, "database", "d", + os.Getenv("IBACKUP_LOCAL_DB_BACKUP_PATH"), "path to iBackup database file") + filestatusCmd.Flags().BoolVarP(&filestatusIrods, "irods", "i", false, + "do additional checking in iRods") +} + +func fileSummary(dbPath, filePath string, useIrods bool) error { + db, err := set.NewRO(dbPath) + if err != nil { + return err + } + + sets, err := db.GetAll() + if err != nil { + return err + } + + fsg := newFSG(db, filePath, useIrods) + + if fsg.baton != nil { + defer fsg.baton.StopIgnoreError() + } + + if err := fsg.printFileStatuses(sets); err != nil { + return err + } + + if !fsg.found { + cliPrint("file not found in any registered set\n") + } + + return nil +} + +type fileStatusGetter struct { + db *set.DBRO + filePath string + baton *extendo.Client + useIRods bool + found bool + + md5once sync.Once + md5sum string +} + +func newFSG(db *set.DBRO, filePath string, useIRods bool) *fileStatusGetter { + fsg := &fileStatusGetter{ + db: db, + filePath: filePath, + useIRods: useIRods, + } + + if !useIRods { + return fsg + } + + put.GetBatonHandler() //nolint:errcheck + + if client, err := extendo.FindAndStart("--unbuffered", "--no-error"); err != nil { + warn("error occurred invoking baton; disabling irods mode: %s", err) + + fsg.useIRods = false + } else { + fsg.baton = client + } + + return fsg +} + +func (fsg *fileStatusGetter) printFileStatuses(sets []*set.Set) error { + for _, set := range sets { + if err := fsg.printFileStatusIfInSet(set); err != nil { + return err + } + } + + return nil +} + +func (fsg *fileStatusGetter) printFileStatusIfInSet(s *set.Set) error { + entry, err := fsg.db.GetFileEntryForSet(s.ID(), fsg.filePath) + if err != nil { + var errr set.Error + + if ok := errors.As(err, &errr); ok && errr.Msg == set.ErrInvalidEntry { + return nil + } + + return err + } + + if entry != nil { + fsg.found = true + + if err := fsg.printFileStatus(s, entry); err != nil { + return err + } + } + + return nil +} + +func (fsg *fileStatusGetter) printFileStatus(set *set.Set, f *set.Entry) error { + dest, err := set.TransformPath(f.Path) + if err != nil { + return err + } + + lastAttemptTime, err := put.TimeToMeta(f.LastAttempt) + if err != nil { + return err + } + + cliPrint("file found in set: %s\n", set.Name) + cliPrint(" status: %s\n", f.Status) + cliPrint(" size: %s\n", humanize.IBytes(f.Size)) + cliPrint(" destination: %s\n", dest) + cliPrint(" last attempted: %s\n", lastAttemptTime) + + if f.LastError != "" { + cliPrint(" last error: %s\n", f.LastError) + } + + if fsg.useIRods { + return fsg.printIRodsStatus(f.Path, dest) + } + + return nil +} + +func (fsg *fileStatusGetter) printIRodsStatus(local, remote string) error { + file, err := fsg.baton.ListItem(extendo.Args{AVU: true, Checksum: true}, extendo.RodsItem{ + IPath: filepath.Dir(remote), + IName: filepath.Base(remote), + }) + if err != nil { + return err + } + + uploadDate, remoteMTime := fsg.getIrodsTimesFromAVUs(file.IAVUs) + + if uploadDate != "" { + cliPrint("iRods upload date: %s\n", uploadDate) + } + + if remoteMTime != "" { + localTime, err := getMTime(local) + if err != nil { + return err + } + + cliPrint(" local mTime: %s\n", localTime) + cliPrint(" iRods mTime: %s\n", remoteMTime) + } + + cliPrint(" iRods checksum: %s\n", file.IChecksum) + cliPrint(" local checksum: %s\n", fsg.calcMD5Sum(local)) + + return nil +} + +func (fsg *fileStatusGetter) getIrodsTimesFromAVUs(avus []extendo.AVU) (string, string) { + var uploadDate, remoteMTime string + + for _, avu := range avus { + if avu.Attr == put.MetaKeyDate { + uploadDate = avu.Value + } + + if avu.Attr == put.MetaKeyMtime { + remoteMTime = avu.Value + } + } + + return uploadDate, remoteMTime +} + +func getMTime(local string) (string, error) { + stat, err := os.Stat(local) + if err != nil { + return "", err + } + + return put.TimeToMeta(stat.ModTime()) +} + +func (fsg *fileStatusGetter) calcMD5Sum(path string) string { + fsg.md5once.Do(func() { + f, err := os.Open(path) + if err != nil { + fsg.md5sum = err.Error() + + return + } + + m := md5.New() //nolint:gosec + + _, err = io.Copy(m, f) + f.Close() + + if err != nil { + fsg.md5sum = err.Error() + } else { + fsg.md5sum = fmt.Sprintf("%x", m.Sum(nil)) + } + }) + + return fsg.md5sum +} diff --git a/go.mod b/go.mod index 8a9345a..0077fa4 100644 --- a/go.mod +++ b/go.mod @@ -101,7 +101,7 @@ require ( github.com/lestrrat-go/blackmagic v1.0.1 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect github.com/lestrrat-go/iter v1.0.2 // indirect - github.com/lestrrat-go/jwx v1.2.25 // indirect + github.com/lestrrat-go/jwx v1.2.26 // indirect github.com/lestrrat-go/option v1.0.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect diff --git a/go.sum b/go.sum index 180cf56..76d1307 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,7 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= +github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= @@ -239,8 +240,8 @@ github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbq github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI= github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4= github.com/lestrrat-go/jwx v1.2.18/go.mod h1:bWTBO7IHHVMtNunM8so9MT8wD+euEY1PzGEyCnuI2qM= -github.com/lestrrat-go/jwx v1.2.25 h1:tAx93jN2SdPvFn08fHNAhqFJazn5mBBOB8Zli0g0otA= -github.com/lestrrat-go/jwx v1.2.25/go.mod h1:zoNuZymNl5lgdcu6P7K6ie2QRll5HVfF4xwxBBK1NxY= +github.com/lestrrat-go/jwx v1.2.26 h1:4iFo8FPRZGDYe1t19mQP0zTRqA7n8HnJ5lkIiDvJcB0= +github.com/lestrrat-go/jwx v1.2.26/go.mod h1:MaiCdGbn3/cckbOFSCluJlJMmp9dmZm5hDuIkx8ftpQ= github.com/lestrrat-go/option v0.0.0-20210103042652-6f1ecfceda35/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= @@ -367,8 +368,9 @@ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1F github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/thanhpk/randstr v1.0.6 h1:psAOktJFD4vV9NEVb3qkhRSMvYh4ORRaj1+w/hn4B+o= github.com/thanhpk/randstr v1.0.6/go.mod h1:M/H2P1eNLZzlDwAzpkkkUvoyNNMbzRGhESZuEQk3r0U= github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw= @@ -425,7 +427,6 @@ golang.org/x/crypto v0.0.0-20201217014255-9d1352758620/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= @@ -435,6 +436,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -452,6 +454,7 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= @@ -462,6 +465,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -492,6 +496,7 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -499,6 +504,7 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -508,6 +514,7 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= @@ -521,6 +528,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.9.2 h1:UXbndbirwCAx6TULftIfie/ygDNCwxEie+IiNP1IcNc= golang.org/x/tools v0.9.2/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/main_test.go b/main_test.go index 222a6f8..5653f11 100644 --- a/main_test.go +++ b/main_test.go @@ -388,6 +388,8 @@ func TestNoServer(t *testing.T) { } func TestStatus(t *testing.T) { + const toRemote = " => /remote" + Convey("With a started server", t, func() { s := NewTestServer(t) So(s, ShouldNotBeNil) @@ -496,7 +498,7 @@ Num files: 0; Symlinks: 0; Hardlinks: 0; Size files: 0 B Uploaded: 0; Failed: 0; Missing: 0; Abnormal: 0 Completed in: 0s Directories: - `+localDir+" => /remote"+localDir) + `+localDir+toRemote+localDir) }) Convey("Given an added set with an inaccessible subfolder, print the error to the user", func() { @@ -593,7 +595,7 @@ Discovery: Num files: 4; Symlinks: 2; Hardlinks: 1; Size files: 0 B (and counting) Uploaded: 0; Failed: 0; Missing: 0; Abnormal: 0 Directories: - `+dir+" => /remote") + `+dir+toRemote) }) Convey("Sets added with friendly monitor durations show the correct monitor duration", func() { @@ -695,7 +697,38 @@ Num files: 0; Symlinks: 0; Hardlinks: 0; Size files: 0 B Uploaded: 0; Failed: 0; Missing: 0; Abnormal: 0 Completed in: 0s Directories: - `+dir+" => /remote") + `+dir+toRemote) + }) + }) + }) +} + +func TestFileStatus(t *testing.T) { + Convey("With a started server", t, func() { + s := NewTestServer(t) + So(s, ShouldNotBeNil) + + Convey("Given an added set defined with files", func() { + dir := t.TempDir() + tempTestFile, err := os.CreateTemp(dir, "testFileSet") + So(err, ShouldBeNil) + + _, err = io.WriteString(tempTestFile, dir+`/path/to/some/file +`+dir+`/path/to/other/file`) + So(err, ShouldBeNil) + + exitCode, _ := s.runBinary(t, "add", "--files", tempTestFile.Name(), + "--name", "testAddFiles", "--transformer", "prefix="+dir+":/remote") + So(exitCode, ShouldEqual, 0) + + s.waitForStatus("testAddFiles", "Status: complete", 1*time.Second) + err = s.Shutdown() + So(err, ShouldBeNil) + + Convey("You can request the status of a file in a set", func() { + exitCode, out := s.runBinary(t, "filestatus", "--database", s.dbFile, dir+"/path/to/some/file") + So(exitCode, ShouldEqual, 0) + So(out, ShouldContainSubstring, "destination: /remote/path/to/some/file") }) }) }) @@ -910,11 +943,13 @@ func TestPuts(t *testing.T) { output := getRemoteMeta(remoteFile) So(output, ShouldNotContainSubstring, "ibackup:hardlink") + const expectedPrefix = "attribute: ibackup:hardlink\nvalue: " + output = getRemoteMeta(remoteLink1) - So(output, ShouldContainSubstring, "attribute: ibackup:hardlink\nvalue: "+link1) + So(output, ShouldContainSubstring, expectedPrefix+link1) output = getRemoteMeta(remoteLink2) - So(output, ShouldContainSubstring, "attribute: ibackup:hardlink\nvalue: "+link2) + So(output, ShouldContainSubstring, expectedPrefix+link2) attrFind := "attribute: ibackup:remotehardlink\nvalue: " attrPos := strings.Index(output, attrFind) @@ -928,7 +963,7 @@ func TestPuts(t *testing.T) { So(remoteInode, ShouldStartWith, s.remoteHardlinkPrefix) output = getRemoteMeta(remoteInode) - So(output, ShouldContainSubstring, "attribute: ibackup:hardlink\nvalue: "+file) + So(output, ShouldContainSubstring, expectedPrefix+file) }) Convey("Adding a failing set then re-adding it still allows retrying the failures", func() { diff --git a/put/info.go b/put/info.go index 62c7e5d..602d288 100644 --- a/put/info.go +++ b/put/info.go @@ -62,7 +62,7 @@ func Stat(localPath string) (*ObjectInfo, error) { return nil, err } - mtime, err := timeToMeta(fi.ModTime()) + mtime, err := TimeToMeta(fi.ModTime()) if err != nil { return nil, err } @@ -83,10 +83,10 @@ func Stat(localPath string) (*ObjectInfo, error) { }, nil } -// timeToMeta converts a time to a string suitable for storing as metadata, in +// TimeToMeta converts a time to a string suitable for storing as metadata, in // a way that ObjectInfo.ModTime() will understand and be able to convert back // again. -func timeToMeta(t time.Time) (string, error) { +func TimeToMeta(t time.Time) (string, error) { b, err := t.UTC().Truncate(time.Second).MarshalText() if err != nil { return "", err diff --git a/put/request.go b/put/request.go index 1544a23..574df36 100644 --- a/put/request.go +++ b/put/request.go @@ -339,7 +339,7 @@ func cloneMap(m map[string]string) map[string]string { // addDate adds the current date to Meta, replacing any exisiting value. func (r *Request) addDate() { - date, _ := timeToMeta(time.Now()) //nolint:errcheck + date, _ := TimeToMeta(time.Now()) //nolint:errcheck r.Meta[MetaKeyDate] = date } diff --git a/set/db.go b/set/db.go index abf3ff9..f49824b 100644 --- a/set/db.go +++ b/set/db.go @@ -45,16 +45,16 @@ import ( ) type Error struct { - msg string + Msg string id string } func (e Error) Error() string { if e.id != "" { - return fmt.Sprintf("%s [%s]", e.msg, e.id) + return fmt.Sprintf("%s [%s]", e.Msg, e.id) } - return e.msg + return e.Msg } const ( @@ -87,11 +87,38 @@ const ( workerPoolSizeFiles = 16 ) +// DBRO is the read-only component of the DB struct. +type DBRO struct { + db *bolt.DB + + ch codec.Handle +} + +// NewRO returns a *DBRO that can be used to query a set database. Provide +// the path to the database file. +// +// Returns an error if database can't be opened. +func NewRO(path string) (*DBRO, error) { + boltDB, err := bolt.Open(path, dbOpenMode, &bolt.Options{ + ReadOnly: true, + OpenFile: func(name string, _ int, _ os.FileMode) (*os.File, error) { + return os.Open(name) + }, + }) + if err != nil { + return nil, err + } + + return &DBRO{ + db: boltDB, + ch: new(codec.BincHandle), + }, nil +} + // DB is used to create and query a database for storing backup sets (lists of // files a user wants to have backed up) and their backup status. type DB struct { - db *bolt.DB - ch codec.Handle + DBRO mountList []string filePool *workerpool.WorkerPool @@ -119,8 +146,10 @@ func New(path, backupPath string) (*DB, error) { } db := &DB{ - db: boltDB, - ch: new(codec.BincHandle), + DBRO: DBRO{ + db: boltDB, + ch: new(codec.BincHandle), + }, backupPath: backupPath, minTimeBetweenBackups: 1 * time.Second, @@ -201,7 +230,7 @@ func (d *DB) AddOrUpdate(set *Set) error { func updateDatabaseSetWithUserSetDetails(dbSet, userSet *Set) error { if dbSet.StartedDiscovery.After(dbSet.LastDiscovery) { - return Error{msg: ErrNoAddDuringDiscovery, id: dbSet.ID()} + return Error{Msg: ErrNoAddDuringDiscovery, id: dbSet.ID()} } dbSet.Transformer = userSet.Transformer @@ -364,7 +393,7 @@ func (d *DB) SetDirEntries(setID string, entries []*walk.Dirent) error { // the byte slice version of the set id and the sets bucket so you can easily // put the set back again after making changes. Returns an error of setID isn't // in the database. -func (d *DB) getSetByID(tx *bolt.Tx, setID string) (*Set, []byte, *bolt.Bucket, error) { +func (d *DBRO) getSetByID(tx *bolt.Tx, setID string) (*Set, []byte, *bolt.Bucket, error) { b := tx.Bucket([]byte(setsBucket)) bid := []byte(setID) @@ -556,7 +585,7 @@ func (d *DB) updateSetAfterDiscovery(setID string) (*Set, error) { return updatedSet, err } -func (d *DB) countAllFilesInSet(tx *bolt.Tx, setID string) uint64 { +func (d *DBRO) countAllFilesInSet(tx *bolt.Tx, setID string) uint64 { var numFiles uint64 cb := func([]byte) { @@ -665,7 +694,7 @@ func (d *DB) updateFileEntry(tx *bolt.Tx, setID string, r *put.Request, setDisco // getEntry finds the Entry for the given path in the given set. Returns it // along with the bucket it was in, so you can alter the Entry and put it back. // Returns an error if the entry can't be found. -func (d *DB) getEntry(tx *bolt.Tx, setID, path string) (*Entry, *bolt.Bucket, error) { +func (d *DBRO) getEntry(tx *bolt.Tx, setID, path string) (*Entry, *bolt.Bucket, error) { setsBucket := tx.Bucket([]byte(setsBucket)) var ( @@ -691,7 +720,7 @@ func (d *DB) getEntry(tx *bolt.Tx, setID, path string) (*Entry, *bolt.Bucket, er // the setsBucket for the kind (fileBucket, discoveredBucket or dirBucket) and // set ID. If it doesn't exist, just returns nil. Also returns the subbucket it // was in. The entry will have isDir true if kind is dirBucket. -func (d *DB) getEntryFromSubbucket(kind, setID, path string, setsBucket *bolt.Bucket) (*Entry, *bolt.Bucket) { +func (d *DBRO) getEntryFromSubbucket(kind, setID, path string, setsBucket *bolt.Bucket) (*Entry, *bolt.Bucket) { subBucketName := []byte(kind + separator + setID) b := setsBucket.Bucket(subBucketName) @@ -764,7 +793,7 @@ func (d *DB) removeFailedLookup(tx *bolt.Tx, setID, path string) error { } // getBucketAndKeyForFailedLookup returns our failedBucket and a lookup key. -func (d *DB) getBucketAndKeyForFailedLookup(tx *bolt.Tx, setID, path string) (*bolt.Bucket, []byte) { +func (d *DBRO) getBucketAndKeyForFailedLookup(tx *bolt.Tx, setID, path string) (*bolt.Bucket, []byte) { return tx.Bucket([]byte(failedBucket)), []byte(setID + separator + path) } @@ -831,7 +860,7 @@ func (d *DB) fixSetCounts(entry *Entry, set *Set) { } // GetAll returns all the Sets previously added to the database. -func (d *DB) GetAll() ([]*Set, error) { +func (d *DBRO) GetAll() ([]*Set, error) { var sets []*Set err := d.db.View(func(tx *bolt.Tx) error { @@ -853,7 +882,7 @@ func (d *DB) GetAll() ([]*Set, error) { // decodeSet takes a byte slice representation of a Set as stored in the db by // AddOrUpdate(), and converts it back in to a *Set. -func (d *DB) decodeSet(v []byte) *Set { +func (d *DBRO) decodeSet(v []byte) *Set { dec := codec.NewDecoderBytes(v, d.ch) var set *Set @@ -865,7 +894,7 @@ func (d *DB) decodeSet(v []byte) *Set { // GetByRequester returns all the Sets previously added to the database by the // given requester. -func (d *DB) GetByRequester(requester string) ([]*Set, error) { +func (d *DBRO) GetByRequester(requester string) ([]*Set, error) { var sets []*Set err := d.db.View(func(tx *bolt.Tx) error { @@ -891,7 +920,7 @@ func (d *DB) GetByRequester(requester string) ([]*Set, error) { // GetByNameAndRequester returns the set with the given name and requester. // // Returns nil error when no set found. -func (d *DB) GetByNameAndRequester(name, requester string) (*Set, error) { +func (d *DBRO) GetByNameAndRequester(name, requester string) (*Set, error) { sets, err := d.GetByRequester(requester) if err != nil { return nil, err @@ -908,7 +937,7 @@ func (d *DB) GetByNameAndRequester(name, requester string) (*Set, error) { // GetByID returns the Sets with the given ID previously added to the database. // Returns nil if such a set does not exist. -func (d *DB) GetByID(id string) *Set { +func (d *DBRO) GetByID(id string) *Set { var set *Set d.db.View(func(tx *bolt.Tx) error { //nolint:errcheck @@ -927,7 +956,7 @@ func (d *DB) GetByID(id string) *Set { // GetFileEntries returns all the file entries for the given set (both // SetFileEntries and SetDiscoveredEntries). -func (d *DB) GetFileEntries(setID string) ([]*Entry, error) { +func (d *DBRO) GetFileEntries(setID string) ([]*Entry, error) { entries, err := d.getEntries(setID, fileBucket) if err != nil { return nil, err @@ -941,10 +970,28 @@ func (d *DB) GetFileEntries(setID string) ([]*Entry, error) { return append(entries, entries2...), nil } +// GetFileEntryForSet returns the file entry for the given path in the given +// set. +func (d *DBRO) GetFileEntryForSet(setID, filePath string) (*Entry, error) { + var entry *Entry + + if err := d.db.View(func(tx *bolt.Tx) error { + var err error + + entry, _, err = d.getEntry(tx, setID, filePath) + + return err + }); err != nil { + return nil, err + } + + return entry, nil +} + // GetDefinedFileEntry returns the first defined file entry for the given set. // // Will return nil Entry if SetFileEntries hasn't been called. -func (d *DB) GetDefinedFileEntry(setID string) (*Entry, error) { +func (d *DBRO) GetDefinedFileEntry(setID string) (*Entry, error) { entry, err := d.getDefinedFileEntry(setID) if err != nil { return nil, err @@ -955,7 +1002,7 @@ func (d *DB) GetDefinedFileEntry(setID string) (*Entry, error) { // getEntries returns all the entries for the given set from the given sub // bucket prefix. -func (d *DB) getEntries(setID, bucketName string) ([]*Entry, error) { +func (d *DBRO) getEntries(setID, bucketName string) ([]*Entry, error) { var entries []*Entry cb := func(v []byte) { @@ -973,7 +1020,7 @@ func (d *DB) getEntries(setID, bucketName string) ([]*Entry, error) { // getEntries returns all the entries for the given set from the given sub // bucket prefix. -func (d *DB) getDefinedFileEntry(setID string) (*Entry, error) { +func (d *DBRO) getDefinedFileEntry(setID string) (*Entry, error) { var entry *Entry err := d.db.View(func(tx *bolt.Tx) error { @@ -1018,7 +1065,7 @@ func getEntriesViewFunc(tx *bolt.Tx, setID, bucketName string, cb getEntriesView // decodeEntry takes a byte slice representation of an Entry as stored in the db // by Set*Entries(), and converts it back in to an *Entry. -func (d *DB) decodeEntry(v []byte) *Entry { +func (d *DBRO) decodeEntry(v []byte) *Entry { dec := codec.NewDecoderBytes(v, d.ch) var entry *Entry @@ -1031,7 +1078,7 @@ func (d *DB) decodeEntry(v []byte) *Entry { // GetFailedEntries returns up to 10 of the file entries for the given set (both // SetFileEntries and SetDiscoveredEntries) that have a failed status. Also // returns the number of failed entries that were not returned. -func (d *DB) GetFailedEntries(setID string) ([]*Entry, int, error) { +func (d *DBRO) GetFailedEntries(setID string) ([]*Entry, int, error) { entries := make([]*Entry, 0, maxFailedEntries) skipped := 0 @@ -1055,12 +1102,12 @@ func (d *DB) GetFailedEntries(setID string) ([]*Entry, int, error) { // GetPureFileEntries returns all the file entries for the given set (only // SetFileEntries, not SetDiscoveredEntries). -func (d *DB) GetPureFileEntries(setID string) ([]*Entry, error) { +func (d *DBRO) GetPureFileEntries(setID string) ([]*Entry, error) { return d.getEntries(setID, fileBucket) } // GetDirEntries returns all the dir entries for the given set. -func (d *DB) GetDirEntries(setID string) ([]*Entry, error) { +func (d *DBRO) GetDirEntries(setID string) ([]*Entry, error) { return d.getEntries(setID, dirBucket) } diff --git a/set/inode.go b/set/inode.go index 37124ea..e3b08f8 100644 --- a/set/inode.go +++ b/set/inode.go @@ -114,7 +114,7 @@ func (d *DB) handleInode(tx *bolt.Tx, de *walk.Dirent, transformerID string) (st func splitTransformerPath(tp string) (string, string, error) { transformerID, hardlinkDest, ok := strings.Cut(tp, transformerInodeSeparator) if !ok { - return "", "", &Error{msg: ErrInvalidTransformerPath} + return "", "", &Error{Msg: ErrInvalidTransformerPath} } return transformerID, hardlinkDest, nil @@ -288,15 +288,10 @@ func getRemotePath(tx *bolt.Tx, transformerID, path string) (string, error) { v := tb.Get([]byte(transformerID)) if v == nil { - return "", &Error{msg: ErrInvalidTransformerPath} + return "", &Error{Msg: ErrInvalidTransformerPath} } s := &Set{Transformer: string(v)} - t, err := s.MakeTransformer() - if err != nil { - return "", err - } - - return t(path) + return s.TransformPath(path) } diff --git a/set/set.go b/set/set.go index b36c4b4..4932de1 100644 --- a/set/set.go +++ b/set/set.go @@ -246,6 +246,20 @@ func (s *Set) Size() string { return sfiles } +func (s *Set) TransformPath(path string) (string, error) { + transformer, err := s.MakeTransformer() + if err != nil { + return "", err + } + + dest, err := transformer(path) + if err != nil { + return "", err + } + + return dest, nil +} + // MakeTransformer turns our Transformer string in to a put.HumgenTransformer or // a put.PrefixTransformer as appropriate. func (s *Set) MakeTransformer() (put.PathTransformer, error) { diff --git a/set/set_test.go b/set/set_test.go index 3eba595..d367b47 100644 --- a/set/set_test.go +++ b/set/set_test.go @@ -244,6 +244,12 @@ func TestSetDB(t *testing.T) { retrieved2 = db.GetByID(set2.ID()) So(retrieved2.DeleteLocal, ShouldBeFalse) }) + + Convey("And transform paths for the set", func() { + dest, errr := retrieved.TransformPath("/local/sub/foo.txt") + So(errr, ShouldBeNil) + So(dest, ShouldEqual, "/remote/sub/foo.txt") + }) }) Convey("Then get all the Sets and their entries", func() { @@ -282,6 +288,16 @@ func TestSetDB(t *testing.T) { So(len(dEntries), ShouldEqual, 0) }) + Convey("The get an particular entry from a set", func() { + entry, errr := db.GetFileEntryForSet(set2.ID(), "/a/b.txt") + So(errr, ShouldBeNil) + So(entry, ShouldResemble, &Entry{Path: "/a/b.txt"}) + + entry, errr = db.GetFileEntryForSet(set2.ID(), "/not/a/file.txt") + So(errr, ShouldNotBeNil) + So(entry, ShouldBeNil) + }) + Convey("Then get all the Sets for a particular Requester", func() { sets, errg := db.GetByRequester("jim") So(errg, ShouldBeNil) @@ -1341,7 +1357,7 @@ func TestBackup(t *testing.T) { So(err, ShouldBeNil) testBackupOK := func(path string) { - backupUpDB, errn := New(path, "") + backupUpDB, errn := NewRO(path) So(errn, ShouldBeNil) So(backupUpDB, ShouldNotBeNil)