Skip to content

Commit

Permalink
Add summary sub-command (#53)
Browse files Browse the repository at this point in the history
* New tplot package for terminal plots.

* Update deps for sec vulns.

* Move TransformPath test to better place.

* New set.UsageSummary() method used by summary cmd.
  • Loading branch information
sb10 authored Nov 16, 2023
1 parent 554e64b commit cb1bd61
Show file tree
Hide file tree
Showing 12 changed files with 675 additions and 48 deletions.
7 changes: 4 additions & 3 deletions cmd/filestatus.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,10 @@ var filestatusCmd = &cobra.Command{
Short: "Get the status of a file in the database",
Long: `Get the status of a file in the database.
Prints out a summary of the given file for each set the file appears in.
The user who started the server can use this sub-command to print out a the
status of the given file for each set the file appears in.
The --database option should be the path to the local backup of the iBackup
The --database option should be the path to the local backup of the ibackup
database, defaulting to the value of the IBACKUP_LOCAL_DB_BACKUP_PATH
environmental variable.
Expand All @@ -51,7 +52,7 @@ func init() {
RootCmd.AddCommand(filestatusCmd)

filestatusCmd.Flags().StringVarP(&filestatusDB, "database", "d",
os.Getenv("IBACKUP_LOCAL_DB_BACKUP_PATH"), "path to iBackup database file")
os.Getenv("IBACKUP_LOCAL_DB_BACKUP_PATH"), "path to ibackup database file")
filestatusCmd.Flags().BoolVarP(&filestatusIrods, "irods", "i", false,
"do additional checking in iRods")
}
Expand Down
118 changes: 118 additions & 0 deletions cmd/summary.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
/*******************************************************************************
* Copyright (c) 2023 Genome Research Ltd.
*
* Author: Sendu Bala <[email protected]>
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be included
* in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
******************************************************************************/

package cmd

import (
"fmt"
"os"

//nolint:misspell
"github.com/dustin/go-humanize"
"github.com/spf13/cobra"
"github.com/wtsi-hgi/ibackup/set"
"github.com/wtsi-hgi/ibackup/tplot"
)

const bySetMax = 20

// options for this cmd.
var summaryDB string

// summaryCmd represents the summary command.
var summaryCmd = &cobra.Command{
Use: "summary",
Short: "Get a summary of backed up sets",
Long: `Get a summary of backed up sets.
The user who started the server can use this sub-command to summarise what has
been backed up (following 'ibackup add' calls).
The --database option should be the path to the local backup of the ibackup
database, defaulting to the value of the IBACKUP_LOCAL_DB_BACKUP_PATH
environmental variable.
The output will look nicer if you have https://github.com/red-data-tools/YouPlot
installed.
`,
Run: func(cmd *cobra.Command, args []string) {
err := summary(summaryDB)
if err != nil {
die(err.Error())
}
},
}

func init() {
RootCmd.AddCommand(summaryCmd)

summaryCmd.Flags().StringVarP(&summaryDB, "database", "d",
os.Getenv("IBACKUP_LOCAL_DB_BACKUP_PATH"), "path to ibackup database file")
}

func summary(dbPath string) error {
db, err := set.NewRO(dbPath)
if err != nil {
return err
}

sets, err := db.GetAll()
if err != nil {
return err
}

usage := set.UsageSummary(sets)

cliPrint("Total size: %s\nTotal files: %s\n",
humanize.IBytes(usage.Total.Size), humanize.Comma(int64(usage.Total.Number)))

tp := tplot.New()

err = plotSANs(tp, usage.ByRequester, 0, "By user (TB backed up)")
if err != nil {
return err
}

err = plotSANs(tp, usage.BySet, bySetMax, fmt.Sprintf("Largest %d sets (TB)", bySetMax))
if err != nil {
return err
}

return plotSANs(tp, usage.ByMonth, 0, "Backed up (TB) each month")
}

func plotSANs(tp *tplot.TPlotter, sans []*set.SizeAndNumber, max int, title string) error {
data := tplot.NewData(title)

for i, san := range sans {
data.Add(san.For, san.SizeTiB())

if max > 0 && i == max-1 {
break
}
}

return tp.Plot(data)
}
12 changes: 6 additions & 6 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ require (
github.com/wtsi-ssg/wr v0.5.7
github.com/wtsi-ssg/wrstat/v4 v4.0.2
go.etcd.io/bbolt v1.3.7
golang.org/x/term v0.8.0
golang.org/x/term v0.14.0
)

require (
Expand All @@ -49,7 +49,7 @@ require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
github.com/docker/distribution v2.8.2+incompatible // indirect
github.com/docker/docker v24.0.2+incompatible // indirect
github.com/docker/docker v24.0.7+incompatible // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/docker/spdystream v0.2.0 // indirect
Expand Down Expand Up @@ -136,13 +136,13 @@ require (
github.com/wtsi-hgi/godirwalk v1.18.1 // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect
golang.org/x/arch v0.3.0 // indirect
golang.org/x/crypto v0.9.0 // indirect
golang.org/x/crypto v0.15.0 // indirect
golang.org/x/mod v0.10.0 // indirect
golang.org/x/net v0.10.0 // indirect
golang.org/x/net v0.18.0 // indirect
golang.org/x/oauth2 v0.8.0 // indirect
golang.org/x/sync v0.2.0 // indirect
golang.org/x/sys v0.8.0 // indirect
golang.org/x/text v0.9.0 // indirect
golang.org/x/sys v0.14.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.9.2 // indirect
google.golang.org/appengine v1.6.7 // indirect
Expand Down
19 changes: 12 additions & 7 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WA
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker v24.0.2+incompatible h1:eATx+oLz9WdNVkQrr0qjQ8HvRJ4bOOxfzEo8R+dA3cg=
github.com/docker/docker v24.0.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
Expand Down Expand Up @@ -430,8 +430,9 @@ golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80=
golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA=
golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
Expand All @@ -455,8 +456,9 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8=
golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
Expand Down Expand Up @@ -498,15 +500,17 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8=
golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
Expand All @@ -515,8 +519,9 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
Expand Down
14 changes: 13 additions & 1 deletion main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ func (s *TestServer) prepareFilePaths(dir string) {
"PATH=" + os.Getenv("PATH"),
"HOME=" + home,
"IRODS_ENVIRONMENT_FILE=" + os.Getenv("IRODS_ENVIRONMENT_FILE"),
"GEM_HOME=" + os.Getenv("GEM_HOME"),
}
}

Expand Down Expand Up @@ -217,8 +218,12 @@ func (s *TestServer) runBinary(t *testing.T, args ...string) (int, string) {
if err != nil {

Check failure on line 218 in main_test.go

View workflow job for this annotation

GitHub Actions / lint

`if err != nil` has complex nested blocks (complexity: 3) (nestif)
var exitError *exec.ExitError
if !errors.As(err, &exitError) {
t.Logf("binary gave error: %s\noutput was: %s\n", err, string(outB))
t.Logf("\nbinary gave error: %s\noutput was: %s\n", err, out)
} else {
t.Logf("\nnon ExitError error: %s\noutput was: %s\n", err, out)
}
} else if cmd.ProcessState.ExitCode() != 0 {
t.Logf("\nno error, but non-0 exit; binary output: %s\n", out)
}

return cmd.ProcessState.ExitCode(), out
Expand Down Expand Up @@ -913,6 +918,7 @@ func TestPuts(t *testing.T) {

s.schedulerDeployment = schedulerDeployment
s.remoteHardlinkPrefix = filepath.Join(remotePath, "hardlinks")
s.backupFile = filepath.Join(dir, "db.bak")

s.startServer()

Expand Down Expand Up @@ -964,6 +970,12 @@ func TestPuts(t *testing.T) {

output = getRemoteMeta(remoteInode)
So(output, ShouldContainSubstring, expectedPrefix+file)

Convey("and summary tells you about the set", func() {
exitCode, out := s.runBinary(t, "summary", "--database", s.backupFile)
So(exitCode, ShouldEqual, 0)
So(out, ShouldContainSubstring, "Total size: 9 B")
})
})

Convey("Adding a failing set then re-adding it still allows retrying the failures", func() {
Expand Down
26 changes: 16 additions & 10 deletions server/discover.go
Original file line number Diff line number Diff line change
Expand Up @@ -189,16 +189,7 @@ func (s *Server) doSetDirWalks(entries []*set.Entry, given *set.Set, entriesCh c
thisEntry := entry

s.dirPool.Submit(func() {
err := s.checkAndWalkDir(dir, func(entry *walk.Dirent) error {
if !(entry.IsRegular() || entry.IsSymlink()) {
return nil
}

entriesCh <- entry

return nil
}, warnChan)

err := s.checkAndWalkDir(dir, onlyRegularAndSymlinks(entriesCh), warnChan)
errCh <- s.handleMissingDirectories(err, thisEntry, given)
})
}
Expand Down Expand Up @@ -237,6 +228,21 @@ func (s *Server) checkAndWalkDir(dir string, cb walk.PathCallback, warnChan chan
})
}

// onlyRegularAndSymlinks sends every entry found on the walk to the given
// entriesCh, except for entries that are not regular files or symlinks, which
// are silently skipped.
func onlyRegularAndSymlinks(entriesCh chan *walk.Dirent) func(entry *walk.Dirent) error {
return func(entry *walk.Dirent) error {
if !(entry.IsRegular() || entry.IsSymlink()) {
return nil
}

entriesCh <- entry

return nil
}
}

// handleMissingDirectories checks if the given error is not nil, and if so
// records in the database that the entry has problems or is missing.
func (s *Server) handleMissingDirectories(dirStatErr error, entry *set.Entry, given *set.Set) error {
Expand Down
4 changes: 2 additions & 2 deletions server/server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2424,8 +2424,8 @@ func TestServer(t *testing.T) {
ok := <-racCalled
So(ok, ShouldBeTrue)

gotSet, err := client.GetSetByID(exampleSet.Requester, exampleSet.ID())
So(err, ShouldBeNil)
gotSet, errg := client.GetSetByID(exampleSet.Requester, exampleSet.ID())
So(errg, ShouldBeNil)
So(gotSet.Status, ShouldEqual, set.PendingUpload)
So(gotSet.NumFiles, ShouldEqual, 2)
So(gotSet.Uploaded, ShouldEqual, 0)
Expand Down
7 changes: 2 additions & 5 deletions set/inode.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ import (
bolt "go.etcd.io/bbolt"
)

const transformerInodeSeparator = ":"

// getMountPoints retrieves a list of mount point paths to be used when
// determining hardlinks. The list is sorted longest first and stored on the
// server object.
Expand Down Expand Up @@ -70,16 +72,12 @@ func (d *DB) getMountPoints() error {
return nil
}

const transformerInodeSeparator = ":"

// handleInode records the inode of the given Dirent in the database, and
// returns the path to the first local file with that inode if we've seen if
// before.
func (d *DB) handleInode(tx *bolt.Tx, de *walk.Dirent, transformerID string) (string, error) {
key := d.inodeMountPointKeyFromDirent(de)

b := tx.Bucket([]byte(inodeBucket))

transformerPath := transformerID + transformerInodeSeparator + de.Path

v := b.Get(key)
Expand All @@ -88,7 +86,6 @@ func (d *DB) handleInode(tx *bolt.Tx, de *walk.Dirent, transformerID string) (st
}

files := d.decodeIMPValue(v, de.Inode)

if len(files) == 0 {
return "", b.Put(key, d.encodeToBytes([]string{transformerPath}))
}
Expand Down
Loading

0 comments on commit cb1bd61

Please sign in to comment.