Improve slack messaging (#70)

* Record and have client.GetQueueStatus report number of iRODS connections

* Slackers now log errors instead of returning them, to avoid waiting for messages to be sent.

* Include num iRODS connections in status output

* Send slack message when num irods connections changes.

* Record and on the set more fine grained uploaded status( Replaced and Skipped).

* Report new fine-grained uploaded status in status output

* Track and slack and output in status the size of actual uploads

cmd/put.go

@@ -52,6 +52,8 @@ const (
 	putMetaParts   = 2
 	putSet         = "manual"
 
+	numIRODSConnections = 2
+
 	// minMBperSecondUploadSpeed is the slowest MB/s we think an upload should
 	// take; if it drops below this and is still uploading, we'll report to the
 	// server the upload might be stuck.
@@ -184,13 +186,23 @@ func handleServerMode(started time.Time) {
 		os.Exit(0)
 	}
 
+	err = client.MakingIRODSConnections(numIRODSConnections)
+	if err != nil {
+		die(err.Error())
+	}
+
 	uploadStarts, uploadResults, skipResults, dfunc := handlePut(client, requests)
 
 	err = client.SendPutResultsToServer(uploadStarts, uploadResults, skipResults,
 		minMBperSecondUploadSpeed, minTimeForUpload, maxStuckTime, appLogger)
 
 	dfunc()
 
+	errm := client.ClosedIRODSConnections()
+	if errm != nil {
+		warn(errm.Error())
+	}
+
 	if err != nil {
 		warn("%s", err)
 

cmd/server.go

@@ -152,13 +152,18 @@ database that you've made, to investigate.
 			warn("ldap options not supplied, will assume all user passwords are correct!")
 		}
 
+		logWriter := setServerLogger(serverLogPath)
 		token := os.Getenv("IBACKUP_SLACK_TOKEN")
 		channel := os.Getenv("IBACKUP_SLACK_CHANNEL")
 
 		var slacker set.Slacker
 
 		if token != "" && channel != "" {
-			slacker = slack.New(slack.Config{Token: token, Channel: channel})
+			slacker = slack.New(slack.Config{
+				Token:       token,
+				Channel:     channel,
+				ErrorLogger: logWriter,
+			})
 		} else {
 			if serverStillRunningMsgFreq != "" {
 				die("--still_running requires slack variables")
@@ -179,8 +184,6 @@ database that you've made, to investigate.
 			}
 		}
 
-		logWriter := setServerLogger(serverLogPath)
-
 		conf := server.Config{
 			HTTPLogger:           logWriter,
 			Slacker:              slacker,

cmd/status.go

@@ -220,8 +220,8 @@ func status(client *server.Client, sf statusFilterer, user, name string, details
 func displayQueueStatus(qs *server.QStatus) {
 	info("Global put queue status: %d queued; %d reserved to be worked on; %d failed",
 		qs.Total, qs.Reserved, qs.Failed)
-	info("Global put client status (/%d): %d creating collections; %d currently uploading",
-		numPutClients, qs.CreatingCollections, qs.Uploading)
+	info("Global put client status (/%d): %d iRODS connections; %d creating collections; %d currently uploading",
+		numPutClients, qs.IRODSConnections, qs.CreatingCollections, qs.Uploading)
 
 	if qs.Stuck != nil {
 		if gasClientCLI(serverURL, serverCert).CanReadServerToken() {
@@ -336,10 +336,10 @@ func displaySet(s *set.Set, showRequesters bool) { //nolint:funlen,gocyclo
 	}
 
 	cliPrint("Discovery: %s\n", s.Discovered())
-	cliPrint("Num files: %s; Symlinks: %d; Hardlinks: %d; Size files: %s\n",
-		s.Count(), s.Symlinks, s.Hardlinks, s.Size())
-	cliPrint("Uploaded: %d; Failed: %d; Missing: %d; Abnormal: %d\n",
-		s.Uploaded, s.Failed, s.Missing, s.Abnormal)
+	cliPrint("Num files: %s; Symlinks: %d; Hardlinks: %d; Size (total/recently uploaded): %s / %s\n",
+		s.Count(), s.Symlinks, s.Hardlinks, s.Size(), s.UploadedSize())
+	cliPrint("Uploaded: %d; Replaced: %d; Skipped: %d; Failed: %d; Missing: %d; Abnormal: %d\n",
+		s.Uploaded, s.Replaced, s.Skipped, s.Failed, s.Missing, s.Abnormal)
 
 	switch s.Status {
 	case set.Complete:
@@ -406,7 +406,7 @@ func determineETADetailsFromSize(s *set.Set) (basedOn, unit string, total, done
 	remaining, speed float64, timeUnit time.Duration) {
 	basedOn = "last completed size"
 	total = s.LastCompletedSize
-	done = s.SizeFiles
+	done = s.SizeTotal
 	remaining = bytesToMB(total - done)
 
 	if done == 0 {