added syncer retry &fail

altlimit · Jan 6, 2022 · 0bdccb4 · 0bdccb4
1 parent ebbadbc
commit 0bdccb4
Show file tree

Hide file tree

Showing 3 changed files with 60 additions and 17 deletions.
diff --git a/backend/sync/sync.go b/backend/sync/sync.go
@@ -3,13 +3,15 @@ package sync
 import (
 	"fmt"
 	"log"
+	"time"
 
 	"github.com/altlimit/dmedia/model"
 )
 
 type (
 	Sync interface {
 		Valid() bool
+		CanRetry(error) bool
 		Upload(cType string, path string) (string, error)
 		Delete(meta string) error
 	}
@@ -103,29 +105,66 @@ func SyncLocation(userID int64, loc *model.SyncLocation, syncer Sync) {
 		failedDelete int
 	)
 	for _, m := range medias {
-		meta, err := syncer.Upload(m.ContentType, m.Path(userID))
+		var (
+			meta string
+			err  error
+		)
+		for i := 0; i < 3; i++ {
+			meta, err = syncer.Upload(m.ContentType, m.Path(userID))
+			if err != nil {
+				if syncer.CanRetry(err) {
+					log.Println("SyncLocation[", userID, "][", loc.ID, loc.Name, "] upload", m.ID, "error", err, "retrying in 1 minute x", i)
+					time.Sleep(time.Second * 1)
+					continue
+				}
+			}
+			break
+		}
+		as := model.SyncMedia{
+			LocationID: loc.ID,
+			MediaID:    m.ID,
+			Meta:       meta, // this is empty for permanent failure
+		}
 		if err != nil {
+			if syncer.CanRetry(err) {
+				log.Println("SyncLocation[", userID, "][", loc.ID, loc.Name, "] upload", m.ID, "error", err, "retry later")
+				continue
+			}
 			log.Println("SyncLocation[", userID, "][", loc.ID, loc.Name, "] upload", m.ID, "error", err)
 			failedUpload++
-			continue
+		} else {
+			uploaded++
+			log.Println("SyncLocation[", userID, "][", loc.ID, loc.Name, "] uploaded", m.ID, "progress", uploaded, "/", toUpload, "failed", failedUpload)
 		}
-		uploaded++
-		log.Println("SyncLocation[", userID, "][", loc.ID, loc.Name, "] uploaded", m.ID, "progress", uploaded, "/", toUpload, "failed", failedUpload)
-		addSync = append(addSync, model.SyncMedia{
-			LocationID: loc.ID,
-			MediaID:    m.ID,
-			Meta:       meta,
-		})
+		addSync = append(addSync, as)
 	}
 	for _, sm := range delMedias {
-		err = syncer.Delete(sm.Meta)
+		err = nil
+		// empty meta means never uploaded so we don't need to delete anything but the record
+		if sm.Meta != "" {
+			for i := 0; i < 3; i++ {
+				err = syncer.Delete(sm.Meta)
+				if err != nil {
+					if syncer.CanRetry(err) {
+						log.Println("SyncLocation[", userID, "][", loc.ID, loc.Name, "] delete", sm.MediaID, "error", err, "retrying in 1 minute x", i)
+						time.Sleep(time.Minute * 1)
+						continue
+					}
+				}
+				break
+			}
+		}
 		if err != nil {
+			if syncer.CanRetry(err) {
+				log.Println("SyncLocation[", userID, "][", loc.ID, loc.Name, "] delete", sm.MediaID, "error", err, "retry later")
+				continue
+			}
 			log.Println("SyncLocation[", userID, "][", loc.ID, loc.Name, "] delete", sm.MediaID, "error", err)
 			failedDelete++
-			continue
+		} else {
+			deleted++
+			log.Println("SyncLocation[", userID, "][", loc.ID, loc.Name, "] deleted", sm.MediaID, "progress", deleted, "/", toDelete, "failed", failedDelete)
 		}
-		deleted++
-		log.Println("SyncLocation[", userID, "][", loc.ID, loc.Name, "] deleted", sm.MediaID, "progress", deleted, "/", toDelete, "failed", failedDelete)
 		delSync = append(delSync, sm)
 	}
 

diff --git a/backend/sync/telegram.go b/backend/sync/telegram.go
@@ -39,6 +39,10 @@ func (t *Telegram) Valid() bool {
 	return false
 }
 
+func (t *Telegram) CanRetry(err error) bool {
+	return strings.Contains(err.Error(), "429") || strings.Contains(err.Error(), "500")
+}
+
 func (t *Telegram) getURL(method string) string {
 	return fmt.Sprintf("https://api.telegram.org/bot%s/%s", t.Token, method)
 }

diff --git a/tests/api.http b/tests/api.http
@@ -86,7 +86,7 @@ Content-Type: application/json
 
 ###
 
-DELETE {{baseUrl}}/api/syncs/3
+DELETE {{baseUrl}}/api/syncs/8
 Authorization: {{auth}}
 Content-Type: application/json
 
@@ -125,7 +125,7 @@ Content-Type: multipart/form-data; boundary=----WebKitFormBoundaryosFv4xxeUbP0jj
 Content-Disposition: form-data; name="file"; filename="test.jpg"
 Content-Type: image/jpeg
 
-< ./test2.jpg
+< ../tmp/test2.jpg
 ------WebKitFormBoundaryosFv4xxeUbP0jjyQ--
 
 ###
@@ -139,7 +139,7 @@ Content-Type: multipart/form-data; boundary=----WebKitFormBoundaryosFv4xxeUbP0jj
 Content-Disposition: form-data; name="file"; filename="test3.mp4"
 Content-Type: video/mp4
 
-< ./test3.mp4
+< ../tmp/test3.mp4
 ------WebKitFormBoundaryosFv4xxeUbP0jjyQ--
 
 ###
@@ -164,7 +164,7 @@ Authorization: {{auth}}
 
 ###
 
-DELETE {{baseUrl}}/api/media/1
+DELETE {{baseUrl}}/api/media/12
 Authorization: {{auth}}
 
 ###