downloader: small refactoring

erigon-lib/downloader/downloader.go

@@ -1950,16 +1950,12 @@ func (d *Downloader) ReCalcStats(interval time.Duration) {
 		downloading[file] = &i
 	}
 
-	webDownloadClient := d.webDownloadClient
-
 	webDownloadInfo := map[string]webDownloadInfo{}
 
 	for key, value := range d.webDownloadInfo {
 		webDownloadInfo[key] = value
 	}
 
-	ctx := d.ctx
-
 	d.lock.RUnlock()
 
 	//Call this methods outside of `lock` critical section, because they have own locks with contention
@@ -2089,74 +2085,16 @@ func (d *Downloader) ReCalcStats(interval time.Duration) {
 
 	stats.BytesDownload = uint64(downloadedBytes)
 
-	var webTransfers int32
-
-	if webDownloadClient != nil {
-		webStats, _ := webDownloadClient.Stats(ctx)
-
-		if webStats != nil {
-			if len(webStats.Transferring) != 0 && stats.Completed {
-				stats.Completed = false
-			}
-
-			for _, transfer := range webStats.Transferring {
-				stats.MetadataReady++
-				webTransfers++
-
-				bytesCompleted := transfer.Bytes
-				tLen := transfer.Size
-				transferName := transfer.Name
-
-				delete(downloading, transferName)
-
-				if bytesCompleted > tLen {
-					bytesCompleted = tLen
-				}
-
-				stats.BytesTotal += tLen
-
-				stats.BytesDownload += bytesCompleted
+	var webTransfers int
 
-				if transfer.Percentage == 0 {
-					zeroProgress = append(zeroProgress, transferName)
-				}
-
-				var seeds []diagnostics.SegmentPeer
-				var webseedRates []interface{}
-				if peerUrl, err := url.Parse(transfer.Group); err == nil {
-					rate := uint64(transfer.SpeedAvg)
-					seeds = []diagnostics.SegmentPeer{
-						{
-							Url:          peerUrl.Host,
-							DownloadRate: rate,
-						}}
-
-					if shortUrl, err := url.JoinPath(peerUrl.Host, peerUrl.Path); err == nil {
-						webseedRates = []interface{}{strings.TrimSuffix(shortUrl, "/"), fmt.Sprintf("%s/s", common.ByteCount(rate))}
-					}
-				}
-
-				// more detailed statistic: download rate of each peer (for each file)
-				if transfer.Percentage != 0 {
-					logger.Log(verbosity, "[snapshots] progress", "file", transferName, "progress", fmt.Sprintf("%.2f%%", float32(transfer.Percentage)), "webseeds", 1)
-					logger.Log(verbosity, "[snapshots] web peers", webseedRates...)
-				}
-
-				diagnostics.Send(diagnostics.SegmentDownloadStatistics{
-					Name:            transferName,
-					TotalBytes:      tLen,
-					DownloadedBytes: bytesCompleted,
-					Webseeds:        seeds,
-				})
-			}
-		}
+	// Calculate web download stats (not used at the moment)
+	if d.webDownloadClient != nil {
+		var zp []string
+		webTransfers, zp = d.collectWebDownloadClientStats(&downloading, &stats)
+		zeroProgress = append(zeroProgress, zp...)
 	}
 
 	if len(downloading) > 0 {
-		if webDownloadClient != nil {
-			webTransfers += int32(len(downloading))
-		}
-
 		stats.Completed = false
 	}
 
@@ -2278,7 +2216,7 @@ func (d *Downloader) ReCalcStats(interval time.Duration) {
 	}
 
 	stats.PeersUnique = int32(len(peers))
-	stats.FilesTotal = int32(len(torrents)) + webTransfers
+	stats.FilesTotal = int32(len(torrents)) + int32(webTransfers)
 
 	d.lock.Lock()
 	d.stats = stats
@@ -2968,3 +2906,73 @@ func (d *Downloader) CompletedTorrents() map[string]completedTorrentInfo {
 
 	return d.completedTorrents
 }
+
+func (d *Downloader) collectWebDownloadClientStats(downloading *map[string]*downloadInfo, stats *AggStats) (int, []string) {
+	zeroProgress := make([]string, 0)
+	webTransfers := 0
+	webDownloadClient := d.webDownloadClient
+	ctx := d.ctx
+	logger := d.logger
+	verbosity := d.verbosity
+
+	webStats, _ := webDownloadClient.Stats(ctx)
+
+	if webStats != nil {
+		if len(webStats.Transferring) != 0 && stats.Completed {
+			stats.Completed = false
+		}
+
+		for _, transfer := range webStats.Transferring {
+			stats.MetadataReady++
+			webTransfers++
+
+			bytesCompleted := transfer.Bytes
+			tLen := transfer.Size
+			transferName := transfer.Name
+
+			if bytesCompleted > tLen {
+				bytesCompleted = tLen
+			}
+
+			stats.BytesTotal += tLen
+
+			stats.BytesCompleted += bytesCompleted
+
+			if transfer.Percentage == 0 {
+				zeroProgress = append(zeroProgress, transferName)
+			}
+
+			var seeds []diagnostics.SegmentPeer
+			var webseedRates []interface{}
+			if peerUrl, err := url.Parse(transfer.Group); err == nil {
+				rate := uint64(transfer.SpeedAvg)
+				seeds = []diagnostics.SegmentPeer{
+					{
+						Url:          peerUrl.Host,
+						DownloadRate: rate,
+					}}
+
+				if shortUrl, err := url.JoinPath(peerUrl.Host, peerUrl.Path); err == nil {
+					webseedRates = []interface{}{strings.TrimSuffix(shortUrl, "/"), fmt.Sprintf("%s/s", common.ByteCount(rate))}
+				}
+			}
+
+			// more detailed statistic: download rate of each peer (for each file)
+			if transfer.Percentage != 0 {
+				logger.Log(verbosity, "[snapshots] progress", "file", transferName, "progress", fmt.Sprintf("%.2f%%", float32(transfer.Percentage)), "webseeds", 1)
+				logger.Log(verbosity, "[snapshots] web peers", webseedRates...)
+			}
+
+			diagnostics.Send(diagnostics.SegmentDownloadStatistics{
+				Name:            transferName,
+				TotalBytes:      tLen,
+				DownloadedBytes: bytesCompleted,
+				Webseeds:        seeds,
+			})
+		}
+	}
+
+	webTransfers += len(*downloading)
+
+	return webTransfers, zeroProgress
+}