diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index 22e40c13eaa..1b8d1f642a4 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -1950,16 +1950,12 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { downloading[file] = &i } - webDownloadClient := d.webDownloadClient - webDownloadInfo := map[string]webDownloadInfo{} for key, value := range d.webDownloadInfo { webDownloadInfo[key] = value } - ctx := d.ctx - d.lock.RUnlock() //Call this methods outside of `lock` critical section, because they have own locks with contention @@ -2089,74 +2085,16 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { stats.BytesDownload = uint64(downloadedBytes) - var webTransfers int32 - - if webDownloadClient != nil { - webStats, _ := webDownloadClient.Stats(ctx) - - if webStats != nil { - if len(webStats.Transferring) != 0 && stats.Completed { - stats.Completed = false - } - - for _, transfer := range webStats.Transferring { - stats.MetadataReady++ - webTransfers++ - - bytesCompleted := transfer.Bytes - tLen := transfer.Size - transferName := transfer.Name - - delete(downloading, transferName) - - if bytesCompleted > tLen { - bytesCompleted = tLen - } - - stats.BytesTotal += tLen - - stats.BytesDownload += bytesCompleted + var webTransfers int - if transfer.Percentage == 0 { - zeroProgress = append(zeroProgress, transferName) - } - - var seeds []diagnostics.SegmentPeer - var webseedRates []interface{} - if peerUrl, err := url.Parse(transfer.Group); err == nil { - rate := uint64(transfer.SpeedAvg) - seeds = []diagnostics.SegmentPeer{ - { - Url: peerUrl.Host, - DownloadRate: rate, - }} - - if shortUrl, err := url.JoinPath(peerUrl.Host, peerUrl.Path); err == nil { - webseedRates = []interface{}{strings.TrimSuffix(shortUrl, "/"), fmt.Sprintf("%s/s", common.ByteCount(rate))} - } - } - - // more detailed statistic: download rate of each peer (for each file) - if transfer.Percentage != 0 { - logger.Log(verbosity, "[snapshots] progress", "file", transferName, "progress", fmt.Sprintf("%.2f%%", float32(transfer.Percentage)), "webseeds", 1) - logger.Log(verbosity, "[snapshots] web peers", webseedRates...) - } - - diagnostics.Send(diagnostics.SegmentDownloadStatistics{ - Name: transferName, - TotalBytes: tLen, - DownloadedBytes: bytesCompleted, - Webseeds: seeds, - }) - } - } + // Calculate web download stats (not used at the moment) + if d.webDownloadClient != nil { + var zp []string + webTransfers, zp = d.collectWebDownloadClientStats(&downloading, &stats) + zeroProgress = append(zeroProgress, zp...) } if len(downloading) > 0 { - if webDownloadClient != nil { - webTransfers += int32(len(downloading)) - } - stats.Completed = false } @@ -2278,7 +2216,7 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { } stats.PeersUnique = int32(len(peers)) - stats.FilesTotal = int32(len(torrents)) + webTransfers + stats.FilesTotal = int32(len(torrents)) + int32(webTransfers) d.lock.Lock() d.stats = stats @@ -2968,3 +2906,73 @@ func (d *Downloader) CompletedTorrents() map[string]completedTorrentInfo { return d.completedTorrents } + +func (d *Downloader) collectWebDownloadClientStats(downloading *map[string]*downloadInfo, stats *AggStats) (int, []string) { + zeroProgress := make([]string, 0) + webTransfers := 0 + webDownloadClient := d.webDownloadClient + ctx := d.ctx + logger := d.logger + verbosity := d.verbosity + + webStats, _ := webDownloadClient.Stats(ctx) + + if webStats != nil { + if len(webStats.Transferring) != 0 && stats.Completed { + stats.Completed = false + } + + for _, transfer := range webStats.Transferring { + stats.MetadataReady++ + webTransfers++ + + bytesCompleted := transfer.Bytes + tLen := transfer.Size + transferName := transfer.Name + + if bytesCompleted > tLen { + bytesCompleted = tLen + } + + stats.BytesTotal += tLen + + stats.BytesCompleted += bytesCompleted + + if transfer.Percentage == 0 { + zeroProgress = append(zeroProgress, transferName) + } + + var seeds []diagnostics.SegmentPeer + var webseedRates []interface{} + if peerUrl, err := url.Parse(transfer.Group); err == nil { + rate := uint64(transfer.SpeedAvg) + seeds = []diagnostics.SegmentPeer{ + { + Url: peerUrl.Host, + DownloadRate: rate, + }} + + if shortUrl, err := url.JoinPath(peerUrl.Host, peerUrl.Path); err == nil { + webseedRates = []interface{}{strings.TrimSuffix(shortUrl, "/"), fmt.Sprintf("%s/s", common.ByteCount(rate))} + } + } + + // more detailed statistic: download rate of each peer (for each file) + if transfer.Percentage != 0 { + logger.Log(verbosity, "[snapshots] progress", "file", transferName, "progress", fmt.Sprintf("%.2f%%", float32(transfer.Percentage)), "webseeds", 1) + logger.Log(verbosity, "[snapshots] web peers", webseedRates...) + } + + diagnostics.Send(diagnostics.SegmentDownloadStatistics{ + Name: transferName, + TotalBytes: tLen, + DownloadedBytes: bytesCompleted, + Webseeds: seeds, + }) + } + } + + webTransfers += len(*downloading) + + return webTransfers, zeroProgress +}