HPCC-31984 Capture additional stats from CSmartRowBuffer temp files

* Capture stats such as StCycleSpillElapsedCycles and StTimeSpillElapsed from temp file * Have CSmartRowBuffer use StSizeDiskWrite from tempFileIO for noteSize (should mean actual disk size used for size tracking) Signed-off-by: Shamser Ahmed <[email protected]>
hpcc-systems · Aug 13, 2024 · 2c3b65b · 2c3b65b
1 parent 0e5fafa
commit 2c3b65b
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 1 deletion.
diff --git a/thorlcr/activities/hashdistrib/thhashdistribslave.cpp b/thorlcr/activities/hashdistrib/thhashdistribslave.cpp
@@ -1433,6 +1433,8 @@ class CDistributorBase : implements IHashDistributor, implements IExceptionHandl
 
     virtual void mergeStats(CRuntimeStatisticCollection &stats) const
     {
+        if (piperd)
+            mergeRemappedStats(stats, piperd, diskToTempStatsMap);
         sender.mergeStats(stats);
     }
     // IExceptionHandler impl.

diff --git a/thorlcr/thorutil/thbuf.cpp b/thorlcr/thorutil/thbuf.cpp
@@ -424,6 +424,13 @@ class CSmartRowBuffer: public CSimpleInterface, implements ISmartRowBuffer, impl
     {
         return this;
     }
+    virtual unsigned __int64 getStatistic(StatisticKind kind) const override
+    {
+        if (tempFileIO)
+            return tempFileIO->getStatistic(kind);
+        else
+            return 0;
+    }
 };
 
 
@@ -607,6 +614,10 @@ class CSmartRowInMemoryBuffer: public CSimpleInterface, implements ISmartRowBuff
     {
         return this;
     }
+    virtual unsigned __int64 getStatistic(StatisticKind kind) const override
+    {
+        return 0;
+    }
 };
 
 
@@ -733,6 +744,7 @@ class CCompressedSpillingRowStream: public CSimpleInterfaceOf<ISmartRowBuffer>,
     RowEntry readFromStreamMarker = { nullptr, 0, 0 };
 
     // misc
+    CRuntimeStatisticCollection inactiveStats;
     bool grouped = false; // ctor input parameter
     CriticalSection readerWriterCS;
 #ifdef STRESSTEST_SPILLING_ROWSTREAM
@@ -755,6 +767,9 @@ class CCompressedSpillingRowStream: public CSimpleInterfaceOf<ISmartRowBuffer>,
     }
     void createNextOutputStream()
     {
+        if (currentOutputIFileIO)
+            mergeRemappedStats(inactiveStats, currentOutputIFileIO, diskToTempStatsMap);
+
         VStringBuffer tmpFilename("%s.%u", baseTmpFilename.get(), writeTempFileNum++);
         trace("WRITE: writing to %s", tmpFilename.str());
         Owned<IFile> iFile = createIFile(tmpFilename);
@@ -1005,7 +1020,8 @@ class CCompressedSpillingRowStream: public CSimpleInterfaceOf<ISmartRowBuffer>,
 
     explicit CCompressedSpillingRowStream(CActivityBase *_activity, const char *_baseTmpFilename, bool _grouped, IThorRowInterfaces *rowIf, const LookAheadOptions &_options, ICompressHandler *_compressHandler)
         : activity(*_activity), baseTmpFilename(_baseTmpFilename), grouped(_grouped), options(_options), compressHandler(_compressHandler),
-          meta(rowIf->queryRowMetaData()), serializer(rowIf->queryRowSerializer()), allocator(rowIf->queryRowAllocator()), deserializer(rowIf->queryRowDeserializer())
+          meta(rowIf->queryRowMetaData()), serializer(rowIf->queryRowSerializer()), allocator(rowIf->queryRowAllocator()), deserializer(rowIf->queryRowDeserializer()),
+          inactiveStats(spillStatistics)
     {
         size32_t minSize = meta->getMinRecordSize();
 
@@ -1053,6 +1069,13 @@ class CCompressedSpillingRowStream: public CSimpleInterfaceOf<ISmartRowBuffer>,
     {
         return this;
     }
+    virtual unsigned __int64 getStatistic(StatisticKind kind) const
+    {
+        unsigned __int64 v = inactiveStats.queryStatistic(kind).get();
+        if (currentOutputIFileIO)
+            v += currentOutputIFileIO->getStatistic(kind);
+        return v;
+    }
 // IRowStream
     virtual const void *nextRow() override
     {

diff --git a/thorlcr/thorutil/thbuf.hpp b/thorlcr/thorutil/thbuf.hpp
@@ -61,6 +61,7 @@ struct LookAheadOptions : CommonBufferRowRWStreamOptions
 interface ISmartRowBuffer: extends IRowStream
 {
     virtual IRowWriter *queryWriter() = 0;
+    virtual unsigned __int64 getStatistic(StatisticKind kind) const = 0;
 };
 
 class CActivityBase;