Merge branch 'master' into release/5.4

Conflicts:
	configure/BUILD.conf

configure/BUILD.conf

@@ -1,5 +1,6 @@
 BUILD_NUMBER=1
-EPICS_BASE_VERSION=7.0.8
-BOOST_VERSION=1.81.0
-PVAPY_VERSION=5.4.0
+EPICS_BASE_VERSION=7.0.8.1
+BOOST_VERSION=1.85.0
+PVAPY_VERSION=5.4.1
 PVAPY_GIT_VERSION=master
+PVAPY_USE_CPP11=1

documentation/RELEASE_NOTES.md

@@ -1,3 +1,15 @@
+## Release 5.4.1 (2024/07/25)
+
+- Fixed issue with MultiChannel class initialization
+- Fixed issue with numpy arrays larger than 2GB
+- Added support for C++11 build
+- Added support for OSX ARM platform
+- Updated fabio support in AD simulation server
+- Conda/pip package dependencies: 
+  - EPICS BASE = 7.0.8.1.1.pvapy (base 7.0.8.1 + pvAccessCPP PR #192 + pvDatabaseCPP PRs #82,83), 
+  - BOOST      = 1.85.0
+  - NUMPY      >= 1.26, < 2.0 (for python >= 3.12); >= 1.22, < 2.0 (for python >= 3.8); >= 1.19, < 1.21 (for python < 3.8)
+
 ## Release 5.4.0 (2024/05/31)
 
 - Added method for PvaServer record updates via python dictionary, which

examples/pixelStatistics.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python
+
+from collections.abc import Mapping, Sequence
+from typing import Any, Final
+import ctypes
+import ctypes.util
+import os
+import tempfile
+import time
+
+import numpy
+
+from pvapy.hpc.adImageProcessor import AdImageProcessor
+from pvapy.utility.floatWithUnits import FloatWithUnits
+import pvaccess as pva
+
+
+def find_epics_db() -> None:
+    if not os.environ.get('EPICS_DB_INCLUDE_PATH'):
+        pvDataLib = ctypes.util.find_library('pvData')
+
+        if pvDataLib:
+            pvDataLib = os.path.realpath(pvDataLib)
+            epicsLibDir = os.path.dirname(pvDataLib)
+            dbdDir = os.path.realpath(f'{epicsLibDir}/../../dbd')
+            os.environ['EPICS_DB_INCLUDE_PATH'] = dbdDir
+        else:
+            raise Exception('Cannot find dbd directory, please set EPICS_DB_INCLUDE_PATH'
+                            'environment variable')
+
+
+def create_ca_ioc(pvseq: Sequence[str]) -> pva.CaIoc:
+    # create database and start IOC
+    dbFile = tempfile.NamedTemporaryFile(delete=False)
+    dbFile.write(b'record(ao, "$(NAME)") {}\n')
+    dbFile.close()
+
+    ca_ioc = pva.CaIoc()
+    ca_ioc.loadDatabase('base.dbd', '', '')
+    ca_ioc.registerRecordDeviceDriver()
+
+    for pv in pvseq:
+        print(f'Creating CA ca record: {pv}')
+        ca_ioc.loadRecords(dbFile.name, f'NAME={pv}')
+
+    ca_ioc.start()
+    os.unlink(dbFile.name)
+    return ca_ioc
+
+
+class PixelStatisticsProcessor(AdImageProcessor):
+    DESAT_PV: Final[str] = 'pvapy:desat'
+    DESAT_KW: Final[str] = 'desat_threshold'
+    SAT_PV: Final[str] = 'pvapy:sat'
+    SAT_KW: Final[str] = 'sat_threshold'
+    SUM_PV: Final[str] = 'pvapy:sum'
+
+    def __init__(self, config_dict: Mapping[str, Any] = {}) -> None:
+        super().__init__(config_dict)
+        find_epics_db()
+
+        self._desat_threshold = config_dict.get(self.DESAT_KW, 1)
+        self._sat_threshold = config_dict.get(self.SAT_KW, 254)
+        self._ca_ioc = pva.CaIoc()
+
+        # statistics
+        self.num_frames_processed = 0
+        self.processing_time_s = 0
+
+    def start(self) -> None:
+        self._ca_ioc = create_ca_ioc([self.DESAT_PV, self.SAT_PV, self.SUM_PV])
+        self.logger.debug(self._ca_ioc.getRecordNames())
+
+    def configure(self, config_dict: Mapping[str, Any]) -> None:
+        try:
+            self._desat_threshold = int(config_dict[self.DESAT_KW])
+        except KeyError:
+            pass
+        except ValueError:
+            self.logger.warning('Failed to parse desaturation threshold!')
+        else:
+            self.logger.debug(f'Desaturation threshold: {self._desat_threshold}')
+
+        try:
+            self._sat_threshold = int(config_dict[self.SAT_KW])
+        except KeyError:
+            pass
+        except ValueError:
+            self.logger.warning('Failed to parse saturation threshold!')
+        else:
+            self.logger.debug(f'Saturation threshold: {self._sat_threshold}')
+
+    def process(self, pvObject: pva.PvObject) -> pva.PvObject:
+        t0 = time.time()
+
+        (frameId, image, nx, ny, nz, colorMode, fieldKey) = self.reshapeNtNdArray(pvObject)
+
+        if nx is None:
+            self.logger.debug(f'Frame id {frameId} contains an empty image.')
+            return pvObject
+
+        desat_pixels = numpy.count_nonzero(image < self._desat_threshold)
+        self._ca_ioc.putField(self.DESAT_PV, desat_pixels)
+
+        sat_pixels = numpy.count_nonzero(image > self._sat_threshold)
+        self._ca_ioc.putField(self.SAT_PV, sat_pixels)
+
+        sum_pixels = image.sum()
+        self._ca_ioc.putField(self.SUM_PV, sum_pixels)
+
+        t1 = time.time()
+        self.processing_time_s += (t1 - t0)
+
+        return pvObject
+
+    def stop(self) -> None:
+        pass
+
+    def resetStats(self) -> None:
+        self.num_frames_processed = 0
+        self.processing_time_s = 0
+
+    def getStats(self) -> Mapping[str, Any]:
+        processed_frame_rate_Hz = 0
+
+        if self.processing_time_s > 0:
+            processed_frame_rate_Hz = self.num_frames_processed / self.processing_time_s
+
+        return {
+            'num_frames_processed': self.num_frames_processed,
+            'processing_time_s': FloatWithUnits(self.processing_time_s, 's'),
+            'processed_frame_rate_Hz': FloatWithUnits(processed_frame_rate_Hz, 'fps'),
+        }
+
+    def getStatsPvaTypes(self) -> Mapping[str, Any]:
+        return {
+            'num_frames_processed': pva.UINT,
+            'processing_time_s': pva.DOUBLE,
+            'processed_frame_rate_Hz': pva.DOUBLE,
+        }

pvapy/cli/adSimServer.py

@@ -139,6 +139,7 @@ def __init__(self, filePath, config):
         self.nInputFrames = 0
         self.rows = 0
         self.cols = 0
+        self.file = None
         if not fabio:
             raise Exception('Missing fabio support.')
         if not filePath:
@@ -155,10 +156,12 @@ def __init__(self, filePath, config):
 
     def loadInputFile(self):
         try:
-            self.frames = fabio.open(self.filePath).data
-            self.frames = np.expand_dims(self.frames, 0);
+            self.file = fabio.open(self.filePath)
+            self.frames = self.file.data
+            if self.frames is not None:
+                self.frames = np.expand_dims(self.frames, 0)
             print(f'Loaded input file {self.filePath}')
-            self.nInputFrames += 1;
+            self.nInputFrames += self.file.nframes
             return 1
         except Exception as ex:
             print(f'Cannot load input file {self.filePath}: {ex}, skipping it')
@@ -194,15 +197,17 @@ def getFrameData(self, frameId):
                 frameData = np.resize(frameData, (self.cfg['file_info']['height'], self.cfg['file_info']['width']))
                 return frameData
             return None
-        # other formats: no need for other processing
-        if frameId < self.nInputFrames:
-            return self.frames[frameId]
+        # other formats: one frame, just return data. Multiple frames, get selected frame.
+        if self.nInputFrames == 1:
+            return self.file.data
+        elif frameId < self.nInputFrames and frameId >= 0:
+            return self.file.getframe(frameId).data
         return None
 
     def getFrameInfo(self):
-        if self.frames is not None and not self.bin:
-            frames, self.rows, self.cols = self.frames.shape
-            self.dtype = self.frames.dtype
+        if self.file is not None and self.frames is not None and not self.bin:
+            self.dtype = self.file.dtype
+            frames, self.cols, self.rows = self.frames.shape
         elif self.frames is not None and self.bin:
             self.dtype = self.frames.dtype
         return (self.nInputFrames, self.rows, self.cols, self.colorMode, self.dtype, self.compressorName)
@@ -262,11 +267,11 @@ def generateFrames(self):
         #                  [0,0,0,1,2,3,2,0,0,0],
         #                  [0,0,0,0,0,0,0,0,0,0]], dtype=np.uint16)
 
-  
+
         frameArraySize = (self.nf, self.ny, self.nx)
         if self.colorMode != AdImageUtility.COLOR_MODE_MONO:
             frameArraySize = (self.nf, self.ny, self.nx, 3)
-            
+
         dt = np.dtype(self.datatype)
         if not self.datatype.startswith('float'):
             dtinfo = np.iinfo(dt)
@@ -358,10 +363,13 @@ def __init__(self, inputDirectory, inputFile, mmapMode, hdfDataset, hdfCompressi
             self.frameGeneratorList.append(NumpyRandomGenerator(nf, nx, ny, colorMode, datatype, minimum, maximum))
 
         self.nInputFrames = 0
+        multipleFrameImages = False
         for fg in self.frameGeneratorList:
             nInputFrames, self.rows, self.cols, colorMode, self.dtype, self.compressorName = fg.getFrameInfo()
+            if nInputFrames > 1:
+                multipleFrameImages = True
             self.nInputFrames += nInputFrames
-        if self.nFrames > 0:
+        if self.nFrames > 0 and not multipleFrameImages:
             self.nInputFrames = min(self.nFrames, self.nInputFrames)
 
         fg = self.frameGeneratorList[0]
@@ -509,7 +517,7 @@ def getFrameFromCache(self):
             # Using dictionary
             cachedFrameId = self.currentFrameId % self.nInputFrames
             if cachedFrameId not in self.frameCache:
-            # In case frames were not generated on time, just use first frame
+                # In case frames were not generated on time, just use first frame
                 cachedFrameId = 0
             ntnda = self.frameCache[cachedFrameId]
         else:

src/pvaccess/MultiChannel.481.cpp

@@ -34,6 +34,8 @@ MultiChannel::MultiChannel(const bp::list& channelNames, PvProvider::ProviderTyp
     , monitorThreadRunning(false)
     , monitorActive(false)
 {
+    PvObject::initializeBoostNumPy();
+    PyGilManager::evalInitThreads();
     nChannels = bp::len(channelNames);
     epvd::shared_vector<std::string> names(nChannels);
     for (unsigned int i = 0; i < nChannels; i++) {

src/pvaccess/PyPvDataUtility.h

@@ -5,14 +5,14 @@
 #define PY_PV_DATA_UTILITY_H
 
 #include <string>
-#include "pv/pvData.h"
 #include "boost/python/str.hpp"
 #include "boost/python/extract.hpp"
 #include "boost/python/object.hpp"
 #include "boost/python/list.hpp"
 #include "boost/python/dict.hpp"
 #include "boost/python/tuple.hpp"
 #include "boost/shared_ptr.hpp"
+#include "pv/pvData.h"
 
 #include "pvapy.environment.h"
 
@@ -379,10 +379,10 @@ void booleanArrayToPyList(const epics::pvData::PVScalarArrayPtr& pvScalarArrayPt
 template<typename PvArrayType, typename CppType>
 void scalarArrayToPyList(const epics::pvData::PVScalarArrayPtr& pvScalarArrayPtr, boost::python::list& pyList) 
 {
-    int nDataElements = pvScalarArrayPtr->getLength();
+    unsigned long long nDataElements = pvScalarArrayPtr->getLength();
     typename PvArrayType::const_svector data;
     pvScalarArrayPtr->PVScalarArray::template getAs<CppType>(data);
-    for (int i = 0; i < nDataElements; ++i) {
+    for (unsigned long long i = 0; i < nDataElements; ++i) {
         pyList.append(data[i]);
     }
 }
@@ -391,7 +391,7 @@ void scalarArrayToPyList(const epics::pvData::PVScalarArrayPtr& pvScalarArrayPtr
 template<typename PvArrayType, typename CppType>
 void copyScalarArrayToScalarArray(const epics::pvData::PVScalarArrayPtr& srcPvScalarArrayPtr, epics::pvData::PVScalarArrayPtr& destPvScalarArrayPtr)
 {
-    int nDataElements = srcPvScalarArrayPtr->getLength();
+    unsigned long long nDataElements = srcPvScalarArrayPtr->getLength();
     typename PvArrayType::const_svector data;
     srcPvScalarArrayPtr->PVScalarArray::template getAs<CppType>(data);
 
@@ -403,7 +403,7 @@ void copyScalarArrayToScalarArray(const epics::pvData::PVScalarArrayPtr& srcPvSc
 template<typename PvArrayType, typename CppType>
 numpy_::ndarray getScalarArrayAsNumPyArray(const epics::pvData::PVScalarArrayPtr& pvScalarArrayPtr)
 {
-    int nDataElements = pvScalarArrayPtr->getLength();
+    unsigned long long nDataElements = pvScalarArrayPtr->getLength();
     typename PvArrayType::const_svector data;
     pvScalarArrayPtr->PVScalarArray::template getAs<CppType>(data);
     const CppType* arrayData = data.data();
@@ -417,7 +417,17 @@ numpy_::ndarray getScalarArrayAsNumPyArray(const epics::pvData::PVScalarArrayPtr
 template<typename CppType, typename NumPyType>
 void setScalarArrayFieldFromNumPyArrayImpl(const numpy_::ndarray& ndArray, const std::string& fieldName, epics::pvData::PVStructurePtr& pvStructurePtr)
 {
-    int nDataElements = ndArray.shape(0);
+    int nDimensions = ndArray.get_nd();
+    unsigned long long nDataElements = 1;
+    if (nDimensions) {
+        for(int i = 0; i < nDimensions; i++) {
+            nDataElements *= ndArray.shape(i);
+        }
+    }
+    else {
+        nDataElements = 0;
+    }
+
     numpy_::dtype dtype = ndArray.get_dtype();
     numpy_::dtype expectedDtype = numpy_::dtype::get_builtin<NumPyType>();
 

src/pvaccess/RpcServiceImpl.h

@@ -17,7 +17,7 @@ class RpcServiceImpl : public epics::pvAccess::RPCService
     POINTER_DEFINITIONS(RpcServiceImpl);
     RpcServiceImpl(const boost::python::object& pyService);
     virtual ~RpcServiceImpl();
-    epics::pvData::PVStructurePtr request(const epics::pvData::PVStructurePtr& args);
+    virtual epics::pvData::PVStructurePtr request(const epics::pvData::PVStructurePtr& args);
 private:
     static PvaPyLogger logger;
     boost::python::object pyService;

src/pvaccess/StringUtility.cpp

@@ -33,21 +33,6 @@ std::string toString(bool b)
     return "false";
 }
 
-#ifndef WINDOWS
-std::string& leftTrim(std::string& s) 
-{
-    s.erase(s.begin(), std::find_if(s.begin(), s.end(),
-        std::not1(std::ptr_fun<int, int>(std::isspace))));
-    return s;
-}
-
-std::string& rightTrim(std::string& s) 
-{
-    s.erase(std::find_if(s.rbegin(), s.rend(), 
-        std::not1(std::ptr_fun<int, int>(std::isspace))).base(), s.end());
-    return s;
-}
-#else
 std::string& leftTrim(std::string& s) 
 {
     s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](int ch) {
@@ -63,7 +48,6 @@ std::string& rightTrim(std::string& s)
     }).base(), s.end());
     return s;
 }
-#endif
 
 std::string& trim(std::string& s) 
 {

tools/autoconf/configure.ac

@@ -1,14 +1,14 @@
-AC_INIT([pvaPy], [2.0.0], [[email protected]])
+AC_INIT([pvaPy], [5.4.0], [[email protected]])
 AM_INIT_AUTOMAKE([-Wall -Werror foreign])
 AC_CONFIG_FILES([Makefile])
 AC_CONFIG_MACRO_DIR([m4])
 AX_PYTHON_DEVEL([>=],[2.6])
-AX_BOOST_BASE([1.40], [], [AC_MSG_ERROR(required Boost library version >= 1.40.)])
+AX_BOOST_BASE([1.78], [], [AC_MSG_ERROR(required Boost library version >= 1.78.)])
 AX_BOOST_PYTHON
 AX_BOOST_PYTHON_NUMPY
 AX_BOOST_NUMPY
 AX_EPICS_BASE([3.14.12])
-AX_EPICS4([4.0.3])
-AX_PVAPY([2.0.0])
+AX_EPICS4([7.0.0])
+AX_PVAPY([5.4.0])
 #AC_OUTPUT