Skip to content

Commit b420632

Browse files
committed
Merge branch 'master' into release/5.4
Conflicts: configure/BUILD.conf
2 parents 0b9f06c + e2a9906 commit b420632

File tree

23 files changed

+300
-70
lines changed

23 files changed

+300
-70
lines changed

configure/BUILD.conf

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
BUILD_NUMBER=1
2-
EPICS_BASE_VERSION=7.0.8
3-
BOOST_VERSION=1.81.0
4-
PVAPY_VERSION=5.4.0
2+
EPICS_BASE_VERSION=7.0.8.1
3+
BOOST_VERSION=1.85.0
4+
PVAPY_VERSION=5.4.1
55
PVAPY_GIT_VERSION=master
6+
PVAPY_USE_CPP11=1

documentation/RELEASE_NOTES.md

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,15 @@
1+
## Release 5.4.1 (2024/07/25)
2+
3+
- Fixed issue with MultiChannel class initialization
4+
- Fixed issue with numpy arrays larger than 2GB
5+
- Added support for C++11 build
6+
- Added support for OSX ARM platform
7+
- Updated fabio support in AD simulation server
8+
- Conda/pip package dependencies:
9+
- EPICS BASE = 7.0.8.1.1.pvapy (base 7.0.8.1 + pvAccessCPP PR #192 + pvDatabaseCPP PRs #82,83),
10+
- BOOST = 1.85.0
11+
- NUMPY >= 1.26, < 2.0 (for python >= 3.12); >= 1.22, < 2.0 (for python >= 3.8); >= 1.19, < 1.21 (for python < 3.8)
12+
113
## Release 5.4.0 (2024/05/31)
214

315
- Added method for PvaServer record updates via python dictionary, which

examples/pixelStatistics.py

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
#!/usr/bin/env python
2+
3+
from collections.abc import Mapping, Sequence
4+
from typing import Any, Final
5+
import ctypes
6+
import ctypes.util
7+
import os
8+
import tempfile
9+
import time
10+
11+
import numpy
12+
13+
from pvapy.hpc.adImageProcessor import AdImageProcessor
14+
from pvapy.utility.floatWithUnits import FloatWithUnits
15+
import pvaccess as pva
16+
17+
18+
def find_epics_db() -> None:
19+
if not os.environ.get('EPICS_DB_INCLUDE_PATH'):
20+
pvDataLib = ctypes.util.find_library('pvData')
21+
22+
if pvDataLib:
23+
pvDataLib = os.path.realpath(pvDataLib)
24+
epicsLibDir = os.path.dirname(pvDataLib)
25+
dbdDir = os.path.realpath(f'{epicsLibDir}/../../dbd')
26+
os.environ['EPICS_DB_INCLUDE_PATH'] = dbdDir
27+
else:
28+
raise Exception('Cannot find dbd directory, please set EPICS_DB_INCLUDE_PATH'
29+
'environment variable')
30+
31+
32+
def create_ca_ioc(pvseq: Sequence[str]) -> pva.CaIoc:
33+
# create database and start IOC
34+
dbFile = tempfile.NamedTemporaryFile(delete=False)
35+
dbFile.write(b'record(ao, "$(NAME)") {}\n')
36+
dbFile.close()
37+
38+
ca_ioc = pva.CaIoc()
39+
ca_ioc.loadDatabase('base.dbd', '', '')
40+
ca_ioc.registerRecordDeviceDriver()
41+
42+
for pv in pvseq:
43+
print(f'Creating CA ca record: {pv}')
44+
ca_ioc.loadRecords(dbFile.name, f'NAME={pv}')
45+
46+
ca_ioc.start()
47+
os.unlink(dbFile.name)
48+
return ca_ioc
49+
50+
51+
class PixelStatisticsProcessor(AdImageProcessor):
52+
DESAT_PV: Final[str] = 'pvapy:desat'
53+
DESAT_KW: Final[str] = 'desat_threshold'
54+
SAT_PV: Final[str] = 'pvapy:sat'
55+
SAT_KW: Final[str] = 'sat_threshold'
56+
SUM_PV: Final[str] = 'pvapy:sum'
57+
58+
def __init__(self, config_dict: Mapping[str, Any] = {}) -> None:
59+
super().__init__(config_dict)
60+
find_epics_db()
61+
62+
self._desat_threshold = config_dict.get(self.DESAT_KW, 1)
63+
self._sat_threshold = config_dict.get(self.SAT_KW, 254)
64+
self._ca_ioc = pva.CaIoc()
65+
66+
# statistics
67+
self.num_frames_processed = 0
68+
self.processing_time_s = 0
69+
70+
def start(self) -> None:
71+
self._ca_ioc = create_ca_ioc([self.DESAT_PV, self.SAT_PV, self.SUM_PV])
72+
self.logger.debug(self._ca_ioc.getRecordNames())
73+
74+
def configure(self, config_dict: Mapping[str, Any]) -> None:
75+
try:
76+
self._desat_threshold = int(config_dict[self.DESAT_KW])
77+
except KeyError:
78+
pass
79+
except ValueError:
80+
self.logger.warning('Failed to parse desaturation threshold!')
81+
else:
82+
self.logger.debug(f'Desaturation threshold: {self._desat_threshold}')
83+
84+
try:
85+
self._sat_threshold = int(config_dict[self.SAT_KW])
86+
except KeyError:
87+
pass
88+
except ValueError:
89+
self.logger.warning('Failed to parse saturation threshold!')
90+
else:
91+
self.logger.debug(f'Saturation threshold: {self._sat_threshold}')
92+
93+
def process(self, pvObject: pva.PvObject) -> pva.PvObject:
94+
t0 = time.time()
95+
96+
(frameId, image, nx, ny, nz, colorMode, fieldKey) = self.reshapeNtNdArray(pvObject)
97+
98+
if nx is None:
99+
self.logger.debug(f'Frame id {frameId} contains an empty image.')
100+
return pvObject
101+
102+
desat_pixels = numpy.count_nonzero(image < self._desat_threshold)
103+
self._ca_ioc.putField(self.DESAT_PV, desat_pixels)
104+
105+
sat_pixels = numpy.count_nonzero(image > self._sat_threshold)
106+
self._ca_ioc.putField(self.SAT_PV, sat_pixels)
107+
108+
sum_pixels = image.sum()
109+
self._ca_ioc.putField(self.SUM_PV, sum_pixels)
110+
111+
t1 = time.time()
112+
self.processing_time_s += (t1 - t0)
113+
114+
return pvObject
115+
116+
def stop(self) -> None:
117+
pass
118+
119+
def resetStats(self) -> None:
120+
self.num_frames_processed = 0
121+
self.processing_time_s = 0
122+
123+
def getStats(self) -> Mapping[str, Any]:
124+
processed_frame_rate_Hz = 0
125+
126+
if self.processing_time_s > 0:
127+
processed_frame_rate_Hz = self.num_frames_processed / self.processing_time_s
128+
129+
return {
130+
'num_frames_processed': self.num_frames_processed,
131+
'processing_time_s': FloatWithUnits(self.processing_time_s, 's'),
132+
'processed_frame_rate_Hz': FloatWithUnits(processed_frame_rate_Hz, 'fps'),
133+
}
134+
135+
def getStatsPvaTypes(self) -> Mapping[str, Any]:
136+
return {
137+
'num_frames_processed': pva.UINT,
138+
'processing_time_s': pva.DOUBLE,
139+
'processed_frame_rate_Hz': pva.DOUBLE,
140+
}

pvapy/cli/adSimServer.py

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@ def __init__(self, filePath, config):
139139
self.nInputFrames = 0
140140
self.rows = 0
141141
self.cols = 0
142+
self.file = None
142143
if not fabio:
143144
raise Exception('Missing fabio support.')
144145
if not filePath:
@@ -155,10 +156,12 @@ def __init__(self, filePath, config):
155156

156157
def loadInputFile(self):
157158
try:
158-
self.frames = fabio.open(self.filePath).data
159-
self.frames = np.expand_dims(self.frames, 0);
159+
self.file = fabio.open(self.filePath)
160+
self.frames = self.file.data
161+
if self.frames is not None:
162+
self.frames = np.expand_dims(self.frames, 0)
160163
print(f'Loaded input file {self.filePath}')
161-
self.nInputFrames += 1;
164+
self.nInputFrames += self.file.nframes
162165
return 1
163166
except Exception as ex:
164167
print(f'Cannot load input file {self.filePath}: {ex}, skipping it')
@@ -194,15 +197,17 @@ def getFrameData(self, frameId):
194197
frameData = np.resize(frameData, (self.cfg['file_info']['height'], self.cfg['file_info']['width']))
195198
return frameData
196199
return None
197-
# other formats: no need for other processing
198-
if frameId < self.nInputFrames:
199-
return self.frames[frameId]
200+
# other formats: one frame, just return data. Multiple frames, get selected frame.
201+
if self.nInputFrames == 1:
202+
return self.file.data
203+
elif frameId < self.nInputFrames and frameId >= 0:
204+
return self.file.getframe(frameId).data
200205
return None
201206

202207
def getFrameInfo(self):
203-
if self.frames is not None and not self.bin:
204-
frames, self.rows, self.cols = self.frames.shape
205-
self.dtype = self.frames.dtype
208+
if self.file is not None and self.frames is not None and not self.bin:
209+
self.dtype = self.file.dtype
210+
frames, self.cols, self.rows = self.frames.shape
206211
elif self.frames is not None and self.bin:
207212
self.dtype = self.frames.dtype
208213
return (self.nInputFrames, self.rows, self.cols, self.colorMode, self.dtype, self.compressorName)
@@ -262,11 +267,11 @@ def generateFrames(self):
262267
# [0,0,0,1,2,3,2,0,0,0],
263268
# [0,0,0,0,0,0,0,0,0,0]], dtype=np.uint16)
264269

265-
270+
266271
frameArraySize = (self.nf, self.ny, self.nx)
267272
if self.colorMode != AdImageUtility.COLOR_MODE_MONO:
268273
frameArraySize = (self.nf, self.ny, self.nx, 3)
269-
274+
270275
dt = np.dtype(self.datatype)
271276
if not self.datatype.startswith('float'):
272277
dtinfo = np.iinfo(dt)
@@ -358,10 +363,13 @@ def __init__(self, inputDirectory, inputFile, mmapMode, hdfDataset, hdfCompressi
358363
self.frameGeneratorList.append(NumpyRandomGenerator(nf, nx, ny, colorMode, datatype, minimum, maximum))
359364

360365
self.nInputFrames = 0
366+
multipleFrameImages = False
361367
for fg in self.frameGeneratorList:
362368
nInputFrames, self.rows, self.cols, colorMode, self.dtype, self.compressorName = fg.getFrameInfo()
369+
if nInputFrames > 1:
370+
multipleFrameImages = True
363371
self.nInputFrames += nInputFrames
364-
if self.nFrames > 0:
372+
if self.nFrames > 0 and not multipleFrameImages:
365373
self.nInputFrames = min(self.nFrames, self.nInputFrames)
366374

367375
fg = self.frameGeneratorList[0]
@@ -509,7 +517,7 @@ def getFrameFromCache(self):
509517
# Using dictionary
510518
cachedFrameId = self.currentFrameId % self.nInputFrames
511519
if cachedFrameId not in self.frameCache:
512-
# In case frames were not generated on time, just use first frame
520+
# In case frames were not generated on time, just use first frame
513521
cachedFrameId = 0
514522
ntnda = self.frameCache[cachedFrameId]
515523
else:

src/pvaccess/MultiChannel.481.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ MultiChannel::MultiChannel(const bp::list& channelNames, PvProvider::ProviderTyp
3434
, monitorThreadRunning(false)
3535
, monitorActive(false)
3636
{
37+
PvObject::initializeBoostNumPy();
38+
PyGilManager::evalInitThreads();
3739
nChannels = bp::len(channelNames);
3840
epvd::shared_vector<std::string> names(nChannels);
3941
for (unsigned int i = 0; i < nChannels; i++) {

src/pvaccess/PyPvDataUtility.h

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,14 @@
55
#define PY_PV_DATA_UTILITY_H
66

77
#include <string>
8-
#include "pv/pvData.h"
98
#include "boost/python/str.hpp"
109
#include "boost/python/extract.hpp"
1110
#include "boost/python/object.hpp"
1211
#include "boost/python/list.hpp"
1312
#include "boost/python/dict.hpp"
1413
#include "boost/python/tuple.hpp"
1514
#include "boost/shared_ptr.hpp"
15+
#include "pv/pvData.h"
1616

1717
#include "pvapy.environment.h"
1818

@@ -379,10 +379,10 @@ void booleanArrayToPyList(const epics::pvData::PVScalarArrayPtr& pvScalarArrayPt
379379
template<typename PvArrayType, typename CppType>
380380
void scalarArrayToPyList(const epics::pvData::PVScalarArrayPtr& pvScalarArrayPtr, boost::python::list& pyList)
381381
{
382-
int nDataElements = pvScalarArrayPtr->getLength();
382+
unsigned long long nDataElements = pvScalarArrayPtr->getLength();
383383
typename PvArrayType::const_svector data;
384384
pvScalarArrayPtr->PVScalarArray::template getAs<CppType>(data);
385-
for (int i = 0; i < nDataElements; ++i) {
385+
for (unsigned long long i = 0; i < nDataElements; ++i) {
386386
pyList.append(data[i]);
387387
}
388388
}
@@ -391,7 +391,7 @@ void scalarArrayToPyList(const epics::pvData::PVScalarArrayPtr& pvScalarArrayPtr
391391
template<typename PvArrayType, typename CppType>
392392
void copyScalarArrayToScalarArray(const epics::pvData::PVScalarArrayPtr& srcPvScalarArrayPtr, epics::pvData::PVScalarArrayPtr& destPvScalarArrayPtr)
393393
{
394-
int nDataElements = srcPvScalarArrayPtr->getLength();
394+
unsigned long long nDataElements = srcPvScalarArrayPtr->getLength();
395395
typename PvArrayType::const_svector data;
396396
srcPvScalarArrayPtr->PVScalarArray::template getAs<CppType>(data);
397397

@@ -403,7 +403,7 @@ void copyScalarArrayToScalarArray(const epics::pvData::PVScalarArrayPtr& srcPvSc
403403
template<typename PvArrayType, typename CppType>
404404
numpy_::ndarray getScalarArrayAsNumPyArray(const epics::pvData::PVScalarArrayPtr& pvScalarArrayPtr)
405405
{
406-
int nDataElements = pvScalarArrayPtr->getLength();
406+
unsigned long long nDataElements = pvScalarArrayPtr->getLength();
407407
typename PvArrayType::const_svector data;
408408
pvScalarArrayPtr->PVScalarArray::template getAs<CppType>(data);
409409
const CppType* arrayData = data.data();
@@ -417,7 +417,17 @@ numpy_::ndarray getScalarArrayAsNumPyArray(const epics::pvData::PVScalarArrayPtr
417417
template<typename CppType, typename NumPyType>
418418
void setScalarArrayFieldFromNumPyArrayImpl(const numpy_::ndarray& ndArray, const std::string& fieldName, epics::pvData::PVStructurePtr& pvStructurePtr)
419419
{
420-
int nDataElements = ndArray.shape(0);
420+
int nDimensions = ndArray.get_nd();
421+
unsigned long long nDataElements = 1;
422+
if (nDimensions) {
423+
for(int i = 0; i < nDimensions; i++) {
424+
nDataElements *= ndArray.shape(i);
425+
}
426+
}
427+
else {
428+
nDataElements = 0;
429+
}
430+
421431
numpy_::dtype dtype = ndArray.get_dtype();
422432
numpy_::dtype expectedDtype = numpy_::dtype::get_builtin<NumPyType>();
423433

src/pvaccess/RpcServiceImpl.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ class RpcServiceImpl : public epics::pvAccess::RPCService
1717
POINTER_DEFINITIONS(RpcServiceImpl);
1818
RpcServiceImpl(const boost::python::object& pyService);
1919
virtual ~RpcServiceImpl();
20-
epics::pvData::PVStructurePtr request(const epics::pvData::PVStructurePtr& args);
20+
virtual epics::pvData::PVStructurePtr request(const epics::pvData::PVStructurePtr& args);
2121
private:
2222
static PvaPyLogger logger;
2323
boost::python::object pyService;

src/pvaccess/StringUtility.cpp

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -33,21 +33,6 @@ std::string toString(bool b)
3333
return "false";
3434
}
3535

36-
#ifndef WINDOWS
37-
std::string& leftTrim(std::string& s)
38-
{
39-
s.erase(s.begin(), std::find_if(s.begin(), s.end(),
40-
std::not1(std::ptr_fun<int, int>(std::isspace))));
41-
return s;
42-
}
43-
44-
std::string& rightTrim(std::string& s)
45-
{
46-
s.erase(std::find_if(s.rbegin(), s.rend(),
47-
std::not1(std::ptr_fun<int, int>(std::isspace))).base(), s.end());
48-
return s;
49-
}
50-
#else
5136
std::string& leftTrim(std::string& s)
5237
{
5338
s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](int ch) {
@@ -63,7 +48,6 @@ std::string& rightTrim(std::string& s)
6348
}).base(), s.end());
6449
return s;
6550
}
66-
#endif
6751

6852
std::string& trim(std::string& s)
6953
{

tools/autoconf/configure.ac

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
1-
AC_INIT([pvaPy], [2.0.0], [[email protected]])
1+
AC_INIT([pvaPy], [5.4.0], [[email protected]])
22
AM_INIT_AUTOMAKE([-Wall -Werror foreign])
33
AC_CONFIG_FILES([Makefile])
44
AC_CONFIG_MACRO_DIR([m4])
55
AX_PYTHON_DEVEL([>=],[2.6])
6-
AX_BOOST_BASE([1.40], [], [AC_MSG_ERROR(required Boost library version >= 1.40.)])
6+
AX_BOOST_BASE([1.78], [], [AC_MSG_ERROR(required Boost library version >= 1.78.)])
77
AX_BOOST_PYTHON
88
AX_BOOST_PYTHON_NUMPY
99
AX_BOOST_NUMPY
1010
AX_EPICS_BASE([3.14.12])
11-
AX_EPICS4([4.0.3])
12-
AX_PVAPY([2.0.0])
11+
AX_EPICS4([7.0.0])
12+
AX_PVAPY([5.4.0])
1313
#AC_OUTPUT
1414

0 commit comments

Comments
 (0)