Skip to content

Commit a277b3c

Browse files
ironMannironMann
committed
stfb: implement readout interface v2
requires Readout v1.5
1 parent 1bde5ca commit a277b3c

15 files changed

Lines changed: 249 additions & 232 deletions

‎script/datadist_start_standalone.sh.in‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ STF_BUILDER="StfBuilder"
9999
STF_BUILDER+=" --transport shmem"
100100
STF_BUILDER+=" --mq-config $chainConfig"
101101
STF_BUILDER+=" --detector TPC"
102-
STF_BUILDER+=" --detector 4"
102+
STF_BUILDER+=" --detector-rdh 6"
103103
STF_BUILDER+=" --io-threads $IO_THREADS"
104104
STF_BUILDER+=" --max-buffered-stfs 2"
105105

‎script/discovery-flp-epn-chain.json‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
"transport": "shmem",
1010
"sockets": [
1111
{
12-
"type": "pair",
12+
"type": "push",
1313
"method": "bind",
1414
"address": "ipc:///tmp/readout-pipe-0"
1515
}
@@ -25,7 +25,7 @@
2525
"transport": "shmem",
2626
"sockets": [
2727
{
28-
"type": "pair",
28+
"type": "pull",
2929
"method": "connect",
3030
"address": "ipc:///tmp/readout-pipe-0",
3131
"rateLogging": "1"
@@ -37,7 +37,7 @@
3737
"transport": "shmem",
3838
"sockets": [
3939
{
40-
"type": "pair",
40+
"type": "push",
4141
"method": "bind",
4242
"address": "ipc:///tmp/stf-builder-pipe-0",
4343
"rateLogging": "1"
@@ -49,7 +49,7 @@
4949
"transport": "shmem",
5050
"sockets": [
5151
{
52-
"type": "pair",
52+
"type": "pull",
5353
"method": "connect",
5454
"address": "ipc:///tmp/stf-builder-dpl-pipe-0",
5555
"rateLogging": "1"
@@ -66,7 +66,7 @@
6666
"transport": "shmem",
6767
"sockets": [
6868
{
69-
"type": "pair",
69+
"type": "pull",
7070
"method": "connect",
7171
"address": "ipc:///tmp/stf-builder-pipe-0"
7272
}
@@ -83,7 +83,7 @@
8383
"transport": "shmem",
8484
"sockets": [
8585
{
86-
"type": "pair",
86+
"type": "push",
8787
"method": "bind",
8888
"address": "ipc:///tmp/readout-pipe-1"
8989
}
@@ -99,7 +99,7 @@
9999
"transport": "shmem",
100100
"sockets": [
101101
{
102-
"type": "pair",
102+
"type": "pull",
103103
"method": "connect",
104104
"address": "ipc:///tmp/readout-pipe-1"
105105
}
@@ -110,7 +110,7 @@
110110
"transport": "shmem",
111111
"sockets": [
112112
{
113-
"type": "pair",
113+
"type": "push",
114114
"method": "bind",
115115
"address": "ipc:///tmp/stf-builder-pipe-1"
116116
}
@@ -121,7 +121,7 @@
121121
"transport": "shmem",
122122
"sockets": [
123123
{
124-
"type": "pair",
124+
"type": "pull",
125125
"method": "connect",
126126
"address": "ipc:///tmp/stf-builder-dpl-pipe-1"
127127
}
@@ -137,7 +137,7 @@
137137
"transport": "shmem",
138138
"sockets": [
139139
{
140-
"type": "pair",
140+
"type": "pull",
141141
"method": "connect",
142142
"address": "ipc:///tmp/stf-builder-pipe-1"
143143
}
@@ -154,7 +154,7 @@
154154
"transport": "shmem",
155155
"sockets": [
156156
{
157-
"type": "pair",
157+
"type": "push",
158158
"method": "bind",
159159
"address": "ipc:///tmp/readout-pipe-2"
160160
}
@@ -170,7 +170,7 @@
170170
"transport": "shmem",
171171
"sockets": [
172172
{
173-
"type": "pair",
173+
"type": "pull",
174174
"method": "connect",
175175
"address": "ipc:///tmp/readout-pipe-2"
176176
}
@@ -181,7 +181,7 @@
181181
"transport": "shmem",
182182
"sockets": [
183183
{
184-
"type": "pair",
184+
"type": "push",
185185
"method": "bind",
186186
"address": "ipc:///tmp/stf-builder-pipe-2"
187187
}
@@ -192,7 +192,7 @@
192192
"transport": "shmem",
193193
"sockets": [
194194
{
195-
"type": "pair",
195+
"type": "pull",
196196
"method": "connect",
197197
"address": "ipc:///tmp/stf-builder-dpl-pipe-2"
198198
}
@@ -208,7 +208,7 @@
208208
"transport": "shmem",
209209
"sockets": [
210210
{
211-
"type": "pair",
211+
"type": "pull",
212212
"method": "connect",
213213
"address": "ipc:///tmp/stf-builder-pipe-2"
214214
}
@@ -226,7 +226,7 @@
226226
"transport": "zeromq",
227227
"sockets": [
228228
{
229-
"type": "pair",
229+
"type": "pull",
230230
"method": "connect",
231231
"address": "ipc:///tmp/tf-builder-dpl-pipe-0",
232232
"rateLogging": "1"
@@ -243,7 +243,7 @@
243243
"transport": "zeromq",
244244
"sockets": [
245245
{
246-
"type": "pair",
246+
"type": "pull",
247247
"method": "connect",
248248
"address": "ipc:///tmp/tf-builder-dpl-pipe-1",
249249
"rateLogging": "1"
@@ -260,7 +260,7 @@
260260
"transport": "zeromq",
261261
"sockets": [
262262
{
263-
"type": "pair",
263+
"type": "pull",
264264
"method": "connect",
265265
"address": "ipc:///tmp/tf-builder-dpl-pipe-2",
266266
"rateLogging": "1"

‎script/readout_cfg/readout_emu.cfg‎

Lines changed: 21 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -20,21 +20,9 @@ rate=-1
2020
exitTimeout=-1
2121

2222

23-
disableAggregatorSlicing=1
24-
25-
26-
###################################
27-
# data sampling
28-
###################################
29-
30-
[sampling]
31-
32-
# enable/disable data sampling (1/0)
33-
enabled=0
34-
35-
# which class of datasampling to use (FairInjector, MockInjector)
36-
class=MockInjector
37-
23+
aggregatorSliceTimeout=1
24+
aggregatorStfTimeout=2
25+
disableAggregatorSlicing=0
3826

3927

4028
###################################
@@ -55,7 +43,7 @@ equipmentType=dummy
5543
enabled=0
5644
eventMaxSize=20000
5745
eventMinSize=10000
58-
memPoolElementSize=64000
46+
memPoolElementSize=6400
5947

6048
[equipment-dummy-2]
6149
name=dummy-2
@@ -101,12 +89,12 @@ cardId=86:00.0
10189

10290
# collect data statistics
10391
[consumer-stats]
104-
enabled=0
105-
monitoringUpdatePeriod=5
106-
monitoringEnabled=0
107-
consoleUpdate=1
10892
consumerType=stats
109-
monitoringConfig=file:/etc/monitoring.cfg
93+
enabled=1
94+
monitoringUpdatePeriod=1
95+
monitoringEnabled=1
96+
consoleUpdate=1
97+
11098

11199

112100
# recording to file
@@ -170,23 +158,23 @@ disableSending=0
170158

171159
# equipment that emulates CRU data format (RDH in 8k blocks, LHC clocks, etc)
172160
[equipment-emulator-1]
173-
equipmentType=cruEmulator
174-
linkId=100
175161
name=emulator-1
176-
numberOfLinks=1
162+
equipmentType=cruEmulator
163+
enabled=0
164+
linkId=1
165+
numberOfLinks=2
177166
memoryPoolNumberOfPages=10000
178167
memoryPoolPageSize=128k
179-
enabled=1
180168

181169
[equipment-emulator-2]
182170
name=emulator-2
183171
equipmentType=cruEmulator
184-
enabled=0
185-
eventMaxSize=8000
186-
eventMinSize=8000
187-
idleSleepTime=100
188-
memoryPoolNumberOfPages=100
189-
memoryPoolPageSize=3200k
190-
numberOfLinks=16
191-
linkId=200
172+
enabled=1
173+
linkId=2
174+
numberOfLinks=4
175+
PayloadSize=4096
176+
EmptyHbRatio=0.01
177+
#idleSleepTime=100
178+
memoryPoolNumberOfPages=10000
179+
memoryPoolPageSize=128k
192180

‎src/ReadoutEmulator/CruEmulator.cxx‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020

2121
#include <unistd.h>
2222

23+
#include <Headers/DAQID.h>
24+
2325
namespace o2
2426
{
2527
namespace DataDistribution
@@ -86,10 +88,11 @@ void CruLinkEmulator::linkReadoutThread()
8688
// Each channel is reported separately to the O2
8789
ReadoutLinkO2Data linkO2Data;
8890

89-
linkO2Data.mLinkDataHeader.dataOrigin = (rand() % 100 < 70) ? o2::header::gDataOriginTPC : o2::header::gDataOriginITS;
90-
linkO2Data.mLinkDataHeader.dataDescription = o2::header::gDataDescriptionRawData;
91-
linkO2Data.mLinkDataHeader.payloadSerializationMethod = o2::header::gSerializationMethodNone;
92-
linkO2Data.mLinkDataHeader.subSpecification = mLinkID;
91+
linkO2Data.mLinkHeader.mSystemId = (rand() % 100 < 70) ? o2::header::DAQID::TPC : o2::header::DAQID::ITS;
92+
linkO2Data.mLinkHeader.mFeeId = 0xFEE0;
93+
linkO2Data.mLinkHeader.mEquipmentId = 0xE1D0; // ?
94+
linkO2Data.mLinkHeader.mLinkId = mLinkID;
95+
linkO2Data.mLinkHeader.mFlags.mIsRdhFormat = 1;
9396

9497
for (unsigned d = 0; d < cNumDmaChunkPerSuperpage; d++, lHbfToSend--) {
9598

@@ -121,15 +124,13 @@ void CruLinkEmulator::linkReadoutThread()
121124
});
122125
}
123126

124-
// record how many chunks are there in a superpage
125-
linkO2Data.mLinkDataHeader.payloadSize = linkO2Data.mLinkRawData.size();
126127
// Put the link info data into the send queue
127128
mMemHandler->putLinkData(std::move(linkO2Data));
128129

129130
} else {
130131
// signal lost data (no free superpages)
131132
ReadoutLinkO2Data linkO2Data;
132-
linkO2Data.mLinkDataHeader.subSpecification = -1;
133+
linkO2Data.mLinkHeader.mFlags.mIsRdhFormat = 0;
133134

134135
mMemHandler->putLinkData(std::move(linkO2Data));
135136
break;

‎src/ReadoutEmulator/CruMemoryHandler.cxx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ void CruMemoryHandler::put_data_buffer(const char* dataBufferAddr, const std::si
137137
std::lock_guard<std::mutex> lock(lBucket.mLock);
138138

139139
if (lBucket.mUsedSuperPages.count(lSpStartAddr) == 0) {
140-
DDLOGF(fair::Severity::ERROR, "Returned data buffer is not in the list of used superpages!");
140+
DDLOGF_RL(200, fair::Severity::ERROR, "Returned data buffer is not in the list of used superpages!");
141141
return;
142142
}
143143

‎src/ReadoutEmulator/CruMemoryHandler.h‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,7 @@ struct CruDmaPacket {
4949
};
5050

5151
struct ReadoutLinkO2Data {
52-
53-
o2::header::DataHeader mLinkDataHeader;
52+
ReadoutSubTimeframeHeader mLinkHeader;
5453
std::vector<CruDmaPacket> mLinkRawData;
5554
};
5655

‎src/ReadoutEmulator/ReadoutDevice.cxx‎

Lines changed: 19 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -123,19 +123,12 @@ void ReadoutDevice::SendingThread()
123123
// finish an STF every ~1/45 seconds
124124
static const auto cDataTakingStart = std::chrono::high_resolution_clock::now();
125125
static constexpr auto cStfInterval = std::chrono::microseconds(22810);
126-
static uint64_t lNumberSentStfs = 0;
127-
static uint64_t lCurrentTfId = 0;
126+
uint64_t lNumberSentStfs = 0;
127+
uint64_t lCurrentTfId = std::chrono::duration_cast<std::chrono::microseconds>(
128+
std::chrono::high_resolution_clock::now().time_since_epoch()).count() / cStfInterval.count();;
128129

129130
while (IsRunningState()) {
130131

131-
auto isStfFinished =
132-
(std::chrono::high_resolution_clock::now() - cDataTakingStart) - (lNumberSentStfs * cStfInterval) > cStfInterval;
133-
134-
if (isStfFinished) {
135-
lNumberSentStfs += 1;
136-
lCurrentTfId = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now().time_since_epoch()).count() / cStfInterval.count();
137-
}
138-
139132
ReadoutLinkO2Data lCruLinkData;
140133
if (!mCruMemoryHandler->getLinkData(lCruLinkData)) {
141134
DDLOG(fair::Severity::INFO) << "GetLinkData failed. Stopping interface thread.";
@@ -145,15 +138,26 @@ void ReadoutDevice::SendingThread()
145138
mFreeSuperpagesSamples.Fill(mCruMemoryHandler->free_superpages());
146139

147140
// check no data signal
148-
if (lCruLinkData.mLinkDataHeader.subSpecification ==
149-
o2::header::DataHeader::SubSpecificationType(-1)) {
141+
if (lCruLinkData.mLinkHeader.mFlags.mIsRdhFormat == 0) {
150142
// DDLOG(fair::Severity::WARN) << "No Superpages left! Losing data...";
151143
}
152144

153-
ReadoutSubTimeframeHeader lHBFHeader;
145+
ReadoutSubTimeframeHeader lHBFHeader = lCruLinkData.mLinkHeader;
154146
lHBFHeader.mTimeFrameId = lCurrentTfId;
155-
lHBFHeader.mNumberHbf = lCruLinkData.mLinkRawData.size();
156-
lHBFHeader.mLinkId = lCruLinkData.mLinkDataHeader.subSpecification;
147+
lHBFHeader.mTimeframeOrbitFirst = lCurrentTfId * 256;
148+
lHBFHeader.mTimeframeOrbitLast = (lCurrentTfId + 1) * 256 - 1;
149+
lHBFHeader.mFlags.mLastTFMessage = 0;
150+
151+
// last one?
152+
auto isStfFinished =
153+
(std::chrono::high_resolution_clock::now() - cDataTakingStart) - (lNumberSentStfs * cStfInterval) > cStfInterval;
154+
155+
if (isStfFinished) {
156+
lNumberSentStfs += 1;
157+
lCurrentTfId = std::chrono::duration_cast<std::chrono::microseconds>(
158+
std::chrono::high_resolution_clock::now().time_since_epoch()).count() / cStfInterval.count();
159+
lHBFHeader.mFlags.mLastTFMessage = 1;
160+
}
157161

158162
assert(mDataBlockMsgs.empty());
159163
mDataBlockMsgs.reserve(lCruLinkData.mLinkRawData.size());

‎src/StfBuilder/StfBuilderDevice.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ class StfBuilderDevice : public DataDistDevice,
151151
// DROP policy in StfBuilder is to keep most current STFs. This will ensure that all
152152
// StfBuilders have the same set of STFs ready for distribution
153153

154-
DDLOGF(fair::Severity::WARNING, "Dropping oldest STF due to reaching the maximum number of buffered "
154+
DDLOGF_RL(500, fair::Severity::WARNING, "Dropping oldest STF due to reaching the maximum number of buffered "
155155
"STFs in the process ({}). Consider increasing the limit, or reducing the input data rate.",
156156
I().mMaxStfsInPipeline);
157157

0 commit comments

Comments
 (0)