From 7f150622e6ef0037217dac4c6268171adec63ea1 Mon Sep 17 00:00:00 2001 From: wiechula Date: Tue, 25 Feb 2025 18:20:02 +0100 Subject: [PATCH 1/2] Restructure pedestal file creation, add DCS CCDB publishing --- .../base/include/TPCBase/CRUCalibHelpers.h | 15 +++- Detectors/TPC/base/src/CRUCalibHelpers.cxx | 51 ++++++++++++- .../calibration/macro/preparePedestalFiles.C | 71 ++--------------- .../TPCWorkflow/CalDetMergerPublisherSpec.h | 2 +- .../src/CalDetMergerPublisherSpec.cxx | 76 +++++++++++++++++-- .../TPC/workflow/src/tpc-calib-pad-raw.cxx | 4 +- 6 files changed, 141 insertions(+), 78 deletions(-) diff --git a/Detectors/TPC/base/include/TPCBase/CRUCalibHelpers.h b/Detectors/TPC/base/include/TPCBase/CRUCalibHelpers.h index 818a9a41d0dcc..0d6351a326742 100644 --- a/Detectors/TPC/base/include/TPCBase/CRUCalibHelpers.h +++ b/Detectors/TPC/base/include/TPCBase/CRUCalibHelpers.h @@ -93,10 +93,8 @@ constexpr float fixedSizeToFloat(uint32_t value) /// write values of map to fileName /// template -void writeValues(const std::string_view fileName, const DataMap& map, bool onlyFilled = false) +void writeValues(std::ostream& str, const DataMap& map, bool onlyFilled = false) { - std::ofstream str(fileName.data(), std::ofstream::out); - for (const auto& [linkInfo, data] : map) { if (onlyFilled) { if (!std::accumulate(data.begin(), data.end(), uint32_t(0))) { @@ -117,6 +115,13 @@ void writeValues(const std::string_view fileName, const DataMap& map, bool onlyF } } +template +void writeValues(const std::string_view fileName, const DataMap& map, bool onlyFilled = false) +{ + std::ofstream str(fileName.data(), std::ofstream::out); + writeValues(str, map, onlyFilled); +} + template struct is_map { static constexpr bool value = false; @@ -126,7 +131,8 @@ template struct is_map> { static constexpr bool value = true; }; -/// fill cal pad object from HV data map + +/// fill cal pad object from HW data map /// TODO: Function to be tested template typename std::enable_if_t::value, void> @@ -251,6 +257,7 @@ o2::tpc::CalDet getCalPad(const std::string_view fileName, const std::str /// \param minADCROCType can be either one value for all ROC types, or {IROC, OROC}, or {IROC, OROC1, OROC2, OROC3} std::unordered_map preparePedestalFiles(const CalPad& pedestals, const CalPad& noise, std::vector sigmaNoiseROCType = {3, 3, 3, 3}, std::vector minADCROCType = {2, 2, 2, 2}, float pedestalOffset = 0, bool onlyFilled = false, bool maskBad = true, float noisyChannelThreshold = 1.5, float sigmaNoiseNoisyChannels = 4, float badChannelThreshold = 6, bool fixedSize = false); +DataMapU32 getDataMap(const CalPad& calPad); } // namespace o2::tpc::cru_calib_helpers #endif diff --git a/Detectors/TPC/base/src/CRUCalibHelpers.cxx b/Detectors/TPC/base/src/CRUCalibHelpers.cxx index f18baa6571f66..0ae2823078b1f 100644 --- a/Detectors/TPC/base/src/CRUCalibHelpers.cxx +++ b/Detectors/TPC/base/src/CRUCalibHelpers.cxx @@ -130,9 +130,6 @@ std::unordered_map cru_calib_helpers::preparePedestalFiles( pedestalsThreshold["PedestalsPhys"] = CalPad("Pedestals"); pedestalsThreshold["ThresholdMapPhys"] = CalPad("ThresholdMap"); - auto& pedestalsCRU = pedestalsThreshold["Pedestals"]; - auto& thresholdCRU = pedestalsThreshold["ThresholdMap"]; - // ===| prepare values |=== for (size_t iroc = 0; iroc < pedestals.getData().size(); ++iroc) { const ROC roc(iroc); @@ -179,7 +176,7 @@ std::unordered_map cru_calib_helpers::preparePedestalFiles( } float noise = std::abs(rocNoise.getValue(ipad)); // it seems with the new fitting procedure, the noise can also be negative, since in gaus sigma is quadratic - float noiseCorr = noise - (0.847601 + 0.031514 * traceLength); + const float noiseCorr = noise - (0.847601 + 0.031514 * traceLength); if ((pedestal <= 0) || (pedestal > 150) || (noise <= 0) || (noise > 50)) { LOGP(info, "Bad pedestal or noise value in ROC {:2}, CRU {:3}, fec in CRU: {:2}, SAMPA: {}, channel: {:2}, pedestal: {:.4f}, noise {:.4f}", iroc, cruID, fecInPartition, sampa, sampaChannel, pedestal, noise); if (maskBad) { @@ -230,3 +227,49 @@ std::unordered_map cru_calib_helpers::preparePedestalFiles( return pedestalsThreshold; } + +cru_calib_helpers::DataMapU32 cru_calib_helpers::getDataMap(const CalPad& calPad) +{ + const auto& mapper = Mapper::instance(); + + DataMapU32 dataMap; + + for (size_t iroc = 0; iroc < calPad.getData().size(); ++iroc) { + const ROC roc(iroc); + + const auto& calRoc = calPad.getCalArray(iroc); + + const int padOffset = roc.isOROC() ? mapper.getPadsInIROC() : 0; + + // skip empty ROCs + if (!(std::abs(calRoc.getSum()) > 0)) { + continue; + } + + // loop over pads + for (size_t ipad = 0; ipad < calRoc.getData().size(); ++ipad) { + const int globalPad = ipad + padOffset; + const FECInfo& fecInfo = mapper.fecInfo(globalPad); + const CRU cru = mapper.getCRU(roc.getSector(), globalPad); + const uint32_t region = cru.region(); + const int cruID = cru.number(); + const int sampa = fecInfo.getSampaChip(); + const int sampaChannel = fecInfo.getSampaChannel(); + + const PartitionInfo& partInfo = mapper.getMapPartitionInfo()[cru.partition()]; + const int nFECs = partInfo.getNumberOfFECs(); + const int fecOffset = (nFECs + 1) / 2; + const int fecInPartition = fecInfo.getIndex() - partInfo.getSectorFECOffset(); + const int dataWrapperID = fecInPartition >= fecOffset; + const int globalLinkID = (fecInPartition % fecOffset) + dataWrapperID * 12; + + + const int hwChannel = getHWChannel(sampa, sampaChannel, region % 2); + + const auto value = calRoc.getValue(ipad); + dataMap[LinkInfo(cruID, globalLinkID)][hwChannel] = floatToFixedSize(value); + } + } + + return dataMap; +} diff --git a/Detectors/TPC/calibration/macro/preparePedestalFiles.C b/Detectors/TPC/calibration/macro/preparePedestalFiles.C index 0dcd02b64551a..92bc1456e48d7 100644 --- a/Detectors/TPC/calibration/macro/preparePedestalFiles.C +++ b/Detectors/TPC/calibration/macro/preparePedestalFiles.C @@ -60,72 +60,17 @@ void preparePedestalFiles(const std::string_view pedestalFile, std::string outpu f.GetObject("Noise", calNoise); } - DataMapU32 pedestalValues; - DataMapU32 thresholdlValues; - DataMapU32 pedestalValuesPhysics; - DataMapU32 thresholdlValuesPhysics; - auto pedestalsThreshold = preparePedestalFiles(*calPedestal, *calNoise, sigmaNoiseROCType, minADCROCType, pedestalOffset, onlyFilled, maskBad, noisyChannelThreshold, sigmaNoiseNoisyChannels, badChannelThreshold); - // ===| prepare values |=== - for (size_t iroc = 0; iroc < calPedestal->getData().size(); ++iroc) { - const ROC roc(iroc); - - const auto& rocPedestal = calPedestal->getCalArray(iroc); - const auto& rocNoise = calNoise->getCalArray(iroc); - auto& rocOut = output.getCalArray(iroc); + const auto& pedestals = pedestalsThreshold["Pedestals"]; + const auto& thresholds = pedestalsThreshold["ThresholdMap"]; + const auto& pedestalsPhys = pedestalsThreshold["PedestalsPhys"]; + const auto& thresholdsPhys = pedestalsThreshold["ThresholdMapPhys"]; - const int padOffset = roc.isOROC() ? mapper.getPadsInIROC() : 0; - - // skip empty - if (!(std::abs(rocPedestal.getSum() + rocNoise.getSum()) > 0)) { - continue; - } - - // loop over pads - for (size_t ipad = 0; ipad < rocPedestal.getData().size(); ++ipad) { - const int globalPad = ipad + padOffset; - const FECInfo& fecInfo = mapper.fecInfo(globalPad); - const CRU cru = mapper.getCRU(roc.getSector(), globalPad); - const uint32_t region = cru.region(); - const int cruID = cru.number(); - const int sampa = fecInfo.getSampaChip(); - const int sampaChannel = fecInfo.getSampaChannel(); - // int globalLinkID = fecInfo.getIndex(); - - const PartitionInfo& partInfo = mapper.getMapPartitionInfo()[cru.partition()]; - const int nFECs = partInfo.getNumberOfFECs(); - const int fecOffset = (nFECs + 1) / 2; - const int fecInPartition = fecInfo.getIndex() - partInfo.getSectorFECOffset(); - const int dataWrapperID = fecInPartition >= fecOffset; - const int globalLinkID = (fecInPartition % fecOffset) + dataWrapperID * 12; - - const auto pedestal = pedestalsThreshold["Pedestals"].getCalArray(iroc).getValue(ipad); - const auto threshold = pedestalsThreshold["ThresholdMap"].getCalArray(iroc).getValue(ipad); - const auto pedestalHighNoise = pedestalsThreshold["PedestalsPhys"].getCalArray(iroc).getValue(ipad); - const auto thresholdHighNoise = pedestalsThreshold["ThresholdMapPhys"].getCalArray(iroc).getValue(ipad); - - const int hwChannel = getHWChannel(sampa, sampaChannel, region % 2); - // for debugging - // printf("%4d %4d %4d %4d %4d: %u\n", cru.number(), globalLinkID, hwChannel, fecInfo.getSampaChip(), fecInfo.getSampaChannel(), getADCValue(pedestal)); - - // default thresholds - const auto adcPedestal = floatToFixedSize(pedestal); - const auto adcThreshold = floatToFixedSize(threshold); - pedestalValues[LinkInfo(cruID, globalLinkID)][hwChannel] = adcPedestal; - thresholdlValues[LinkInfo(cruID, globalLinkID)][hwChannel] = adcThreshold; - - // higher thresholds for physics data taking - const auto adcPedestalPhysics = floatToFixedSize(pedestalHighNoise); - const auto adcThresholdPhysics = floatToFixedSize(thresholdHighNoise); - pedestalValuesPhysics[LinkInfo(cruID, globalLinkID)][hwChannel] = adcPedestalPhysics; - thresholdlValuesPhysics[LinkInfo(cruID, globalLinkID)][hwChannel] = adcThresholdPhysics; - // for debugging - // if(!(std::abs(pedestal - fixedSizeToFloat(adcPedestal)) <= 0.5 * 0.25)) { - // printf("%4d %4d %4d %4d %4d: %u %.2f %.4f %.4f\n", cru.number(), globalLinkID, hwChannel, sampa, sampaChannel, adcPedestal, fixedSizeToFloat(adcPedestal), pedestal, pedestal - fixedSizeToFloat(adcPedestal)); - //} - } - } + auto pedestalValues = getDataMap(pedestals); + auto thresholdlValues = getDataMap(thresholds); + auto pedestalValuesPhysics = getDataMap(pedestalsPhys); + auto thresholdlValuesPhysics = getDataMap(thresholdsPhys); // text files const auto outFilePedestalTXT(outputDir + "/pedestal_values.txt"); diff --git a/Detectors/TPC/workflow/include/TPCWorkflow/CalDetMergerPublisherSpec.h b/Detectors/TPC/workflow/include/TPCWorkflow/CalDetMergerPublisherSpec.h index 9d365700582b3..b16ef8777193a 100644 --- a/Detectors/TPC/workflow/include/TPCWorkflow/CalDetMergerPublisherSpec.h +++ b/Detectors/TPC/workflow/include/TPCWorkflow/CalDetMergerPublisherSpec.h @@ -23,7 +23,7 @@ namespace o2 namespace tpc { -o2::framework::DataProcessorSpec getCalDetMergerPublisherSpec(uint32_t lanes, bool skipCCDB, bool dumpAfterComplete = false); +o2::framework::DataProcessorSpec getCalDetMergerPublisherSpec(uint32_t lanes, bool skipCCDB, bool sendToDCS, bool dumpAfterComplete = false); } // namespace tpc } // namespace o2 diff --git a/Detectors/TPC/workflow/src/CalDetMergerPublisherSpec.cxx b/Detectors/TPC/workflow/src/CalDetMergerPublisherSpec.cxx index 9b1e08de521a2..a504ffa606b84 100644 --- a/Detectors/TPC/workflow/src/CalDetMergerPublisherSpec.cxx +++ b/Detectors/TPC/workflow/src/CalDetMergerPublisherSpec.cxx @@ -19,6 +19,7 @@ #include #include #include +#include #include @@ -37,6 +38,7 @@ #include "CCDB/CcdbObjectInfo.h" #include "TPCBase/CDBInterface.h" #include "TPCBase/CalDet.h" +#include "TPCBase/CRUCalibHelpers.h" #include "TPCWorkflow/CalibRawPartInfo.h" #include "TPCWorkflow/CalDetMergerPublisherSpec.h" #include "TPCWorkflow/ProcessingHelpers.h" @@ -52,7 +54,7 @@ class CalDetMergerPublisherSpec : public o2::framework::Task using CcdbObjectInfo = o2::ccdb::CcdbObjectInfo; public: - CalDetMergerPublisherSpec(uint32_t lanes, bool skipCCDB, bool dumpAfterComplete = false) : mLanesToExpect(lanes), mCalibInfos(lanes), mSkipCCDB(skipCCDB), mPublishAfterComplete(dumpAfterComplete) {} + CalDetMergerPublisherSpec(uint32_t lanes, bool skipCCDB, bool sendToDCS, bool dumpAfterComplete = false) : mLanesToExpect(lanes), mCalibInfos(lanes), mSkipCCDB(skipCCDB), mSendToDCS(sendToDCS), mPublishAfterComplete(dumpAfterComplete) {} void init(o2::framework::InitContext& ic) final { @@ -154,10 +156,12 @@ class CalDetMergerPublisherSpec : public o2::framework::Task CDBType mCalDetMapType; ///< calibration type of CalDetMap object uint64_t mRunNumber{0}; ///< processed run number uint32_t mLanesToExpect{0}; ///< number of expected lanes sending data + uint32_t mDCSSpecOffset{32768}; ///< offset for DCS specs bool mForceQuit{false}; ///< for quit after processing finished bool mDirectFileDump{false}; ///< directly dump the calibration data to file bool mPublishAfterComplete{false}; ///< dump calibration directly after data from all lanes received bool mSkipCCDB{false}; ///< skip sending of calibration data + bool mSendToDCS{false}; ///< skip sending of calibration data bool mCheckCalibInfos{false}; ///< check calib infos //____________________________________________________________________________ @@ -170,7 +174,6 @@ class CalDetMergerPublisherSpec : public o2::framework::Task } // perhaps should be changed to time of the run - const auto now = std::chrono::system_clock::now(); const long timeStart = mCalibInfos[0].tfIDInfo.creation + mCalibInfos[0].publishCycle; const long timeEnd = o2::ccdb::CcdbObjectInfo::INFINITE_TIMESTAMP; @@ -193,6 +196,11 @@ class CalDetMergerPublisherSpec : public o2::framework::Task o2::header::DataHeader::SubSpecificationType subSpec{(o2::header::DataHeader::SubSpecificationType)mCalDetMapType}; output.snapshot(Output{clbUtils::gDataOriginCDBPayload, "TPC_CALIB", subSpec}, *image.get()); output.snapshot(Output{clbUtils::gDataOriginCDBWrapper, "TPC_CALIB", subSpec}, w); + + // for pedestal calibration send to DCS if requested + if (mSendToDCS && (mCalDetMapType == CDBType::CalPedestalNoise)) { + sendPedestalNoiseToDCS(output); + } } for (auto& [type, object] : mMergedCalDets) { @@ -238,9 +246,62 @@ class CalDetMergerPublisherSpec : public o2::framework::Task } } } + + void sendPedestalNoiseToDCS(DataAllocator& output) + { + auto sendObject = [this, &output](const CalPad& data, const std::string& path, const std::string& fileNameBase = "") { + const long timeStart = mCalibInfos[0].tfIDInfo.creation + mCalibInfos[0].publishCycle; + const long timeEnd = o2::ccdb::CcdbObjectInfo::INFINITE_TIMESTAMP; + + const auto dataMap = cru_calib_helpers::getDataMap(data); + std::ostringstream dataStr; + cru_calib_helpers::writeValues(dataStr, dataMap); + + std::vector dataVec; + const auto& str = dataStr.str(); + std::copy(str.begin(), str.end(), std::back_inserter(dataVec)); + + o2::ccdb::CcdbObjectInfo w; + + w.setPath(path); + w.setFileName(fmt::format("{}_{}_{}.txt", fileNameBase, mRunNumber, timeStart)); + w.setStartValidityTimestamp(timeStart); + w.setEndValidityTimestamp(timeEnd); + + auto md = w.getMetaData(); + md[o2::base::NameConf::CCDBRunTag.data()] = std::to_string(mRunNumber); + w.setMetaData(md); + + LOGP(info, "Sending object to DCS DB {}/{} of size {} ({}) bytes, valid for {} : {}", w.getPath(), w.getFileName(), dataVec.size(), dataStr.str().size(), w.getStartValidityTimestamp(), w.getEndValidityTimestamp()); + + o2::header::DataHeader::SubSpecificationType subSpec{(o2::header::DataHeader::SubSpecificationType)mCalDetMapType + mDCSSpecOffset}; + output.snapshot(Output{clbUtils::gDataOriginCDBPayload, "TPC_CALIB_DCS", subSpec}, dataVec); + output.snapshot(Output{clbUtils::gDataOriginCDBWrapper, "TPC_CALIB_DCS", subSpec}, w); + }; + + const auto& pedestals = mMergedCalDetsMap.at("Pedestals"); + const auto& noise = mMergedCalDetsMap.at("Noise"); + + bool first = true; + for (auto threshold : {2.5f, 3.f, 3.5f}) { + auto pedestalsThreshold = cru_calib_helpers::preparePedestalFiles(pedestals, noise, {threshold}); + + // pedestals don't depend on threshold, publish on first iteration only + if (first) { + const auto& pedestalsPhys = pedestalsThreshold["PedestalsPhys"]; + sendObject(pedestalsPhys, "TPC/Calib/PedestalsPhys", "Pedestals"); + } + + const auto& thresholdsPhys = pedestalsThreshold["ThresholdMapPhys"]; + const auto fileNameBase = fmt::format("ThresholdsPhys-{:.0f}", threshold * 10); + sendObject(thresholdsPhys, "TPC/Calib/" + fileNameBase, fileNameBase); + + first = false; + } + } }; -o2::framework::DataProcessorSpec o2::tpc::getCalDetMergerPublisherSpec(uint32_t lanes, bool skipCCDB, bool dumpAfterComplete) +o2::framework::DataProcessorSpec o2::tpc::getCalDetMergerPublisherSpec(uint32_t lanes, bool skipCCDB, bool sendToDCS, bool dumpAfterComplete) { std::vector outputs; if (!skipCCDB) { @@ -248,6 +309,11 @@ o2::framework::DataProcessorSpec o2::tpc::getCalDetMergerPublisherSpec(uint32_t outputs.emplace_back(ConcreteDataTypeMatcher{clbUtils::gDataOriginCDBWrapper, "TPC_CALIB"}, Lifetime::Sporadic); } + if (sendToDCS) { + outputs.emplace_back(ConcreteDataTypeMatcher{clbUtils::gDataOriginCDBPayload, "TPC_CALIB_DCS"}, Lifetime::Sporadic); + outputs.emplace_back(ConcreteDataTypeMatcher{clbUtils::gDataOriginCDBWrapper, "TPC_CALIB_DCS"}, Lifetime::Sporadic); + } + std::vector inputs; inputs.emplace_back("clbPayload", ConcreteDataTypeMatcher{gDataOriginTPC, "CLBPART"}, Lifetime::Sporadic); inputs.emplace_back("clbInfo", ConcreteDataTypeMatcher{gDataOriginTPC, "CLBPARTINFO"}, Lifetime::Sporadic); @@ -258,11 +324,11 @@ o2::framework::DataProcessorSpec o2::tpc::getCalDetMergerPublisherSpec(uint32_t id.data(), inputs, outputs, - AlgorithmSpec{adaptFromTask(lanes, skipCCDB, dumpAfterComplete)}, + AlgorithmSpec{adaptFromTask(lanes, skipCCDB, sendToDCS, dumpAfterComplete)}, Options{ {"force-quit", VariantType::Bool, false, {"force quit after max-events have been reached"}}, {"direct-file-dump", VariantType::Bool, false, {"directly dump calibration to file"}}, {"check-calib-infos", VariantType::Bool, false, {"make consistency check of calib infos"}}, } // end Options - }; // end DataProcessorSpec + }; // end DataProcessorSpec } diff --git a/Detectors/TPC/workflow/src/tpc-calib-pad-raw.cxx b/Detectors/TPC/workflow/src/tpc-calib-pad-raw.cxx index b58a5f5f84a51..9130e70ace157 100644 --- a/Detectors/TPC/workflow/src/tpc-calib-pad-raw.cxx +++ b/Detectors/TPC/workflow/src/tpc-calib-pad-raw.cxx @@ -61,6 +61,7 @@ void customize(std::vector& workflowOptions) {"configFile", VariantType::String, "", {"configuration file for configurable parameters"}}, {"calib-type", VariantType::String, "pedestal", {"Calibration type to run: pedestal, pulser, ce"}}, {"no-write-ccdb", VariantType::Bool, false, {"skip sending the calibration output to CCDB"}}, + {"send-to-dcs-ccdb", VariantType::Bool, false, {"Send values to DCS DB"}}, {"lanes", VariantType::Int, defaultlanes, {"Number of parallel processing lanes."}}, {"sectors", VariantType::String, sectorDefault.c_str(), {"List of TPC sectors, comma separated ranges, e.g. 0-3,7,9-15"}}, }; @@ -83,6 +84,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& config) std::string inputSpec = config.options().get("input-spec"); const auto skipCCDB = config.options().get("no-write-ccdb"); + const auto sendToDCS = config.options().get("send-to-dcs-ccdb"); const auto publishAfterTFs = config.options().get("publish-after-tfs"); const auto tpcsectors = o2::RangeTokenizer::tokenize(config.options().get("sectors")); @@ -121,7 +123,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& config) workflow.emplace_back(getTPCCalibPadRawSpec(inputSpec, ilane, range, publishAfterTFs, rawType)); } - workflow.emplace_back(getCalDetMergerPublisherSpec(nLanes, skipCCDB, publishAfterTFs > 0)); + workflow.emplace_back(getCalDetMergerPublisherSpec(nLanes, skipCCDB, sendToDCS, publishAfterTFs > 0)); return workflow; } From e646315fa6e85d1e82635d6c1937bd288c9d3f20 Mon Sep 17 00:00:00 2001 From: ALICE Action Bot Date: Wed, 26 Feb 2025 18:55:00 +0000 Subject: [PATCH 2/2] Please consider the following formatting changes --- Detectors/TPC/base/src/CRUCalibHelpers.cxx | 1 - 1 file changed, 1 deletion(-) diff --git a/Detectors/TPC/base/src/CRUCalibHelpers.cxx b/Detectors/TPC/base/src/CRUCalibHelpers.cxx index 0ae2823078b1f..fe6023028f6fe 100644 --- a/Detectors/TPC/base/src/CRUCalibHelpers.cxx +++ b/Detectors/TPC/base/src/CRUCalibHelpers.cxx @@ -263,7 +263,6 @@ cru_calib_helpers::DataMapU32 cru_calib_helpers::getDataMap(const CalPad& calPad const int dataWrapperID = fecInPartition >= fecOffset; const int globalLinkID = (fecInPartition % fecOffset) + dataWrapperID * 12; - const int hwChannel = getHWChannel(sampa, sampaChannel, region % 2); const auto value = calRoc.getValue(ipad);