From 6f9e5eef9c87936dc41ba7878a437a357c2c2b57 Mon Sep 17 00:00:00 2001 From: Marco Giacalone Date: Wed, 5 Aug 2026 20:28:22 +0200 Subject: [PATCH 1/2] Simple event pools merger, with example --- Generators/CMakeLists.txt | 12 +- Generators/src/MergeEventPool.cxx | 287 ++++++++++++++++++++++ run/SimExamples/MergeEventPools/README.md | 8 + run/SimExamples/MergeEventPools/pools.txt | 4 + run/SimExamples/MergeEventPools/run.sh | 15 ++ run/SimExamples/README.md | 1 + 6 files changed, 326 insertions(+), 1 deletion(-) create mode 100644 Generators/src/MergeEventPool.cxx create mode 100644 run/SimExamples/MergeEventPools/README.md create mode 100644 run/SimExamples/MergeEventPools/pools.txt create mode 100755 run/SimExamples/MergeEventPools/run.sh diff --git a/Generators/CMakeLists.txt b/Generators/CMakeLists.txt index 5624ce7df5f07..b36fbf5697ac4 100644 --- a/Generators/CMakeLists.txt +++ b/Generators/CMakeLists.txt @@ -146,7 +146,6 @@ if(doBuildSimulation) # PUBLIC_LINK_LIBRARIES O2::Generators) endif() - o2_add_test_root_macro(share/external/tgenerator.C PUBLIC_LINK_LIBRARIES O2::Generators LABELS generators) @@ -165,6 +164,17 @@ o2_add_test_root_macro(share/egconfig/pythia8_userhooks_charm.C LABELS generators) endif() +o2_add_executable(merge-evtpool + COMPONENT_NAME generators + SOURCES src/MergeEventPool.cxx + PUBLIC_LINK_LIBRARIES O2::CommonUtils + O2::SimulationDataFormat + ROOT::Core + ROOT::RIO + ROOT::Tree + ROOT::Net + Boost::program_options) + o2_data_file(COPY share/external DESTINATION Generators) o2_data_file(COPY share/egconfig DESTINATION Generators) o2_data_file(COPY share/TPCLoopers DESTINATION Generators) diff --git a/Generators/src/MergeEventPool.cxx b/Generators/src/MergeEventPool.cxx new file mode 100644 index 0000000000000..ae078414219dd --- /dev/null +++ b/Generators/src/MergeEventPool.cxx @@ -0,0 +1,287 @@ +// Copyright 2019-2026 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \brief Merges multiple event-pool files (e.g. evtpool.root / genevents_Kine.root, +/// produced by o2-sim --noGeant) into a single "o2sim" tree. +/// +/// This tool merges event pools with TFileMerger (the engine behind hadd). +/// +/// Input handling is added in addition to hadd: files can be given directly, or collected +/// from local text files listing further paths (one per line, '#' comments allowed, +/// resolved recursively). The pools themselves can live on AliEn (alien:// URLs) and are +/// read straight from the storage elements +/// +/// Every input is validated (tree and required branches present) before anything is +/// written, so a bad file is reported and nothing is produced, and the merged pool is +/// checked once more at the end. +/// +/// Usage: +/// +/// # a few pools given directly +/// o2-generators-merge-evtpool -i poolA.root,poolB.root -o merged.root +/// +/// # a local text file listing pools, which may be local and/or alien:// +/// o2-generators-merge-evtpool -i pools.txt -o merged.root +/// +/// Options: --input/-i (required), --output/-o (evtpool.root), --treename/-t (o2sim), +/// --help/-h. Shell variables are expanded in every path, both in --input and inside +/// list files. +/// +/// @author Marco Giacalone, mgiacalo@cern.ch, 08/2026 + +#include "CommonUtils/FileSystemUtils.h" +#include "CommonUtils/StringUtils.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace bpo = boost::program_options; +namespace fs = std::filesystem; + +namespace +{ +const char* kTrackBranch = "MCTrack"; +const char* kHeaderBranch = "MCEventHeader."; +const char* kTrackRefBranch = "TrackRefs"; +const char* kProtocol = "alien://"; + +bool isAlienPath(std::string const& path) +{ + return o2::utils::Str::beginsWith(path, kProtocol); +} + +// Connects to AliEn if that has not happened yet +bool GridOn() +{ + if (gGrid) { + return true; + } + LOG(info) << "Connecting to AliEn ..."; + if (!TGrid::Connect("alien:") || !gGrid) { + LOG(error) << "Could not connect to AliEn; check your alien token"; + return false; + } + return true; +} + +// Reads the lines of a local text file. nullopt if it could not be opened. +std::optional> readLocalListFileLines(std::string const& path) +{ + std::ifstream in(path); + if (!in.is_open()) { + return std::nullopt; + } + std::vector lines; + std::string line; + while (std::getline(in, line)) { + lines.push_back(line); + } + return lines; +} + +// Reads a text file listing input paths, one per line ('#' comments and blank lines +// ignored). Each listed path is either a .root file (local or alien://) or itself +// another list file. The lists themselves are always read locally. +void expandInputEntry(std::string const& rawEntry, std::vector& out, std::vector& stack) +{ + // done here so that the expansion works also when the variables appear in a list file + const auto entry = o2::utils::expandShellVarsInFileName(rawEntry); + if (o2::utils::Str::endsWith(entry, ".root")) { + out.push_back(entry); + return; + } + if (std::find(stack.begin(), stack.end(), entry) != stack.end()) { + LOG(error) << "Reference to an existing list " << entry << "; ignoring"; + return; + } + auto lines = readLocalListFileLines(entry); + if (!lines) { + LOG(error) << "Cannot open " << entry << " (neither a .root file nor a readable local list)"; + return; + } + stack.push_back(entry); + for (auto line : *lines) { + o2::utils::Str::trim(line); + if (line.empty() || line[0] == '#') { + continue; + } + expandInputEntry(line, out, stack); + } + stack.pop_back(); +} + +// Expands a list of raw --input entries (each either a .root file or a list) into +// the flat list of .root files to merge. +std::vector expandInputs(std::vector const& rawEntries) +{ + std::vector result; + std::vector stack; + for (auto const& e : rawEntries) { + expandInputEntry(e, result, stack); + } + return result; +} + +// Checks that a file is readable and holds a tree with the branches expected from a +// standard o2-sim event pool, reporting its event count and compression settings. +// Returns an empty string when the file is usable, the reason otherwise. +std::string inspectFile(std::string const& path, std::string const& treename, + Long64_t& entries, int& compression) +{ + if (!isAlienPath(path) && !fs::exists(path)) { + return "file does not exist"; + } + std::unique_ptr file(TFile::Open(path.c_str(), "READ")); + if (!file || file->IsZombie()) { + return "file cannot be opened"; + } + auto tree = (TTree*)file->Get(treename.c_str()); + if (!tree) { + return "no tree named '" + treename + "' in the file"; + } + if (tree->GetBranch(kTrackBranch) == nullptr || tree->GetBranch(kHeaderBranch) == nullptr || + tree->GetBranch(kTrackRefBranch) == nullptr) { + return std::string("missing the required '") + kTrackBranch + "', '" + kHeaderBranch + "' and/or '" + + kTrackRefBranch + "' branch"; + } + entries = tree->GetEntries(); + compression = file->GetCompressionSettings(); + return {}; +} + +// Checks every input before anything is written. Reports the total number of events and +// the compression settings of the first input +bool checkFiles(std::vector const& files, std::string const& treename, + Long64_t& totalEvents, int& compression) +{ + bool ok = true; + totalEvents = 0; + compression = -1; + for (auto const& f : files) { + Long64_t entries = 0; + int fileCompression = -1; + const auto issue = inspectFile(f, treename, entries, fileCompression); + if (!issue.empty()) { + LOG(error) << "Input file " << f << ": " << issue; + ok = false; + continue; + } + if (compression < 0) { + compression = fileCompression; + } + totalEvents += entries; + LOG(info) << " OK " << f << " (" << entries << " events)"; + } + return ok; +} + +// Re-opens the merged output and checks that it holds the expected tree, branches and +// number of events, so that a truncated or half-written pool does not pass unnoticed. +bool validateOutput(std::string const& outfile, std::string const& treename, Long64_t expected) +{ + Long64_t entries = 0; + int compression = -1; + const auto issue = inspectFile(outfile, treename, entries, compression); + if (!issue.empty()) { + LOG(error) << "Merged file " << outfile << " is not usable: " << issue; + return false; + } + if (entries != expected) { + LOG(error) << "Merged file " << outfile << " has " << entries << " events, but " << expected + << " were merged into it"; + return false; + } + return true; +} +} // namespace + +int main(int argc, char* argv[]) +{ + bpo::options_description options("o2-generators-merge-evtpool options"); + auto add = options.add_options(); + add("input,i", bpo::value()->required(), + "comma-separated list of inputs: event-pool ROOT files (local or alien://), and/or " + "local text files listing more paths (one per line, '#' comments allowed)"); + add("output,o", bpo::value()->default_value("evtpool.root"), + "output ROOT file with the merged event pool"); + add("treename,t", bpo::value()->default_value("o2sim"), "name of the tree to merge"); + add("help,h", "produce help message"); + bpo::variables_map vm; + try { + bpo::store(bpo::parse_command_line(argc, argv, options), vm); + if (vm.count("help")) { + LOG(info) << options; + return 0; + } + bpo::notify(vm); + } catch (const bpo::error& e) { + LOG(fatal) << "Error parsing command-line arguments: " << e.what() << "\n\n" + << options; + return 1; + } + const auto rawEntries = o2::utils::Str::tokenize(vm["input"].as(), ','); + if (rawEntries.empty()) { + LOG(fatal) << "No input files given"; + return 1; + } + const auto infiles = expandInputs(rawEntries); + if (infiles.empty()) { + LOG(fatal) << "No input files resolved from the given --input entries"; + return 1; + } + // Check Grid connection if any input is on AliEn + if (std::any_of(infiles.begin(), infiles.end(), isAlienPath) && !GridOn()) { + LOG(fatal) << "Some inputs live on AliEn but the grid is not available"; + return 1; + } + const std::string outfile = vm["output"].as(); + const std::string treename = vm["treename"].as(); + LOG(info) << "Validating " << infiles.size() << " input file(s) ..."; + Long64_t totalEvents = 0; + int compression = -1; + if (!checkFiles(infiles, treename, totalEvents, compression)) { + LOG(fatal) << "Validation failed; not writing any output"; + return 1; + } + LOG(info) << "Merging " << totalEvents << " events into " << outfile << " ..."; + TFileMerger merger(/*isLocal*/ false, /*histoOneGo*/ false); + merger.SetPrintLevel(0); + if (!merger.OutputFile(outfile.c_str(), "RECREATE", compression)) { + LOG(fatal) << "Cannot create output file " << outfile; + return 1; + } + for (auto const& f : infiles) { + if (!merger.AddFile(f.c_str())) { + LOG(fatal) << "Cannot add " << f << " to the merge"; + return 1; + } + } + if (!merger.Merge()) { + LOG(fatal) << "Merging failed; output " << outfile << " is incomplete"; + return 1; + } + if (!validateOutput(outfile, treename, totalEvents)) { + LOG(fatal) << "The merged pool did not pass the final check"; + return 1; + } + LOG(info) << "Done: wrote " << totalEvents << " events to " << outfile; + return 0; +} diff --git a/run/SimExamples/MergeEventPools/README.md b/run/SimExamples/MergeEventPools/README.md new file mode 100644 index 0000000000000..eee88dc1aab1e --- /dev/null +++ b/run/SimExamples/MergeEventPools/README.md @@ -0,0 +1,8 @@ + + +This example demonstrates how to merge event pools using the dedicated `o2-generators-merge-evtpool`. +The pools to merge are listed in `pools.txt`. They can be local files or `alien://` paths, and a line can also point to +another local file list. AliEn pools are read directly from the storage elements, so a valid alien token is needed. +The merged pool is an ordinary evtpool.root file (by default), so it can be used with the `evtpool` generator. diff --git a/run/SimExamples/MergeEventPools/pools.txt b/run/SimExamples/MergeEventPools/pools.txt new file mode 100644 index 0000000000000..637afa1206667 --- /dev/null +++ b/run/SimExamples/MergeEventPools/pools.txt @@ -0,0 +1,4 @@ +# One evtpool.root file per line; '#' comments and blank lines are ignored. +# Paths can be local or on AliEn, and a line can also point to another (local) list file. +alien:///alice/sim/2026/EP26a2/19/001/evtpool.root +alien:///alice/sim/2026/EP26a2/19/002/evtpool.root diff --git a/run/SimExamples/MergeEventPools/run.sh b/run/SimExamples/MergeEventPools/run.sh new file mode 100755 index 0000000000000..6154decc4c6a0 --- /dev/null +++ b/run/SimExamples/MergeEventPools/run.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# A simple example showing how to merge several event pools. +# The pools are listed in pools.txt. In the example they are read from AliEn, so an alien token +# is needed. +# Additionally this could be run as +# o2-generators-merge-evtpool -i poolA.root,poolB.root -o merged.root +# to give the pools directly, or +# o2-generators-merge-evtpool -i poolA.root,pools.txt -o merged.root +# to mix single pools with a list. The list files themselves must be local. + +set -x + +# merge the pools listed in pools.txt +o2-generators-merge-evtpool -i pools.txt -o evtpool.root diff --git a/run/SimExamples/README.md b/run/SimExamples/README.md index 3a54625acf413..c276c05b650f2 100644 --- a/run/SimExamples/README.md +++ b/run/SimExamples/README.md @@ -24,6 +24,7 @@ * \subpage refrunSimExamplesPythia * \subpage refrunSimExamplesForceDecay_Lambda_Neutron_Dalitz * \subpage refrunSimExamplesJustPrimaryKinematics +* \subpage refrunSimExamplesMergeEventPools * \subpage refrunSimExamplesSelective_Transport * \subpage refrunSimExamplesSelective_Transport_pi0 * \subpage refrunSimExamplesStepMonitoringSimple1 From 2d05e70fe2bd5773face3eae0f9050b93de0f468 Mon Sep 17 00:00:00 2001 From: Marco Giacalone Date: Sun, 16 Aug 2026 15:04:45 +0200 Subject: [PATCH 2/2] Implement code review --- Generators/src/MergeEventPool.cxx | 184 ++++++++++++++++++++++-------- 1 file changed, 137 insertions(+), 47 deletions(-) diff --git a/Generators/src/MergeEventPool.cxx b/Generators/src/MergeEventPool.cxx index ae078414219dd..093c4565895a4 100644 --- a/Generators/src/MergeEventPool.cxx +++ b/Generators/src/MergeEventPool.cxx @@ -20,8 +20,11 @@ /// read straight from the storage elements /// /// Every input is validated (tree and required branches present) before anything is -/// written, so a bad file is reported and nothing is produced, and the merged pool is -/// checked once more at the end. +/// written, and the merged pool is checked once more at the end. Anything that cannot be +/// resolved or opened aborts the merge, but this is bypassable with --skip-non-existing-files. +/// A failed run never leaves a file that looks finished (temporary filename during merge). +/// +/// What went into the merge is written in the root file as a "mergeInfo" map /// /// Usage: /// @@ -31,8 +34,8 @@ /// # a local text file listing pools, which may be local and/or alien:// /// o2-generators-merge-evtpool -i pools.txt -o merged.root /// -/// Options: --input/-i (required), --output/-o (evtpool.root), --treename/-t (o2sim), -/// --help/-h. Shell variables are expanded in every path, both in --input and inside +/// Options: --input/-i (required), --output/-o (evtpool.root), --check-tree/-t (o2sim), +/// --skip-non-existing-files, --help/-h. Shell variables are expanded in every path, both in --input and inside /// list files. /// /// @author Marco Giacalone, mgiacalo@cern.ch, 08/2026 @@ -43,6 +46,8 @@ #include #include #include +#include +#include #include #include #include @@ -50,6 +55,7 @@ #include #include #include +#include #include #include @@ -100,44 +106,56 @@ std::optional> readLocalListFileLines(std::string const // Reads a text file listing input paths, one per line ('#' comments and blank lines // ignored). Each listed path is either a .root file (local or alien://) or itself // another list file. The lists themselves are always read locally. -void expandInputEntry(std::string const& rawEntry, std::vector& out, std::vector& stack) +// Returns how many entries, here or in a nested list, could not be resolved. +size_t expandInputEntry(std::string const& rawEntry, std::vector& out, std::vector& stack) { // done here so that the expansion works also when the variables appear in a list file const auto entry = o2::utils::expandShellVarsInFileName(rawEntry); if (o2::utils::Str::endsWith(entry, ".root")) { out.push_back(entry); - return; + return 0; } if (std::find(stack.begin(), stack.end(), entry) != stack.end()) { LOG(error) << "Reference to an existing list " << entry << "; ignoring"; - return; + return 1; } auto lines = readLocalListFileLines(entry); if (!lines) { LOG(error) << "Cannot open " << entry << " (neither a .root file nor a readable local list)"; - return; + return 1; } stack.push_back(entry); + size_t unresolved = 0; for (auto line : *lines) { o2::utils::Str::trim(line); if (line.empty() || line[0] == '#') { continue; } - expandInputEntry(line, out, stack); + unresolved += expandInputEntry(line, out, stack); } stack.pop_back(); + return unresolved; } -// Expands a list of raw --input entries (each either a .root file or a list) into -// the flat list of .root files to merge. -std::vector expandInputs(std::vector const& rawEntries) +// Expands a list of raw --input entries (each either a .root file or a list) into the flat +// list of .root files to merge, dropping repetitions. Returns how many entries did not resolve. +size_t expandInputs(std::vector const& rawEntries, std::vector& infiles) { - std::vector result; + std::vector resolved; std::vector stack; + size_t unresolved = 0; for (auto const& e : rawEntries) { - expandInputEntry(e, result, stack); + unresolved += expandInputEntry(e, resolved, stack); + } + std::set seen; + for (auto const& f : resolved) { + if (seen.insert(f).second) { + infiles.push_back(f); + } else { + LOG(warning) << "Input " << f << " is listed more than once; merging it only once"; + } } - return result; + return unresolved; } // Checks that a file is readable and holds a tree with the branches expected from a @@ -146,12 +164,9 @@ std::vector expandInputs(std::vector const& rawEntries std::string inspectFile(std::string const& path, std::string const& treename, Long64_t& entries, int& compression) { - if (!isAlienPath(path) && !fs::exists(path)) { - return "file does not exist"; - } std::unique_ptr file(TFile::Open(path.c_str(), "READ")); if (!file || file->IsZombie()) { - return "file cannot be opened"; + return "file does not exist or cannot be opened"; } auto tree = (TTree*)file->Get(treename.c_str()); if (!tree) { @@ -167,10 +182,11 @@ std::string inspectFile(std::string const& path, std::string const& treename, return {}; } -// Checks every input before anything is written. Reports the total number of events and -// the compression settings of the first input +// Checks every input before anything is written, collecting the usable ones and reporting +// the total number of events and the compression settings of the first usable input. +// Returns true when every input passed. bool checkFiles(std::vector const& files, std::string const& treename, - Long64_t& totalEvents, int& compression) + std::vector& usable, Long64_t& totalEvents, int& compression) { bool ok = true; totalEvents = 0; @@ -188,11 +204,43 @@ bool checkFiles(std::vector const& files, std::string const& treena compression = fileCompression; } totalEvents += entries; + usable.push_back(f); LOG(info) << " OK " << f << " (" << entries << " events)"; } return ok; } +// Records what the merge was asked for and what actually went into it. +void writeMergeInfo(std::string const& outfile, std::vector const& requested, + std::vector const& merged, size_t unresolved, Long64_t events) +{ + std::unique_ptr file(TFile::Open(outfile.c_str(), "UPDATE")); + if (!file || file->IsZombie()) { + LOG(warning) << "Cannot add the merge information to " << outfile; + return; + } + // the files that were asked for but did not make it, so that the gap can be named from + // the file alone and not just counted + std::string mergedList, skippedList; + for (auto const& f : requested) { + if (std::find(merged.begin(), merged.end(), f) != merged.end()) { + mergedList += f + "\n"; + } else { + skippedList += f + "\n"; + } + } + TMap info; + info.SetOwnerKeyValue(); + info.Add(new TObjString("inputsRequested"), new TObjString(std::to_string(requested.size()).c_str())); + info.Add(new TObjString("inputsMerged"), new TObjString(std::to_string(merged.size()).c_str())); + info.Add(new TObjString("inputsUnresolved"), new TObjString(std::to_string(unresolved).c_str())); + info.Add(new TObjString("events"), new TObjString(std::to_string(events).c_str())); + info.Add(new TObjString("mergedFiles"), new TObjString(mergedList.c_str())); + info.Add(new TObjString("skippedFiles"), new TObjString(skippedList.c_str())); + file->cd(); + info.Write("mergeInfo", TObject::kSingleKey); +} + // Re-opens the merged output and checks that it holds the expected tree, branches and // number of events, so that a truncated or half-written pool does not pass unnoticed. bool validateOutput(std::string const& outfile, std::string const& treename, Long64_t expected) @@ -222,7 +270,11 @@ int main(int argc, char* argv[]) "local text files listing more paths (one per line, '#' comments allowed)"); add("output,o", bpo::value()->default_value("evtpool.root"), "output ROOT file with the merged event pool"); - add("treename,t", bpo::value()->default_value("o2sim"), "name of the tree to merge"); + add("check-tree,t", bpo::value()->default_value("o2sim"), + "name of the tree the inputs and the merged pool are checked against; everything the " + "input files contain is merged regardless"); + add("skip-non-existing-files", bpo::bool_switch(), + "skip inputs that cannot be resolved or opened instead of aborting the merge"); add("help,h", "produce help message"); bpo::variables_map vm; try { @@ -233,55 +285,93 @@ int main(int argc, char* argv[]) } bpo::notify(vm); } catch (const bpo::error& e) { - LOG(fatal) << "Error parsing command-line arguments: " << e.what() << "\n\n" + LOG(error) << "Error parsing command-line arguments: " << e.what() << "\n\n" << options; return 1; } const auto rawEntries = o2::utils::Str::tokenize(vm["input"].as(), ','); if (rawEntries.empty()) { - LOG(fatal) << "No input files given"; + LOG(error) << "No input files given"; + return 1; + } + // option similar in aodMerger + const bool skipMissing = vm["skip-non-existing-files"].as(); + std::vector infiles; + const size_t unresolved = expandInputs(rawEntries, infiles); + if (unresolved > 0 && !skipMissing) { + LOG(error) << "Some --input entries could not be resolved; " + "pass --skip-non-existing-files to merge the rest anyway"; return 1; } - const auto infiles = expandInputs(rawEntries); if (infiles.empty()) { - LOG(fatal) << "No input files resolved from the given --input entries"; + LOG(error) << "No input files resolved from the given --input entries"; return 1; } // Check Grid connection if any input is on AliEn if (std::any_of(infiles.begin(), infiles.end(), isAlienPath) && !GridOn()) { - LOG(fatal) << "Some inputs live on AliEn but the grid is not available"; + LOG(error) << "Some inputs live on AliEn but the grid is not available"; return 1; } const std::string outfile = vm["output"].as(); - const std::string treename = vm["treename"].as(); + const std::string treename = vm["check-tree"].as(); LOG(info) << "Validating " << infiles.size() << " input file(s) ..."; + std::vector usable; Long64_t totalEvents = 0; int compression = -1; - if (!checkFiles(infiles, treename, totalEvents, compression)) { - LOG(fatal) << "Validation failed; not writing any output"; + if (!checkFiles(infiles, treename, usable, totalEvents, compression) && !skipMissing) { + LOG(error) << "Validation failed; not writing any output " + "(pass --skip-non-existing-files to merge the rest anyway)"; return 1; } - LOG(info) << "Merging " << totalEvents << " events into " << outfile << " ..."; - TFileMerger merger(/*isLocal*/ false, /*histoOneGo*/ false); - merger.SetPrintLevel(0); - if (!merger.OutputFile(outfile.c_str(), "RECREATE", compression)) { - LOG(fatal) << "Cannot create output file " << outfile; + if (usable.empty()) { + LOG(error) << "None of the input files could be used; not writing any output"; return 1; } - for (auto const& f : infiles) { - if (!merger.AddFile(f.c_str())) { - LOG(fatal) << "Cannot add " << f << " to the merge"; - return 1; + + // merged into a temporary name and renamed only once the result has been checked, so that + // a failed job never leaves something behind that looks like a finished pool + const std::string partfile = outfile + ".part"; + auto discardPart = [&partfile]() { + std::error_code ec; + fs::remove(partfile, ec); + return 1; + }; + + LOG(info) << "Merging " << totalEvents << " events from " << usable.size() << " file(s) into " + << outfile << " ..."; + { + TFileMerger merger(/*isLocal*/ false, /*histoOneGo*/ false); + merger.SetPrintLevel(0); + if (!merger.OutputFile(partfile.c_str(), "RECREATE", compression)) { + LOG(error) << "Cannot create output file " << partfile; + return discardPart(); + } + for (auto const& f : usable) { + if (!merger.AddFile(f.c_str())) { + LOG(error) << "Cannot add " << f << " to the merge"; + return discardPart(); + } + } + if (!merger.Merge()) { + LOG(error) << "Merging failed; no output written"; + return discardPart(); } } - if (!merger.Merge()) { - LOG(fatal) << "Merging failed; output " << outfile << " is incomplete"; - return 1; + + writeMergeInfo(partfile, infiles, usable, unresolved, totalEvents); + if (!validateOutput(partfile, treename, totalEvents)) { + LOG(error) << "The merged pool did not pass the final check; no output written"; + return discardPart(); } - if (!validateOutput(outfile, treename, totalEvents)) { - LOG(fatal) << "The merged pool did not pass the final check"; - return 1; + + std::error_code ec; + fs::rename(partfile, outfile, ec); + if (ec) { + LOG(error) << "Cannot move " << partfile << " to " << outfile << ": " << ec.message(); + return discardPart(); } - LOG(info) << "Done: wrote " << totalEvents << " events to " << outfile; + + LOG(info) << "Done: wrote " << totalEvents << " events from " << usable.size() << " of " + << infiles.size() << " input file(s) to " << outfile; return 0; }