From 2fc2834b75dacac572348b3445d5f5f3d5860176 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 14 Nov 2025 09:23:01 +0100 Subject: [PATCH 01/11] added init script --- scripts/enforce_same_events.py | 167 +++++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 scripts/enforce_same_events.py diff --git a/scripts/enforce_same_events.py b/scripts/enforce_same_events.py new file mode 100644 index 00000000..17a0bd7b --- /dev/null +++ b/scripts/enforce_same_events.py @@ -0,0 +1,167 @@ +from astropy.table import vstack, join +import numpy as np + +from ctapipe.io import read_table, write_table +from ctapipe.core import Tool +from ctapipe.core.traits import ( + ComponentName, + Path, + Unicode, + Bool, + List, +) +from ctapipe.instrument import SubarrayDescription +from ctapipe.reco.reconstructor import ReconstructionProperty +from ctapipe.reco.stereo_combination import StereoCombiner + +DL2_SUBARRAY_GROUP = "/dl2/event/subarray" +DL2_TELESCOPE_GROUP = "/dl2/event/telescope" +SUBARRAY_EVENT_KEYS = ["obs_id", "event_id"] +TELESCOPE_EVENT_KEYS = ["obs_id", "event_id", "tel_id"] + +__all__ = [ + "EnforceSameEvents", +] + + +class EnforceSameEvents(Tool): + """ + Append a subarray table to the hdf5 file after the monoscopic predictions. + + This tool reads the monoscopic predictions from the input hdf5 file and combines them + to a subarray table using the ctapipe stereo combiner. The subarray table is then written + to the hdf5 file. + + Parameters + ---------- + input_url : str + Input ctapipe HDF5 files including monoscopic predictions. + prefix : str + Name of the reconstruction algorithm used to generate the dl2 data. + reco_tasks : list + List of reconstruction tasks to be used for the stereo combination. + stereo_combiner_cls : str + Which ctapipe stereo combination method to use after the monoscopic reconstruction. + overwrite_tables : bool + Overwrite the table in the hdf5 file if it exists. + """ + name = "EnforceSameEvents" + description = "Append a subarray table to the hdf5 file after the monoscopic predictions." + + input_url = Path( + help="Input ctapipe HDF5 files including monoscopic predictions.", + allow_none=False, + exists=True, + directory_ok=False, + file_ok=True, + ).tag(config=True) + + output_url = Path( + help="Output ctapipe HDF5 files including monoscopic predictions.", + allow_none=False, + exists=True, + directory_ok=False, + file_ok=True, + ).tag(config=True) + + prefix = Unicode( + default_value="CTLearn", + allow_none=False, + help="Name of the reconstruction algorithm used to generate the dl2 data.", + ).tag(config=True) + + reco_tasks = List( + default_value=["classification", "energy", "geometry"], + allow_none=False, + help="List of reconstruction tasks to be used for the stereo combination.", + ).tag(config=True) + + dl2_telescope= Bool( + default_value=True, + help="Whether to create dl2 telescope group if it does not exist.", + ).tag(config=True) + + dl2_subarray = Bool( + default_value=True, + help="Whether to create dl2 subarray group if it does not exist.", + ).tag(config=True) + + aliases = { + ("i", "input"): "EnforceSameEvents.input_url", + ("o", "output"): "EnforceSameEvents.output_url", + ("p", "prefix"): "EnforceSameEvents.prefix", + ("r", "reco-tasks"): "EnforceSameEvents.reco_tasks", + "dl2-telescope": "EnforceSameEvents.dl2_telescope", + "dl2-subarray": "EnforceSameEvents.dl2_subarray", + } + + def setup(self): + # Set up the reconstruction properties for the stereo combiner + self.reco_properties = { + "geometry": ReconstructionProperty.GEOMETRY, + "energy": ReconstructionProperty.ENERGY, + "classification": ReconstructionProperty.PARTICLE_TYPE, + } + # Read the SubarrayDescription from the input file + self.subarray = SubarrayDescription.from_hdf(self.input_url) + + + def start(self): + # Loop over the reconstruction tasks and combine the telescope tables to a subarray table + for reco_task in self.reco_tasks: + self.log.info("Processing %s...", reco_task) + + # Read the telescope tables from the input file + tel_tables = [] + if self.dl2_telescope: + for tel_id in self.subarray.tel_ids: + input_tel_table = read_table( + self.input_url, + f"{DL2_TELESCOPE_GROUP}/{reco_task}/{self.prefix}/tel_{tel_id:03}", + ) + output_tel_table = read_table( + self.output_url, + f"{DL2_TELESCOPE_GROUP}/{reco_task}/{self.prefix}/tel_{tel_id:03}", + ) + joined_table = join( + input_tel_table, + output_tel_table, + keys=TELESCOPE_EVENT_KEYS, + join_type="inner", + ) + self.log.info("Input table for telescope %03d has %d rows", tel_id, len(input_tel_table)) + self.log.info("Output table for telescope %03d has %d rows", tel_id, len(output_tel_table)) + self.log.info("Joined table for telescope %03d has %d rows", tel_id, len(joined_table)) + self.log.info(joined_table.colnames) + # Stack the telescope tables to a common table + #tel_tables = vstack(tel_tables) + # Sort the table by the telescope event keys + #tel_tables.sort(TELESCOPE_EVENT_KEYS) + + # Sort the table by the subarray event keys + #subarray_table.sort(SUBARRAY_EVENT_KEYS) + # Save the prediction to the file + #write_table( + # subarray_table, + # self.input_url, + # f"{DL2_SUBARRAY_GROUP}/{reco_task}/{self.prefix}", + # overwrite=self.overwrite_tables, + #) + #self.log.info( + # "DL2 prediction data was stored in '%s' under '%s'", + # self.input_url, + # f"{DL2_SUBARRAY_GROUP}/{reco_task}/{self.prefix}", + #) + + def finish(self): + # Shutting down the tool + self.log.info("Tool is shutting down") + +def main(): + # Run the tool + tool = EnforceSameEvents() + tool.run() + + +if __name__ == "__main__": + main() From 708397fa36a56c4d3fa202b61ce62afa0b7af31a Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 14 Nov 2025 10:40:18 +0100 Subject: [PATCH 02/11] further improvements to same events script --- scripts/enforce_same_events.py | 86 +++++++++++++++++++++++++--------- 1 file changed, 65 insertions(+), 21 deletions(-) diff --git a/scripts/enforce_same_events.py b/scripts/enforce_same_events.py index 17a0bd7b..8697c29f 100644 --- a/scripts/enforce_same_events.py +++ b/scripts/enforce_same_events.py @@ -1,8 +1,9 @@ from astropy.table import vstack, join import numpy as np +import os -from ctapipe.io import read_table, write_table -from ctapipe.core import Tool +from ctapipe.io import read_table, write_table, HDF5Merger +from ctapipe.core import Tool, ToolConfigurationError from ctapipe.core.traits import ( ComponentName, Path, @@ -11,8 +12,6 @@ List, ) from ctapipe.instrument import SubarrayDescription -from ctapipe.reco.reconstructor import ReconstructionProperty -from ctapipe.reco.stereo_combination import StereoCombiner DL2_SUBARRAY_GROUP = "/dl2/event/subarray" DL2_TELESCOPE_GROUP = "/dl2/event/telescope" @@ -48,7 +47,8 @@ class EnforceSameEvents(Tool): name = "EnforceSameEvents" description = "Append a subarray table to the hdf5 file after the monoscopic predictions." - input_url = Path( + # is_valid_from + is_valid_from = Path( help="Input ctapipe HDF5 files including monoscopic predictions.", allow_none=False, exists=True, @@ -56,14 +56,22 @@ class EnforceSameEvents(Tool): file_ok=True, ).tag(config=True) - output_url = Path( - help="Output ctapipe HDF5 files including monoscopic predictions.", + is_valid_to = Path( + help="Input ctapipe HDF5 files including monoscopic predictions.", allow_none=False, exists=True, directory_ok=False, file_ok=True, ).tag(config=True) + output_path = Path( + help="Output ctapipe HDF5 files including monoscopic predictions.", + allow_none=False, + exists=False, + directory_ok=False, + file_ok=True, + ).tag(config=True) + prefix = Unicode( default_value="CTLearn", allow_none=False, @@ -77,7 +85,7 @@ class EnforceSameEvents(Tool): ).tag(config=True) dl2_telescope= Bool( - default_value=True, + default_value=False, help="Whether to create dl2 telescope group if it does not exist.", ).tag(config=True) @@ -87,8 +95,9 @@ class EnforceSameEvents(Tool): ).tag(config=True) aliases = { - ("i", "input"): "EnforceSameEvents.input_url", - ("o", "output"): "EnforceSameEvents.output_url", + ("f", "is_valid_from"): "EnforceSameEvents.is_valid_from", + ("t", "is_valid_to"): "EnforceSameEvents.is_valid_to", + ("o", "output"): "EnforceSameEvents.output_path", ("p", "prefix"): "EnforceSameEvents.prefix", ("r", "reco-tasks"): "EnforceSameEvents.reco_tasks", "dl2-telescope": "EnforceSameEvents.dl2_telescope", @@ -96,14 +105,20 @@ class EnforceSameEvents(Tool): } def setup(self): - # Set up the reconstruction properties for the stereo combiner - self.reco_properties = { - "geometry": ReconstructionProperty.GEOMETRY, - "energy": ReconstructionProperty.ENERGY, - "classification": ReconstructionProperty.PARTICLE_TYPE, - } + # Check if the ctapipe HDF5Merger component is enabled + if os.path.exists(self.output_path): + raise ToolConfigurationError( + f"The output file '{self.output_path}' already exists. Please set " + "a different output path or manually remove the existing file." + ) + else: + # Copy selected tables from the input file to the output file + self.log.info("Copying to output destination.") + with HDF5Merger(self.output_path, parent=self) as merger: + merger(self.is_valid_to) + # Read the SubarrayDescription from the input file - self.subarray = SubarrayDescription.from_hdf(self.input_url) + self.subarray = SubarrayDescription.from_hdf(self.is_valid_to) def start(self): @@ -112,27 +127,56 @@ def start(self): self.log.info("Processing %s...", reco_task) # Read the telescope tables from the input file - tel_tables = [] if self.dl2_telescope: for tel_id in self.subarray.tel_ids: input_tel_table = read_table( - self.input_url, + self.is_valid_from, f"{DL2_TELESCOPE_GROUP}/{reco_task}/{self.prefix}/tel_{tel_id:03}", ) output_tel_table = read_table( - self.output_url, + self.is_valid_to, f"{DL2_TELESCOPE_GROUP}/{reco_task}/{self.prefix}/tel_{tel_id:03}", ) joined_table = join( input_tel_table, output_tel_table, keys=TELESCOPE_EVENT_KEYS, - join_type="inner", + join_type="right", ) self.log.info("Input table for telescope %03d has %d rows", tel_id, len(input_tel_table)) self.log.info("Output table for telescope %03d has %d rows", tel_id, len(output_tel_table)) self.log.info("Joined table for telescope %03d has %d rows", tel_id, len(joined_table)) self.log.info(joined_table.colnames) + if self.dl2_subarray: + is_valid_col = f"{self.prefix}_is_valid" + input_subarray_table = read_table( + self.is_valid_from, + f"{DL2_SUBARRAY_GROUP}/{reco_task}/{self.prefix}", + ) + input_subarray_table.keep_columns(SUBARRAY_EVENT_KEYS + [is_valid_col]) + output_subarray_table = read_table( + self.is_valid_to, + f"{DL2_SUBARRAY_GROUP}/{reco_task}/{self.prefix}", + ) + output_subarray_table.remove_columns([is_valid_col]) + if len(input_subarray_table) != len(output_subarray_table): + self.log.warning( + "Input and output subarray tables have different lengths: %d vs %d", + len(input_subarray_table), + len(output_subarray_table), + ) + joined_table = join( + input_subarray_table[0], + output_subarray_table, + keys=SUBARRAY_EVENT_KEYS, + join_type="right", + ) + joined_table[is_valid_col] = joined_table[is_valid_col].filled(False) + + print(joined_table.colnames) + print(joined_table[is_valid_col]) + + self.log.debug(joined_table.colnames) # Stack the telescope tables to a common table #tel_tables = vstack(tel_tables) # Sort the table by the telescope event keys From 91fe179d7dc6446b89f8467ff3655d4343194688 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Sat, 15 Nov 2025 15:45:42 +0100 Subject: [PATCH 03/11] fix script and rename to overwrite is_valid flag --- ctlearn/tools/predict_model.py | 4 +- ...me_events.py => overwrite_isvalid_flag.py} | 139 ++++++++++++------ 2 files changed, 93 insertions(+), 50 deletions(-) rename scripts/{enforce_same_events.py => overwrite_isvalid_flag.py} (57%) diff --git a/ctlearn/tools/predict_model.py b/ctlearn/tools/predict_model.py index 6a32f496..13a2ad0b 100644 --- a/ctlearn/tools/predict_model.py +++ b/ctlearn/tools/predict_model.py @@ -329,8 +329,8 @@ class PredictCTLearnModel(Tool): **flag( "dl2-subarray", "PredictCTLearnModel.dl2_subarray", - "Include dl2 telescope-event-wise data in the output file", - "Exclude dl2 telescope-event-wise data in the output file", + "Include dl2 subarray-event-wise data in the output file", + "Exclude dl2 subarray-event-wise data in the output file", ), **flag( "use-HDF5Merger", diff --git a/scripts/enforce_same_events.py b/scripts/overwrite_isvalid_flag.py similarity index 57% rename from scripts/enforce_same_events.py rename to scripts/overwrite_isvalid_flag.py index 8697c29f..90dc5d2b 100644 --- a/scripts/enforce_same_events.py +++ b/scripts/overwrite_isvalid_flag.py @@ -1,14 +1,16 @@ -from astropy.table import vstack, join +from astropy.table import join, MaskedColumn import numpy as np import os from ctapipe.io import read_table, write_table, HDF5Merger from ctapipe.core import Tool, ToolConfigurationError from ctapipe.core.traits import ( - ComponentName, Path, Unicode, + flag, Bool, + Set, + CInt, List, ) from ctapipe.instrument import SubarrayDescription @@ -19,11 +21,11 @@ TELESCOPE_EVENT_KEYS = ["obs_id", "event_id", "tel_id"] __all__ = [ - "EnforceSameEvents", + "OverwriteIsValidFlag", ] -class EnforceSameEvents(Tool): +class OverwriteIsValidFlag(Tool): """ Append a subarray table to the hdf5 file after the monoscopic predictions. @@ -44,10 +46,9 @@ class EnforceSameEvents(Tool): overwrite_tables : bool Overwrite the table in the hdf5 file if it exists. """ - name = "EnforceSameEvents" + name = "OverwriteIsValidFlag" description = "Append a subarray table to the hdf5 file after the monoscopic predictions." - # is_valid_from is_valid_from = Path( help="Input ctapipe HDF5 files including monoscopic predictions.", allow_none=False, @@ -72,6 +73,17 @@ class EnforceSameEvents(Tool): file_ok=True, ).tag(config=True) + allowed_tels = Set( + trait=CInt(), + default_value=None, + allow_none=True, + help=( + "List of allowed tel_ids, others will be ignored. " + "If None, all telescopes in the input stream " + "will be included restricted by trait ``allowed_tel_types``" + ), + ).tag(config=True) + prefix = Unicode( default_value="CTLearn", allow_none=False, @@ -85,7 +97,7 @@ class EnforceSameEvents(Tool): ).tag(config=True) dl2_telescope= Bool( - default_value=False, + default_value=True, help="Whether to create dl2 telescope group if it does not exist.", ).tag(config=True) @@ -95,13 +107,26 @@ class EnforceSameEvents(Tool): ).tag(config=True) aliases = { - ("f", "is_valid_from"): "EnforceSameEvents.is_valid_from", - ("t", "is_valid_to"): "EnforceSameEvents.is_valid_to", - ("o", "output"): "EnforceSameEvents.output_path", - ("p", "prefix"): "EnforceSameEvents.prefix", - ("r", "reco-tasks"): "EnforceSameEvents.reco_tasks", - "dl2-telescope": "EnforceSameEvents.dl2_telescope", - "dl2-subarray": "EnforceSameEvents.dl2_subarray", + ("f", "is-valid-from"): "OverwriteIsValidFlag.is_valid_from", + ("t", "is-valid-to"): "OverwriteIsValidFlag.is_valid_to", + ("o", "output"): "OverwriteIsValidFlag.output_path", + ("p", "prefix"): "OverwriteIsValidFlag.prefix", + ("r", "reco-tasks"): "OverwriteIsValidFlag.reco_tasks", + } + + flags = { + **flag( + "dl2-telescope", + "OverwriteIsValidFlag.dl2_telescope", + "Include overwrite dl2 telescope-event-wise data in the output file", + "Exclude overwrite dl2 telescope-event-wise data in the output file", + ), + **flag( + "dl2-subarray", + "OverwriteIsValidFlag.dl2_subarray", + "Include overwrite dl2 subarray-event-wise data in the output file", + "Exclude overwrite dl2 subarray-event-wise data in the output file", + ), } def setup(self): @@ -119,7 +144,8 @@ def setup(self): # Read the SubarrayDescription from the input file self.subarray = SubarrayDescription.from_hdf(self.is_valid_to) - + if self.allowed_tels is not None: + self.subarray = self.subarray.select_subarray(self.allowed_tels) def start(self): # Loop over the reconstruction tasks and combine the telescope tables to a subarray table @@ -128,26 +154,51 @@ def start(self): # Read the telescope tables from the input file if self.dl2_telescope: + is_valid_col = f"{self.prefix}_tel_is_valid" for tel_id in self.subarray.tel_ids: + self.log.info("Processing telescope '%03d' ...", tel_id) input_tel_table = read_table( self.is_valid_from, f"{DL2_TELESCOPE_GROUP}/{reco_task}/{self.prefix}/tel_{tel_id:03}", ) + input_tel_table.keep_columns(TELESCOPE_EVENT_KEYS + [is_valid_col]) output_tel_table = read_table( self.is_valid_to, f"{DL2_TELESCOPE_GROUP}/{reco_task}/{self.prefix}/tel_{tel_id:03}", ) - joined_table = join( + output_tel_table.remove_columns([is_valid_col]) + if len(input_tel_table) != len(output_tel_table): + self.log.warning( + "Input and output telescope tables (tel_id '%03d') have different lengths: %d vs %d", + tel_id, + len(input_tel_table), + len(output_tel_table), + ) + joined_tel_table = join( input_tel_table, output_tel_table, keys=TELESCOPE_EVENT_KEYS, join_type="right", ) - self.log.info("Input table for telescope %03d has %d rows", tel_id, len(input_tel_table)) - self.log.info("Output table for telescope %03d has %d rows", tel_id, len(output_tel_table)) - self.log.info("Joined table for telescope %03d has %d rows", tel_id, len(joined_table)) - self.log.info(joined_table.colnames) + # Fill missing values in the is_valid column with False if necessary + if isinstance(joined_tel_table[is_valid_col], MaskedColumn): + joined_tel_table[is_valid_col] = joined_tel_table[is_valid_col].filled(False) + # Sort the table by the telescope event keys + joined_tel_table.sort(TELESCOPE_EVENT_KEYS) + write_table( + joined_tel_table, + self.output_path, + f"{DL2_TELESCOPE_GROUP}/{reco_task}/{self.prefix}/tel_{tel_id:03d}", + overwrite=True, + ) + self.log.info( + "DL2 prediction data was stored in '%s' under '%s'", + self.output_path, + f"{DL2_TELESCOPE_GROUP}/{reco_task}/{self.prefix}/tel_{tel_id:03d}", + ) + if self.dl2_subarray: + self.log.info("Processing subarray ...") is_valid_col = f"{self.prefix}_is_valid" input_subarray_table = read_table( self.is_valid_from, @@ -165,37 +216,29 @@ def start(self): len(input_subarray_table), len(output_subarray_table), ) - joined_table = join( - input_subarray_table[0], + joined_subarray_table = join( + input_subarray_table, output_subarray_table, keys=SUBARRAY_EVENT_KEYS, join_type="right", ) - joined_table[is_valid_col] = joined_table[is_valid_col].filled(False) - - print(joined_table.colnames) - print(joined_table[is_valid_col]) - - self.log.debug(joined_table.colnames) - # Stack the telescope tables to a common table - #tel_tables = vstack(tel_tables) - # Sort the table by the telescope event keys - #tel_tables.sort(TELESCOPE_EVENT_KEYS) - - # Sort the table by the subarray event keys - #subarray_table.sort(SUBARRAY_EVENT_KEYS) - # Save the prediction to the file - #write_table( - # subarray_table, - # self.input_url, - # f"{DL2_SUBARRAY_GROUP}/{reco_task}/{self.prefix}", - # overwrite=self.overwrite_tables, - #) - #self.log.info( - # "DL2 prediction data was stored in '%s' under '%s'", - # self.input_url, - # f"{DL2_SUBARRAY_GROUP}/{reco_task}/{self.prefix}", - #) + # Fill missing values in the is_valid column with False if necessary + if isinstance(joined_subarray_table[is_valid_col], MaskedColumn): + joined_subarray_table[is_valid_col] = joined_subarray_table[is_valid_col].filled(False) + # Sort the table by the subarray event keys + joined_subarray_table.sort(SUBARRAY_EVENT_KEYS) + # Save the prediction to the file + write_table( + joined_subarray_table, + self.output_path, + f"{DL2_SUBARRAY_GROUP}/{reco_task}/{self.prefix}", + overwrite=True, + ) + self.log.info( + "DL2 prediction data was stored in '%s' under '%s'", + self.output_path, + f"{DL2_SUBARRAY_GROUP}/{reco_task}/{self.prefix}", + ) def finish(self): # Shutting down the tool @@ -203,7 +246,7 @@ def finish(self): def main(): # Run the tool - tool = EnforceSameEvents() + tool = OverwriteIsValidFlag() tool.run() From 04e51f1b80ab341738aad986a54b793382b008e2 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Sat, 15 Nov 2025 17:34:04 +0100 Subject: [PATCH 04/11] polish docstrings --- scripts/append_subarray_table.py | 10 +++++-- scripts/overwrite_isvalid_flag.py | 46 +++++++++++++++++-------------- scripts/overwrite_table.py | 3 ++ 3 files changed, 36 insertions(+), 23 deletions(-) diff --git a/scripts/append_subarray_table.py b/scripts/append_subarray_table.py index 26a41ef9..295cb48f 100644 --- a/scripts/append_subarray_table.py +++ b/scripts/append_subarray_table.py @@ -1,3 +1,7 @@ +""" +Append a subarray table to the hdf5 file after the monoscopic predictions. +""" + from astropy.table import vstack import numpy as np @@ -83,9 +87,9 @@ class AppendSubarrayTable(Tool): aliases = { ("i", "input_url"): "AppendSubarrayTable.input_url", ("p", "prefix"): "AppendSubarrayTable.prefix", - ("r", "reco_tasks"): "AppendSubarrayTable.reco_tasks", - ("o", "overwrite_tables"): "AppendSubarrayTable.overwrite_tables", - ("s", "stereo_combiner_cls"): "AppendSubarrayTable.stereo_combiner_cls", + ("r", "reco-tasks"): "AppendSubarrayTable.reco_tasks", + ("o", "overwrite-tables"): "AppendSubarrayTable.overwrite_tables", + ("s", "stereo-combiner-cls"): "AppendSubarrayTable.stereo_combiner_cls", } def setup(self): diff --git a/scripts/overwrite_isvalid_flag.py b/scripts/overwrite_isvalid_flag.py index 90dc5d2b..f90c5f9f 100644 --- a/scripts/overwrite_isvalid_flag.py +++ b/scripts/overwrite_isvalid_flag.py @@ -1,3 +1,7 @@ +""" +Overwrite the is_valid flags in the hdf5 file. +""" + from astropy.table import join, MaskedColumn import numpy as np import os @@ -27,30 +31,33 @@ class OverwriteIsValidFlag(Tool): """ - Append a subarray table to the hdf5 file after the monoscopic predictions. + Overwrite the is_valid flags in the hdf5 file. - This tool reads the monoscopic predictions from the input hdf5 file and combines them - to a subarray table using the ctapipe stereo combiner. The subarray table is then written - to the hdf5 file. + This tool reads the is_valid flags from one ctapipe HDF5 file and overwrites + them in another ctapipe HDF5 file. The user can specify which reconstruction + tasks to consider for the overwrite, as well as the prefix used for the + reconstruction algorithm. The output file will contain the same data as the + input file to which the is_valid flags were written, but with the is_valid + flags replaced by those from the other input file. Parameters ---------- - input_url : str - Input ctapipe HDF5 files including monoscopic predictions. + is_valid_from : str + Input ctapipe HDF5 files from which the is_valid flags will be taken. + is_valid_to : str + Input ctapipe HDF5 files to which the is_valid flags will be overwritten. prefix : str Name of the reconstruction algorithm used to generate the dl2 data. reco_tasks : list - List of reconstruction tasks to be used for the stereo combination. - stereo_combiner_cls : str - Which ctapipe stereo combination method to use after the monoscopic reconstruction. - overwrite_tables : bool - Overwrite the table in the hdf5 file if it exists. + List of reconstruction tasks to be used for the overwrite of the is_valid flag. + output_path : str + Output ctapipe HDF5 files including the overwritten is_valid flags. """ name = "OverwriteIsValidFlag" - description = "Append a subarray table to the hdf5 file after the monoscopic predictions." + description = "Overwrite the is_valid flags in the hdf5 file." is_valid_from = Path( - help="Input ctapipe HDF5 files including monoscopic predictions.", + help="Input ctapipe HDF5 files from which the is_valid flags will be taken.", allow_none=False, exists=True, directory_ok=False, @@ -58,7 +65,7 @@ class OverwriteIsValidFlag(Tool): ).tag(config=True) is_valid_to = Path( - help="Input ctapipe HDF5 files including monoscopic predictions.", + help="Input ctapipe HDF5 files to which the is_valid flags will be overwritten.", allow_none=False, exists=True, directory_ok=False, @@ -66,7 +73,7 @@ class OverwriteIsValidFlag(Tool): ).tag(config=True) output_path = Path( - help="Output ctapipe HDF5 files including monoscopic predictions.", + help="Output ctapipe HDF5 files including the overwritten is_valid flags.", allow_none=False, exists=False, directory_ok=False, @@ -79,8 +86,7 @@ class OverwriteIsValidFlag(Tool): allow_none=True, help=( "List of allowed tel_ids, others will be ignored. " - "If None, all telescopes in the input stream " - "will be included restricted by trait ``allowed_tel_types``" + "If None, all telescopes in the input stream will be included." ), ).tag(config=True) @@ -93,17 +99,17 @@ class OverwriteIsValidFlag(Tool): reco_tasks = List( default_value=["classification", "energy", "geometry"], allow_none=False, - help="List of reconstruction tasks to be used for the stereo combination.", + help="List of reconstruction tasks to be used for the overwrite of the is_valid flag.", ).tag(config=True) dl2_telescope= Bool( default_value=True, - help="Whether to create dl2 telescope group if it does not exist.", + help="Whether to overwrite the is_valid flag in the dl2 telescope group.", ).tag(config=True) dl2_subarray = Bool( default_value=True, - help="Whether to create dl2 subarray group if it does not exist.", + help="Whether to overwrite the is_valid flag in the dl2 subarray group.", ).tag(config=True) aliases = { diff --git a/scripts/overwrite_table.py b/scripts/overwrite_table.py index ea9eb3c1..58c8b03e 100644 --- a/scripts/overwrite_table.py +++ b/scripts/overwrite_table.py @@ -1,3 +1,6 @@ +""" +Script to overwrite a table in a ctapipe hdf5 file with another table. +""" import argparse from ctapipe.io import read_table, write_table From 47d51f8204726bca54b8b5ce610d6dc944b1383d Mon Sep 17 00:00:00 2001 From: Tjark Miener <37835610+TjarkMiener@users.noreply.github.com> Date: Sat, 15 Nov 2025 17:45:27 +0100 Subject: [PATCH 05/11] Update scripts/overwrite_isvalid_flag.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- scripts/overwrite_isvalid_flag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/overwrite_isvalid_flag.py b/scripts/overwrite_isvalid_flag.py index f90c5f9f..2e882bda 100644 --- a/scripts/overwrite_isvalid_flag.py +++ b/scripts/overwrite_isvalid_flag.py @@ -230,7 +230,7 @@ def start(self): ) # Fill missing values in the is_valid column with False if necessary if isinstance(joined_subarray_table[is_valid_col], MaskedColumn): - joined_subarray_table[is_valid_col] = joined_subarray_table[is_valid_col].filled(False) + joined_subarray_table[is_valid_col] = joined_subarray_table[is_valid_col].filled(False) # Sort the table by the subarray event keys joined_subarray_table.sort(SUBARRAY_EVENT_KEYS) # Save the prediction to the file From 3344e53ef73c0df049dc8be0109bc9b242b96b27 Mon Sep 17 00:00:00 2001 From: Tjark Miener <37835610+TjarkMiener@users.noreply.github.com> Date: Sat, 15 Nov 2025 17:45:44 +0100 Subject: [PATCH 06/11] Update scripts/overwrite_isvalid_flag.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- scripts/overwrite_isvalid_flag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/overwrite_isvalid_flag.py b/scripts/overwrite_isvalid_flag.py index 2e882bda..8b5d97a5 100644 --- a/scripts/overwrite_isvalid_flag.py +++ b/scripts/overwrite_isvalid_flag.py @@ -136,7 +136,7 @@ class OverwriteIsValidFlag(Tool): } def setup(self): - # Check if the ctapipe HDF5Merger component is enabled + # Check if output file already exists if os.path.exists(self.output_path): raise ToolConfigurationError( f"The output file '{self.output_path}' already exists. Please set " From 191298cb2e09779f81e7604f3b7870ff268f49c0 Mon Sep 17 00:00:00 2001 From: Tjark Miener <37835610+TjarkMiener@users.noreply.github.com> Date: Sat, 15 Nov 2025 17:46:21 +0100 Subject: [PATCH 07/11] Update scripts/overwrite_isvalid_flag.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- scripts/overwrite_isvalid_flag.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/overwrite_isvalid_flag.py b/scripts/overwrite_isvalid_flag.py index 8b5d97a5..c35b27dc 100644 --- a/scripts/overwrite_isvalid_flag.py +++ b/scripts/overwrite_isvalid_flag.py @@ -165,12 +165,12 @@ def start(self): self.log.info("Processing telescope '%03d' ...", tel_id) input_tel_table = read_table( self.is_valid_from, - f"{DL2_TELESCOPE_GROUP}/{reco_task}/{self.prefix}/tel_{tel_id:03}", + f"{DL2_TELESCOPE_GROUP}/{reco_task}/{self.prefix}/tel_{tel_id:03d}", ) input_tel_table.keep_columns(TELESCOPE_EVENT_KEYS + [is_valid_col]) output_tel_table = read_table( self.is_valid_to, - f"{DL2_TELESCOPE_GROUP}/{reco_task}/{self.prefix}/tel_{tel_id:03}", + f"{DL2_TELESCOPE_GROUP}/{reco_task}/{self.prefix}/tel_{tel_id:03d}", ) output_tel_table.remove_columns([is_valid_col]) if len(input_tel_table) != len(output_tel_table): From 35a2e7dddf6c72475560837fc8d61031221d5f7b Mon Sep 17 00:00:00 2001 From: Tjark Miener <37835610+TjarkMiener@users.noreply.github.com> Date: Sat, 15 Nov 2025 17:46:29 +0100 Subject: [PATCH 08/11] Update scripts/overwrite_isvalid_flag.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- scripts/overwrite_isvalid_flag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/overwrite_isvalid_flag.py b/scripts/overwrite_isvalid_flag.py index c35b27dc..80fabaec 100644 --- a/scripts/overwrite_isvalid_flag.py +++ b/scripts/overwrite_isvalid_flag.py @@ -188,7 +188,7 @@ def start(self): ) # Fill missing values in the is_valid column with False if necessary if isinstance(joined_tel_table[is_valid_col], MaskedColumn): - joined_tel_table[is_valid_col] = joined_tel_table[is_valid_col].filled(False) + joined_tel_table[is_valid_col] = joined_tel_table[is_valid_col].filled(False) # Sort the table by the telescope event keys joined_tel_table.sort(TELESCOPE_EVENT_KEYS) write_table( From 734511b4f9a317667e60d2d0b6d77e97990e83b7 Mon Sep 17 00:00:00 2001 From: Tjark Miener <37835610+TjarkMiener@users.noreply.github.com> Date: Sat, 15 Nov 2025 17:46:39 +0100 Subject: [PATCH 09/11] Update scripts/overwrite_isvalid_flag.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- scripts/overwrite_isvalid_flag.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/overwrite_isvalid_flag.py b/scripts/overwrite_isvalid_flag.py index 80fabaec..48f62635 100644 --- a/scripts/overwrite_isvalid_flag.py +++ b/scripts/overwrite_isvalid_flag.py @@ -157,7 +157,6 @@ def start(self): # Loop over the reconstruction tasks and combine the telescope tables to a subarray table for reco_task in self.reco_tasks: self.log.info("Processing %s...", reco_task) - # Read the telescope tables from the input file if self.dl2_telescope: is_valid_col = f"{self.prefix}_tel_is_valid" From 57ca39347dfb016348f3d5678ec30a35d7164801 Mon Sep 17 00:00:00 2001 From: Tjark Miener <37835610+TjarkMiener@users.noreply.github.com> Date: Sat, 15 Nov 2025 17:46:46 +0100 Subject: [PATCH 10/11] Update scripts/overwrite_isvalid_flag.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- scripts/overwrite_isvalid_flag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/overwrite_isvalid_flag.py b/scripts/overwrite_isvalid_flag.py index 48f62635..d94a834b 100644 --- a/scripts/overwrite_isvalid_flag.py +++ b/scripts/overwrite_isvalid_flag.py @@ -102,7 +102,7 @@ class OverwriteIsValidFlag(Tool): help="List of reconstruction tasks to be used for the overwrite of the is_valid flag.", ).tag(config=True) - dl2_telescope= Bool( + dl2_telescope = Bool( default_value=True, help="Whether to overwrite the is_valid flag in the dl2 telescope group.", ).tag(config=True) From 0e45b5af7508812b238f8c744b442308bba68d36 Mon Sep 17 00:00:00 2001 From: Tjark Miener <37835610+TjarkMiener@users.noreply.github.com> Date: Sat, 15 Nov 2025 17:46:59 +0100 Subject: [PATCH 11/11] Update scripts/overwrite_isvalid_flag.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- scripts/overwrite_isvalid_flag.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/overwrite_isvalid_flag.py b/scripts/overwrite_isvalid_flag.py index d94a834b..e7c06ad2 100644 --- a/scripts/overwrite_isvalid_flag.py +++ b/scripts/overwrite_isvalid_flag.py @@ -3,7 +3,6 @@ """ from astropy.table import join, MaskedColumn -import numpy as np import os from ctapipe.io import read_table, write_table, HDF5Merger