Note
Go to the end to download the full example code.
Creating command-line Tools#
7 from time import sleep
8
9 from ctapipe.core import Component, TelescopeComponent, Tool
10 from ctapipe.core.traits import (
11 FloatTelescopeParameter,
12 Integer,
13 Path,
14 TraitError,
15 observe,
16 )
17 from ctapipe.instrument import SubarrayDescription
18 from ctapipe.utils import get_dataset_path
21 GAMMA_FILE = get_dataset_path("gamma_prod5.simtel.zst")
Setup:#
Create a few Components that we will use later in a Tool:
37 class MyComponent(Component):
38 """A Component that does stuff"""
39
40 value = Integer(default_value=-1, help="Value to use").tag(config=True)
41
42 def do_thing(self):
43 self.log.debug("Did thing")
44
45
46 # in order to have 2 of the same components at once
47 class SecondaryMyComponent(MyComponent):
48 """A second component"""
49
50
51 class AdvancedComponent(Component):
52 """An advanced technique"""
53
54 value1 = Integer(default_value=-1, help="Value to use").tag(config=True)
55 infile = Path(
56 help="input file name",
57 exists=None, # set to True to require existing, False for requiring non-existing
58 directory_ok=False,
59 ).tag(config=True)
60 outfile = Path(help="output file name", exists=False, directory_ok=False).tag(
61 config=True
62 )
63
64 def __init__(self, config=None, parent=None, **kwargs):
65 super().__init__(config=config, parent=parent, **kwargs)
66 # components can have sub components, but these must have
67 # then parent=self as argument and be assigned as member
68 # so the full config can be received later
69 self.subcompent = MyComponent(parent=self)
70
71 @observe("outfile")
72 def on_outfile_changed(self, change):
73 self.log.warning("Outfile was changed to '{}'".format(change))
74
75
76 class TelescopeWiseComponent(TelescopeComponent):
77 """a component that contains parameters that are per-telescope configurable"""
78
79 param = FloatTelescopeParameter(
80 help="Something configurable with telescope patterns", default_value=5.0
81 ).tag(config=True)
85 MyComponent()
88 AdvancedComponent(infile="test.foo", outfile="out.foo")
Outfile was changed to '{'name': 'outfile', 'old': None, 'new': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.25.1/examples/core/out.foo'), 'owner': <__main__.AdvancedComponent object at 0x7918d4610370>, 'type': 'change'}'
TelescopeComponents need to have a subarray given to them in order
to work (since they need one to turn a TelescopeParameter into a
concrete list of values for each telescope. Here we will give a dummy
one:
99 subarray = SubarrayDescription.read(GAMMA_FILE)
100 subarray.info()
Subarray : MonteCarloArray
Num Tels : 180
Footprint: 4.92 km2
Height : 2147.00 m
Lon/Lat : 0.0 deg, 0.0 deg
Type Count Tel IDs
----------------- ----- ---------------
SST_ASTRI_CHEC 120 30-99,131-180
LST_LST_LSTCam 4 1-4
MST_MST_FlashCam 28 5-29,125-127
MST_MST_NectarCam 28 100-124,128-130
This TelescopeParameters can then be set using a list of patterns like:
component.param = [(“type”, “LST*”,3.0),(“type”, “MST*”, 2.0),(id, 25, 4.0)]
These get translated into per-telescope-id values once the subarray is registered. After that one access the per-telescope id values via:
component.param.tel[tel_id]
Now create an executable Tool that contains the Components#
Note that all the components we wish to be configured via the tool must
be added to the classes attribute.
129 class MyTool(Tool):
130 name = "mytool"
131 description = "do some things and stuff"
132 aliases = dict(
133 infile="AdvancedComponent.infile",
134 outfile="AdvancedComponent.outfile",
135 iterations="MyTool.iterations",
136 )
137
138 # Which classes are registered for configuration
139 classes = [
140 MyComponent,
141 AdvancedComponent,
142 SecondaryMyComponent,
143 TelescopeWiseComponent,
144 ]
145
146 # local configuration parameters
147 iterations = Integer(5, help="Number of times to run", allow_none=False).tag(
148 config=True
149 )
150
151 def setup(self):
152 self.comp = MyComponent(parent=self)
153 self.comp2 = SecondaryMyComponent(parent=self)
154 self.comp3 = TelescopeWiseComponent(parent=self, subarray=subarray)
155 self.advanced = AdvancedComponent(parent=self)
156
157 def start(self):
158 self.log.info("Performing {} iterations...".format(self.iterations))
159 for ii in range(self.iterations):
160 self.log.info("ITERATION {}".format(ii))
161 self.comp.do_thing()
162 self.comp2.do_thing()
163 sleep(0.1)
164
165 def finish(self):
166 self.log.warning("Shutting down.")
Get Help info#
The following allows you to print the help info within a Jupyter notebook, but this same information would be displayed if the user types:
mytool --help
182 tool = MyTool()
183 tool
186 tool.print_help()
do some things and stuff
Options
=======
The options below are convenience aliases to configurable class-options,
as listed in the "Equivalent to" description-line of the aliases.
To see all configurable class-options for some <cmd>, use:
<cmd> --help-all
--debug
Set log-level to debug, for the most verbose logging.
Equivalent to: [--Application.log_level=10]
--show-config
Show the application's configuration (human-readable format)
Equivalent to: [--Application.show_config=True]
--show-config-json
Show the application's configuration (json format)
Equivalent to: [--Application.show_config_json=True]
-q, --quiet
Disable console logging.
Equivalent to: [--Tool.quiet=True]
-v, --verbose
Set log level to DEBUG
Equivalent to: [--Tool.log_level=DEBUG]
--overwrite
Overwrite existing output files without asking
Equivalent to: [--Tool.overwrite=True]
-c, --config=<list-item-1>...
List of configuration files with parameters to load in addition to command-
line parameters. The order listed is the order of precedence (later config
parameters overwrite earlier ones), however parameters specified on the
command line always have the highest precedence. Config files may be in
JSON, YAML, TOML, or Python format
Default: []
Equivalent to: [--Tool.config_files]
--log-level=<Enum>
Set the log level by value or name.
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 30
Equivalent to: [--Tool.log_level]
-l, --log-file=<Path>
Filename for the log
Default: None
Equivalent to: [--Tool.log_file]
--log-file-level=<Enum>
Logging Level for File Logging
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 'INFO'
Equivalent to: [--Tool.log_file_level]
--provenance-log=<Path>
Default: traitlets.Undefined
Equivalent to: [--Tool.provenance_log]
--infile=<Path>
input file name
Default: traitlets.Undefined
Equivalent to: [--AdvancedComponent.infile]
--outfile=<Path>
output file name
Default: traitlets.Undefined
Equivalent to: [--AdvancedComponent.outfile]
--iterations=<Long>
Number of times to run
Default: 5
Equivalent to: [--MyTool.iterations]
To see all available configurables, use `--help-all`.
The following is equivalent to the user typing mytool --help-all
193 tool.print_help(classes=True)
do some things and stuff
Options
=======
The options below are convenience aliases to configurable class-options,
as listed in the "Equivalent to" description-line of the aliases.
To see all configurable class-options for some <cmd>, use:
<cmd> --help-all
--debug
Set log-level to debug, for the most verbose logging.
Equivalent to: [--Application.log_level=10]
--show-config
Show the application's configuration (human-readable format)
Equivalent to: [--Application.show_config=True]
--show-config-json
Show the application's configuration (json format)
Equivalent to: [--Application.show_config_json=True]
-q, --quiet
Disable console logging.
Equivalent to: [--Tool.quiet=True]
-v, --verbose
Set log level to DEBUG
Equivalent to: [--Tool.log_level=DEBUG]
--overwrite
Overwrite existing output files without asking
Equivalent to: [--Tool.overwrite=True]
-c, --config=<list-item-1>...
List of configuration files with parameters to load in addition to command-
line parameters. The order listed is the order of precedence (later config
parameters overwrite earlier ones), however parameters specified on the
command line always have the highest precedence. Config files may be in
JSON, YAML, TOML, or Python format
Default: []
Equivalent to: [--Tool.config_files]
--log-level=<Enum>
Set the log level by value or name.
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 30
Equivalent to: [--Tool.log_level]
-l, --log-file=<Path>
Filename for the log
Default: None
Equivalent to: [--Tool.log_file]
--log-file-level=<Enum>
Logging Level for File Logging
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 'INFO'
Equivalent to: [--Tool.log_file_level]
--provenance-log=<Path>
Default: traitlets.Undefined
Equivalent to: [--Tool.provenance_log]
--infile=<Path>
input file name
Default: traitlets.Undefined
Equivalent to: [--AdvancedComponent.infile]
--outfile=<Path>
output file name
Default: traitlets.Undefined
Equivalent to: [--AdvancedComponent.outfile]
--iterations=<Long>
Number of times to run
Default: 5
Equivalent to: [--MyTool.iterations]
Class options
=============
The command-line option below sets the respective configurable class-parameter:
--Class.parameter=value
This line is evaluated in Python, so simple expressions are allowed.
For instance, to set `C.a=[0,1,2]`, you may type this:
--C.a='range(3)'
Application(SingletonConfigurable) options
------------------------------------------
--Application.log_datefmt=<Unicode>
The date format used by logging formatters for %(asctime)s
Default: '%Y-%m-%d %H:%M:%S'
--Application.log_format=<Unicode>
The Logging format template
Default: '[%(name)s]%(highlevel)s %(message)s'
--Application.log_level=<Enum>
Set the log level by value or name.
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 30
--Application.logging_config=<key-1>=<value-1>...
Configure additional log handlers.
The default stderr logs handler is configured by the log_level, log_datefmt
and log_format settings.
This configuration can be used to configure additional handlers (e.g. to
output the log to a file) or for finer control over the default handlers.
If provided this should be a logging configuration dictionary, for more
information see:
https://docs.python.org/3/library/logging.config.html#logging-config-
dictschema
This dictionary is merged with the base logging configuration which defines
the following:
* A logging formatter intended for interactive use called
``console``.
* A logging handler that writes to stderr called
``console`` which uses the formatter ``console``.
* A logger with the name of this application set to ``DEBUG``
level.
This example adds a new handler that writes to a file:
.. code-block:: python
c.Application.logging_config = {
"handlers": {
"file": {
"class": "logging.FileHandler",
"level": "DEBUG",
"filename": "<path/to/file>",
}
},
"loggers": {
"<application-name>": {
"level": "DEBUG",
# NOTE: if you don't list the default "console"
# handler here then it will be disabled
"handlers": ["console", "file"],
},
},
}
Default: {}
--Application.show_config=<Bool>
Instead of starting the Application, dump configuration to stdout
Default: False
--Application.show_config_json=<Bool>
Instead of starting the Application, dump configuration to stdout (as JSON)
Default: False
Tool(Application) options
-------------------------
--Tool.config_files=<list-item-1>...
List of configuration files with parameters to load in addition to command-
line parameters. The order listed is the order of precedence (later config
parameters overwrite earlier ones), however parameters specified on the
command line always have the highest precedence. Config files may be in
JSON, YAML, TOML, or Python format
Default: []
--Tool.log_config=<key-1>=<value-1>...
Default: {}
--Tool.log_datefmt=<Unicode>
The date format used by logging formatters for %(asctime)s
Default: '%Y-%m-%d %H:%M:%S'
--Tool.log_file=<Path>
Filename for the log
Default: None
--Tool.log_file_level=<Enum>
Logging Level for File Logging
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 'INFO'
--Tool.log_format=<Unicode>
The Logging format template
Default: '[%(name)s]%(highlevel)s %(message)s'
--Tool.log_level=<Enum>
Set the log level by value or name.
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 30
--Tool.logging_config=<key-1>=<value-1>...
Configure additional log handlers.
The default stderr logs handler is configured by the log_level, log_datefmt
and log_format settings.
This configuration can be used to configure additional handlers (e.g. to
output the log to a file) or for finer control over the default handlers.
If provided this should be a logging configuration dictionary, for more
information see:
https://docs.python.org/3/library/logging.config.html#logging-config-
dictschema
This dictionary is merged with the base logging configuration which defines
the following:
* A logging formatter intended for interactive use called
``console``.
* A logging handler that writes to stderr called
``console`` which uses the formatter ``console``.
* A logger with the name of this application set to ``DEBUG``
level.
This example adds a new handler that writes to a file:
.. code-block:: python
c.Application.logging_config = {
"handlers": {
"file": {
"class": "logging.FileHandler",
"level": "DEBUG",
"filename": "<path/to/file>",
}
},
"loggers": {
"<application-name>": {
"level": "DEBUG",
# NOTE: if you don't list the default "console"
# handler here then it will be disabled
"handlers": ["console", "file"],
},
},
}
Default: {}
--Tool.overwrite=<Bool>
Default: False
--Tool.provenance_log=<Path>
Default: traitlets.Undefined
--Tool.quiet=<Bool>
Default: False
--Tool.show_config=<Bool>
Instead of starting the Application, dump configuration to stdout
Default: False
--Tool.show_config_json=<Bool>
Instead of starting the Application, dump configuration to stdout (as JSON)
Default: False
MyTool(Tool) options
--------------------
--MyTool.config_files=<list-item-1>...
List of configuration files with parameters to load in addition to command-
line parameters. The order listed is the order of precedence (later config
parameters overwrite earlier ones), however parameters specified on the
command line always have the highest precedence. Config files may be in
JSON, YAML, TOML, or Python format
Default: []
--MyTool.iterations=<Long>
Number of times to run
Default: 5
--MyTool.log_config=<key-1>=<value-1>...
Default: {}
--MyTool.log_datefmt=<Unicode>
The date format used by logging formatters for %(asctime)s
Default: '%Y-%m-%d %H:%M:%S'
--MyTool.log_file=<Path>
Filename for the log
Default: None
--MyTool.log_file_level=<Enum>
Logging Level for File Logging
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 'INFO'
--MyTool.log_format=<Unicode>
The Logging format template
Default: '[%(name)s]%(highlevel)s %(message)s'
--MyTool.log_level=<Enum>
Set the log level by value or name.
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 30
--MyTool.logging_config=<key-1>=<value-1>...
Configure additional log handlers.
The default stderr logs handler is configured by the log_level, log_datefmt
and log_format settings.
This configuration can be used to configure additional handlers (e.g. to
output the log to a file) or for finer control over the default handlers.
If provided this should be a logging configuration dictionary, for more
information see:
https://docs.python.org/3/library/logging.config.html#logging-config-
dictschema
This dictionary is merged with the base logging configuration which defines
the following:
* A logging formatter intended for interactive use called
``console``.
* A logging handler that writes to stderr called
``console`` which uses the formatter ``console``.
* A logger with the name of this application set to ``DEBUG``
level.
This example adds a new handler that writes to a file:
.. code-block:: python
c.Application.logging_config = {
"handlers": {
"file": {
"class": "logging.FileHandler",
"level": "DEBUG",
"filename": "<path/to/file>",
}
},
"loggers": {
"<application-name>": {
"level": "DEBUG",
# NOTE: if you don't list the default "console"
# handler here then it will be disabled
"handlers": ["console", "file"],
},
},
}
Default: {}
--MyTool.overwrite=<Bool>
Default: False
--MyTool.provenance_log=<Path>
Default: traitlets.Undefined
--MyTool.quiet=<Bool>
Default: False
--MyTool.show_config=<Bool>
Instead of starting the Application, dump configuration to stdout
Default: False
--MyTool.show_config_json=<Bool>
Instead of starting the Application, dump configuration to stdout (as JSON)
Default: False
MyComponent(Component) options
------------------------------
--MyComponent.value=<Long>
Value to use
Default: -1
AdvancedComponent(Component) options
------------------------------------
--AdvancedComponent.infile=<Path>
input file name
Default: traitlets.Undefined
--AdvancedComponent.outfile=<Path>
output file name
Default: traitlets.Undefined
--AdvancedComponent.value1=<Long>
Value to use
Default: -1
SecondaryMyComponent(MyComponent) options
-----------------------------------------
--SecondaryMyComponent.value=<Long>
Value to use
Default: -1
TelescopeWiseComponent(TelescopeComponent) options
--------------------------------------------------
--TelescopeWiseComponent.param=<floattelescopeparameter-item-1>...
Something configurable with telescope patterns.
Default: [('type', '*', 5.0)]
Run the tool#
here we pass in argv since it is a Notebook, but if argv is not
specified it’s read from sys.argv, so the following is the same as
running:
mytool –log_level=INFO –infile gamma_test.simtel.gz –iterations=3
As Tools are intended to be executables, they are raising
SystemExit on exit. Here, we use them to demonstrate how it would
work, so we catch the SystemExit.
212 try:
213 tool.run(argv=["--infile", str(GAMMA_FILE), "--outfile", "out.csv"])
214 except SystemExit as e:
215 assert e.code == 0, f"Tool returned with error status {e}"
216
217 tool.log_format = "%(asctime)s : %(levelname)s [%(name)s %(funcName)s] %(message)s"
2025-06-06 08:53:08,430 WARNING [__main__.mytool.AdvancedComponent] (command_line_tools.on_outfile_changed): Outfile was changed to '{'name': 'outfile', 'old': traitlets.Undefined, 'new': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.25.1/examples/core/out.csv'), 'owner': <__main__.AdvancedComponent object at 0x7918e44f6a70>, 'type': 'change'}'
2025-06-06 08:53:08,931 WARNING [__main__.mytool] (command_line_tools.finish): Shutting down.
221 try:
222 tool.run(
223 argv=[
224 "--log-level",
225 "INFO",
226 "--infile",
227 str(GAMMA_FILE),
228 "--outfile",
229 "out.csv",
230 "--iterations",
231 "3",
232 ]
233 )
234 except SystemExit as e:
235 assert e.code == 0, f"Tool returned with error status {e}"
2025-06-06 08:53:09,146 INFO [__main__.mytool] (tool.initialize): Loading config from '[]'
2025-06-06 08:53:09,147 INFO [__main__.mytool] (tool.initialize): ctapipe version 0.25.1
2025-06-06 08:53:09,149 WARNING [__main__.mytool.AdvancedComponent] (command_line_tools.on_outfile_changed): Outfile was changed to '{'name': 'outfile', 'old': traitlets.Undefined, 'new': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.25.1/examples/core/out.csv'), 'owner': <__main__.AdvancedComponent object at 0x7918e44f46d0>, 'type': 'change'}'
2025-06-06 08:53:09,149 INFO [__main__.mytool] (command_line_tools.start): Performing 3 iterations...
2025-06-06 08:53:09,149 INFO [__main__.mytool] (command_line_tools.start): ITERATION 0
2025-06-06 08:53:09,250 INFO [__main__.mytool] (command_line_tools.start): ITERATION 1
2025-06-06 08:53:09,350 INFO [__main__.mytool] (command_line_tools.start): ITERATION 2
2025-06-06 08:53:09,450 WARNING [__main__.mytool] (command_line_tools.finish): Shutting down.
2025-06-06 08:53:09,452 INFO [__main__.mytool] (tool.write_provenance): Output:
2025-06-06 08:53:09,452 INFO [__main__.mytool] (tool.write_provenance): Output:
2025-06-06 08:53:09,454 INFO [__main__.mytool] (tool.run): Finished mytool
here we change the log-level to DEBUG:
242 try:
243 tool.run(
244 argv=[
245 "--log-level",
246 "DEBUG",
247 "--infile",
248 str(GAMMA_FILE),
249 "--outfile",
250 "out.csv",
251 ]
252 )
253 except SystemExit as e:
254 assert e.code == 0, f"Tool returned with error status {e}"
2025-06-06 08:53:09,455 INFO [__main__.mytool] (tool.run): Starting: mytool
2025-06-06 08:53:09,665 INFO [__main__.mytool] (tool.initialize): Loading config from '[]'
2025-06-06 08:53:09,666 INFO [__main__.mytool] (tool.initialize): ctapipe version 0.25.1
2025-06-06 08:53:09,668 WARNING [__main__.mytool.AdvancedComponent] (command_line_tools.on_outfile_changed): Outfile was changed to '{'name': 'outfile', 'old': traitlets.Undefined, 'new': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.25.1/examples/core/out.csv'), 'owner': <__main__.AdvancedComponent object at 0x7918d523ce50>, 'type': 'change'}'
2025-06-06 08:53:09,668 DEBUG [__main__.mytool] (tool.run): CONFIG: {'MyTool': {'config_files': [], 'iterations': 3, 'log_config': {}, 'log_datefmt': '%Y-%m-%d %H:%M:%S', 'log_file': None, 'log_file_level': 'INFO', 'log_format': '%(asctime)s : %(levelname)s [%(name)s %(funcName)s] %(message)s', 'log_level': 10, 'logging_config': {}, 'overwrite': False, 'provenance_log': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.25.1/docs/mytool.provenance.log'), 'quiet': False, 'show_config': False, 'show_config_json': False, 'MyComponent': {'value': -1}, 'SecondaryMyComponent': {'value': -1}, 'TelescopeWiseComponent': {'param': [('type', '*', 5.0)]}, 'AdvancedComponent': {'infile': PosixPath('/home/docs/.cache/ctapipe/minio-cta.zeuthen.desy.de/dpps-testdata-public/data/ctapipe-test-data/v1.1.0/gamma_prod5.simtel.zst'), 'outfile': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.25.1/examples/core/out.csv'), 'value1': -1, 'MyComponent': {'value': -1}}}}
2025-06-06 08:53:09,668 INFO [__main__.mytool] (command_line_tools.start): Performing 3 iterations...
2025-06-06 08:53:09,668 INFO [__main__.mytool] (command_line_tools.start): ITERATION 0
2025-06-06 08:53:09,668 DEBUG [__main__.mytool.MyComponent] (command_line_tools.do_thing): Did thing
2025-06-06 08:53:09,668 DEBUG [__main__.mytool.SecondaryMyComponent] (command_line_tools.do_thing): Did thing
2025-06-06 08:53:09,769 INFO [__main__.mytool] (command_line_tools.start): ITERATION 1
2025-06-06 08:53:09,769 DEBUG [__main__.mytool.MyComponent] (command_line_tools.do_thing): Did thing
2025-06-06 08:53:09,769 DEBUG [__main__.mytool.SecondaryMyComponent] (command_line_tools.do_thing): Did thing
2025-06-06 08:53:09,869 INFO [__main__.mytool] (command_line_tools.start): ITERATION 2
2025-06-06 08:53:09,869 DEBUG [__main__.mytool.MyComponent] (command_line_tools.do_thing): Did thing
2025-06-06 08:53:09,869 DEBUG [__main__.mytool.SecondaryMyComponent] (command_line_tools.do_thing): Did thing
2025-06-06 08:53:09,969 WARNING [__main__.mytool] (command_line_tools.finish): Shutting down.
2025-06-06 08:53:09,971 INFO [__main__.mytool] (tool.write_provenance): Output:
2025-06-06 08:53:09,971 INFO [__main__.mytool] (tool.write_provenance): Output:
2025-06-06 08:53:09,971 INFO [__main__.mytool] (tool.write_provenance): Output:
2025-06-06 08:53:09,971 DEBUG [__main__.mytool] (tool.write_provenance): PROVENANCE: 'Details about provenance is found in /home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.25.1/docs/mytool.provenance.log'
2025-06-06 08:53:09,974 INFO [__main__.mytool] (tool.run): Finished mytool
2025-06-06 08:53:09,975 DEBUG [__main__.mytool] (application.exit): Exiting application: mytool
you can also set parameters directly in the class, rather than using the argument/configfile parser. This is useful if you are calling the Tool from a script rather than the command-line
263 tool.iterations = 1
264 tool.log_level = 0
265
266 try:
267 tool.run(["--infile", str(GAMMA_FILE), "--outfile", "out.csv"])
268 except SystemExit as e:
269 assert e.code == 0, f"Tool returned with error status {e}"
2025-06-06 08:53:09,976 INFO [__main__.mytool] (tool.run): Starting: mytool
2025-06-06 08:53:10,187 WARNING [__main__.mytool.AdvancedComponent] (command_line_tools.on_outfile_changed): Outfile was changed to '{'name': 'outfile', 'old': traitlets.Undefined, 'new': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.25.1/examples/core/out.csv'), 'owner': <__main__.AdvancedComponent object at 0x7918d523fb50>, 'type': 'change'}'
2025-06-06 08:53:10,288 WARNING [__main__.mytool] (command_line_tools.finish): Shutting down.
see what happens when a value is set that is not of the correct type:
276 try:
277 tool.iterations = "badval"
278 except TraitError as E:
279 print("bad value:", E)
280 except SystemExit as e:
281 assert e.code == 0, f"Tool returned with error status {e}"
bad value: The 'iterations' trait of a MyTool instance expected an int, not the str 'badval'.
Example of what happens when you change a parameter that is being “observed” in a class. It’s handler is called:
289 tool.advanced.outfile = "Another.txt"
2025-06-06 08:53:10,294 WARNING [__main__.mytool.AdvancedComponent] (command_line_tools.on_outfile_changed): Outfile was changed to '{'name': 'outfile', 'old': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.25.1/examples/core/out.csv'), 'new': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.25.1/examples/core/Another.txt'), 'owner': <__main__.AdvancedComponent object at 0x7918d523fb50>, 'type': 'change'}'
we see that the handler for outfile was called, and it receive a
change dict that shows the old and new values.
create a tool using a config file:
302 tool2 = MyTool()
305 try:
306 tool2.run(argv=["--config", "config.json"])
307 except SystemExit as e:
308 assert e.code == 0, f"Tool returned with error status {e}"
2025-06-06 08:53:10,508 INFO [__main__.mytool] (tool.initialize): ctapipe version 0.25.1
2025-06-06 08:53:10,510 WARNING [__main__.mytool.AdvancedComponent] (command_line_tools.on_outfile_changed): Outfile was changed to '{'name': 'outfile', 'old': traitlets.Undefined, 'new': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.25.1/examples/core/foo.txt'), 'owner': <__main__.AdvancedComponent object at 0x7918e44f4340>, 'type': 'change'}'
2025-06-06 08:53:10,510 DEBUG [__main__.mytool] (tool.run): CONFIG: {'MyTool': {'config_files': [PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.25.1/examples/core/config.json')], 'iterations': 5, 'log_config': {}, 'log_datefmt': '%Y-%m-%d %H:%M:%S', 'log_file': None, 'log_file_level': 'INFO', 'log_format': '[%(name)s]%(highlevel)s %(message)s', 'log_level': 10, 'logging_config': {}, 'overwrite': False, 'provenance_log': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.25.1/examples/core/mytool.provenance.log'), 'quiet': False, 'show_config': False, 'show_config_json': False, 'MyComponent': {'value': -1}, 'SecondaryMyComponent': {'value': -1}, 'TelescopeWiseComponent': {'param': [('type', '*', 5.0)]}, 'AdvancedComponent': {'infile': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.25.1/examples/core/something.txt'), 'outfile': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.25.1/examples/core/foo.txt'), 'value1': -1, 'MyComponent': {'value': -1}}}}
2025-06-06 08:53:10,511 INFO [__main__.mytool] (command_line_tools.start): Performing 5 iterations...
2025-06-06 08:53:10,511 INFO [__main__.mytool] (command_line_tools.start): ITERATION 0
2025-06-06 08:53:10,511 DEBUG [__main__.mytool.MyComponent] (command_line_tools.do_thing): Did thing
2025-06-06 08:53:10,511 DEBUG [__main__.mytool.SecondaryMyComponent] (command_line_tools.do_thing): Did thing
2025-06-06 08:53:10,611 INFO [__main__.mytool] (command_line_tools.start): ITERATION 1
2025-06-06 08:53:10,611 DEBUG [__main__.mytool.MyComponent] (command_line_tools.do_thing): Did thing
2025-06-06 08:53:10,611 DEBUG [__main__.mytool.SecondaryMyComponent] (command_line_tools.do_thing): Did thing
2025-06-06 08:53:10,711 INFO [__main__.mytool] (command_line_tools.start): ITERATION 2
2025-06-06 08:53:10,711 DEBUG [__main__.mytool.MyComponent] (command_line_tools.do_thing): Did thing
2025-06-06 08:53:10,711 DEBUG [__main__.mytool.SecondaryMyComponent] (command_line_tools.do_thing): Did thing
2025-06-06 08:53:10,812 INFO [__main__.mytool] (command_line_tools.start): ITERATION 3
2025-06-06 08:53:10,812 DEBUG [__main__.mytool.MyComponent] (command_line_tools.do_thing): Did thing
2025-06-06 08:53:10,812 DEBUG [__main__.mytool.SecondaryMyComponent] (command_line_tools.do_thing): Did thing
2025-06-06 08:53:10,912 INFO [__main__.mytool] (command_line_tools.start): ITERATION 4
2025-06-06 08:53:10,912 DEBUG [__main__.mytool.MyComponent] (command_line_tools.do_thing): Did thing
2025-06-06 08:53:10,912 DEBUG [__main__.mytool.SecondaryMyComponent] (command_line_tools.do_thing): Did thing
2025-06-06 08:53:11,012 WARNING [__main__.mytool] (command_line_tools.finish): Shutting down.
2025-06-06 08:53:11,014 INFO [__main__.mytool] (tool.write_provenance): Output:
2025-06-06 08:53:11,014 INFO [__main__.mytool] (tool.write_provenance): Output:
2025-06-06 08:53:11,014 INFO [__main__.mytool] (tool.write_provenance): Output:
2025-06-06 08:53:11,014 INFO [__main__.mytool] (tool.write_provenance): Output:
2025-06-06 08:53:11,014 INFO [__main__.mytool] (tool.write_provenance): Output:
2025-06-06 08:53:11,014 DEBUG [__main__.mytool] (tool.write_provenance): PROVENANCE: 'Details about provenance is found in /home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.25.1/examples/core/mytool.provenance.log'
2025-06-06 08:53:11,019 INFO [__main__.mytool] (tool.run): Finished mytool
2025-06-06 08:53:11,019 DEBUG [__main__.mytool] (application.exit): Exiting application: mytool
311 print(tool2.advanced.infile)
/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.25.1/examples/core/something.txt
314 print(tool2.config)
{'MyTool': {'config_files': ['config.json'], 'log_level': 'DEBUG'}, 'AdvancedComponent': {'infile': 'something.txt', 'outfile': 'foo.txt'}}
317 tool2.is_setup
True
320 tool3 = MyTool()
323 tool3.is_setup
False
326 tool3.initialize(argv=[])
329 tool3.is_setup
False
332 tool3
335 tool.setup()
336 tool
2025-06-06 08:53:11,030 WARNING [__main__.mytool.AdvancedComponent] (command_line_tools.on_outfile_changed): Outfile was changed to '{'name': 'outfile', 'old': traitlets.Undefined, 'new': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.25.1/examples/core/out.csv'), 'owner': <__main__.AdvancedComponent object at 0x7918e44f68c0>, 'type': 'change'}'
339 tool.comp2
Getting the configuration of an instance#
347 tool.get_current_config()
{'MyTool': {'config_files': [], 'iterations': 1, 'log_config': {}, 'log_datefmt': '%Y-%m-%d %H:%M:%S', 'log_file': None, 'log_file_level': 'INFO', 'log_format': '%(asctime)s : %(levelname)s [%(name)s %(funcName)s] %(message)s', 'log_level': 0, 'logging_config': {}, 'overwrite': False, 'provenance_log': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.25.1/docs/mytool.provenance.log'), 'quiet': False, 'show_config': False, 'show_config_json': False, 'MyComponent': {'value': -1}, 'SecondaryMyComponent': {'value': -1}, 'TelescopeWiseComponent': {'param': [('type', '*', 5.0)]}, 'AdvancedComponent': {'infile': PosixPath('/home/docs/.cache/ctapipe/minio-cta.zeuthen.desy.de/dpps-testdata-public/data/ctapipe-test-data/v1.1.0/gamma_prod5.simtel.zst'), 'outfile': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.25.1/examples/core/out.csv'), 'value1': -1, 'MyComponent': {'value': -1}}}}
350 tool.iterations = 12
351 tool.get_current_config()
{'MyTool': {'config_files': [], 'iterations': 12, 'log_config': {}, 'log_datefmt': '%Y-%m-%d %H:%M:%S', 'log_file': None, 'log_file_level': 'INFO', 'log_format': '%(asctime)s : %(levelname)s [%(name)s %(funcName)s] %(message)s', 'log_level': 0, 'logging_config': {}, 'overwrite': False, 'provenance_log': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.25.1/docs/mytool.provenance.log'), 'quiet': False, 'show_config': False, 'show_config_json': False, 'MyComponent': {'value': -1}, 'SecondaryMyComponent': {'value': -1}, 'TelescopeWiseComponent': {'param': [('type', '*', 5.0)]}, 'AdvancedComponent': {'infile': PosixPath('/home/docs/.cache/ctapipe/minio-cta.zeuthen.desy.de/dpps-testdata-public/data/ctapipe-test-data/v1.1.0/gamma_prod5.simtel.zst'), 'outfile': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.25.1/examples/core/out.csv'), 'value1': -1, 'MyComponent': {'value': -1}}}}
Writing a Sample Config File#
359 print(tool.generate_config_file())
# Configuration file for mytool.
c = get_config() #noqa
#------------------------------------------------------------------------------
# Application(SingletonConfigurable) configuration
#------------------------------------------------------------------------------
## This is an application.
## The date format used by logging formatters for %(asctime)s
# Default: '%Y-%m-%d %H:%M:%S'
# c.Application.log_datefmt = '%Y-%m-%d %H:%M:%S'
## The Logging format template
# Default: '[%(name)s]%(highlevel)s %(message)s'
# c.Application.log_format = '[%(name)s]%(highlevel)s %(message)s'
## Set the log level by value or name.
# Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
# Default: 30
# c.Application.log_level = 30
## Configure additional log handlers.
#
# The default stderr logs handler is configured by the log_level, log_datefmt
# and log_format settings.
#
# This configuration can be used to configure additional handlers (e.g. to
# output the log to a file) or for finer control over the default handlers.
#
# If provided this should be a logging configuration dictionary, for more
# information see:
# https://docs.python.org/3/library/logging.config.html#logging-config-
# dictschema
#
# This dictionary is merged with the base logging configuration which defines
# the following:
#
# * A logging formatter intended for interactive use called
# ``console``.
# * A logging handler that writes to stderr called
# ``console`` which uses the formatter ``console``.
# * A logger with the name of this application set to ``DEBUG``
# level.
#
# This example adds a new handler that writes to a file:
#
# .. code-block:: python
#
# c.Application.logging_config = {
# "handlers": {
# "file": {
# "class": "logging.FileHandler",
# "level": "DEBUG",
# "filename": "<path/to/file>",
# }
# },
# "loggers": {
# "<application-name>": {
# "level": "DEBUG",
# # NOTE: if you don't list the default "console"
# # handler here then it will be disabled
# "handlers": ["console", "file"],
# },
# },
# }
# Default: {}
# c.Application.logging_config = {}
## Instead of starting the Application, dump configuration to stdout
# Default: False
# c.Application.show_config = False
## Instead of starting the Application, dump configuration to stdout (as JSON)
# Default: False
# c.Application.show_config_json = False
#------------------------------------------------------------------------------
# Tool(Application) configuration
#------------------------------------------------------------------------------
## This is an application.
## List of configuration files with parameters to load in addition to command-
# line parameters. The order listed is the order of precedence (later config
# parameters overwrite earlier ones), however parameters specified on the
# command line always have the highest precedence. Config files may be in JSON,
# YAML, TOML, or Python format
# Default: []
# c.Tool.config_files = []
# Default: {}
# c.Tool.log_config = {}
## The date format used by logging formatters for %(asctime)s
# See also: Application.log_datefmt
# c.Tool.log_datefmt = '%Y-%m-%d %H:%M:%S'
## Filename for the log
# Default: None
# c.Tool.log_file = None
## Logging Level for File Logging
# Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
# Default: 'INFO'
# c.Tool.log_file_level = 'INFO'
## The Logging format template
# See also: Application.log_format
# c.Tool.log_format = '[%(name)s]%(highlevel)s %(message)s'
## Set the log level by value or name.
# See also: Application.log_level
# c.Tool.log_level = 30
##
# See also: Application.logging_config
# c.Tool.logging_config = {}
# Default: False
# c.Tool.overwrite = False
# Default: traitlets.Undefined
# c.Tool.provenance_log = traitlets.Undefined
# Default: False
# c.Tool.quiet = False
## Instead of starting the Application, dump configuration to stdout
# See also: Application.show_config
# c.Tool.show_config = False
## Instead of starting the Application, dump configuration to stdout (as JSON)
# See also: Application.show_config_json
# c.Tool.show_config_json = False
#------------------------------------------------------------------------------
# MyTool(Tool) configuration
#------------------------------------------------------------------------------
## List of configuration files with parameters to load in addition to command-
# line parameters. The order listed is the order of precedence (later config
# parameters overwrite earlier ones), however parameters specified on the
# command line always have the highest precedence. Config files may be in JSON,
# YAML, TOML, or Python format
# See also: Tool.config_files
# c.MyTool.config_files = []
## Number of times to run
# Default: 5
# c.MyTool.iterations = 5
# See also: Tool.log_config
# c.MyTool.log_config = {}
## The date format used by logging formatters for %(asctime)s
# See also: Application.log_datefmt
# c.MyTool.log_datefmt = '%Y-%m-%d %H:%M:%S'
## Filename for the log
# See also: Tool.log_file
# c.MyTool.log_file = None
## Logging Level for File Logging
# See also: Tool.log_file_level
# c.MyTool.log_file_level = 'INFO'
## The Logging format template
# See also: Application.log_format
# c.MyTool.log_format = '[%(name)s]%(highlevel)s %(message)s'
## Set the log level by value or name.
# See also: Application.log_level
# c.MyTool.log_level = 30
##
# See also: Application.logging_config
# c.MyTool.logging_config = {}
# See also: Tool.overwrite
# c.MyTool.overwrite = False
# See also: Tool.provenance_log
# c.MyTool.provenance_log = traitlets.Undefined
# See also: Tool.quiet
# c.MyTool.quiet = False
## Instead of starting the Application, dump configuration to stdout
# See also: Application.show_config
# c.MyTool.show_config = False
## Instead of starting the Application, dump configuration to stdout (as JSON)
# See also: Application.show_config_json
# c.MyTool.show_config_json = False
#------------------------------------------------------------------------------
# MyComponent(Component) configuration
#------------------------------------------------------------------------------
## A Component that does stuff
## Value to use
# Default: -1
# c.MyComponent.value = -1
#------------------------------------------------------------------------------
# AdvancedComponent(Component) configuration
#------------------------------------------------------------------------------
## An advanced technique
## input file name
# Default: traitlets.Undefined
# c.AdvancedComponent.infile = traitlets.Undefined
## output file name
# Default: traitlets.Undefined
# c.AdvancedComponent.outfile = traitlets.Undefined
## Value to use
# Default: -1
# c.AdvancedComponent.value1 = -1
#------------------------------------------------------------------------------
# SecondaryMyComponent(MyComponent) configuration
#------------------------------------------------------------------------------
## A second component
## Value to use
# See also: MyComponent.value
# c.SecondaryMyComponent.value = -1
#------------------------------------------------------------------------------
# TelescopeWiseComponent(TelescopeComponent) configuration
#------------------------------------------------------------------------------
## a component that contains parameters that are per-telescope configurable
## Something configurable with telescope patterns.
# Default: [('type', '*', 5.0)]
# c.TelescopeWiseComponent.param = [('type', '*', 5.0)]
Total running time of the script: (0 minutes 3.801 seconds)