Note
Go to the end to download the full example code.
Creating command-line Tools#
7 from time import sleep
8
9 from ctapipe.core import Component, TelescopeComponent, Tool
10 from ctapipe.core.traits import (
11 FloatTelescopeParameter,
12 Integer,
13 Path,
14 TraitError,
15 observe,
16 )
17 from ctapipe.instrument import SubarrayDescription
18 from ctapipe.utils import get_dataset_path
21 GAMMA_FILE = get_dataset_path("gamma_prod5.simtel.zst")
Setup:#
Create a few Components that we will use later in a Tool:
37 class MyComponent(Component):
38 """A Component that does stuff"""
39
40 value = Integer(default_value=-1, help="Value to use").tag(config=True)
41
42 def do_thing(self):
43 self.log.debug("Did thing")
44
45
46 # in order to have 2 of the same components at once
47 class SecondaryMyComponent(MyComponent):
48 """A second component"""
49
50
51 class AdvancedComponent(Component):
52 """An advanced technique"""
53
54 value1 = Integer(default_value=-1, help="Value to use").tag(config=True)
55 infile = Path(
56 help="input file name",
57 exists=None, # set to True to require existing, False for requiring non-existing
58 directory_ok=False,
59 ).tag(config=True)
60 outfile = Path(help="output file name", exists=False, directory_ok=False).tag(
61 config=True
62 )
63
64 def __init__(self, config=None, parent=None, **kwargs):
65 super().__init__(config=config, parent=parent, **kwargs)
66 # components can have sub components, but these must have
67 # then parent=self as argument and be assigned as member
68 # so the full config can be received later
69 self.subcompent = MyComponent(parent=self)
70
71 @observe("outfile")
72 def on_outfile_changed(self, change):
73 self.log.warning("Outfile was changed to '{}'".format(change))
74
75
76 class TelescopeWiseComponent(TelescopeComponent):
77 """a component that contains parameters that are per-telescope configurable"""
78
79 param = FloatTelescopeParameter(
80 help="Something configurable with telescope patterns", default_value=5.0
81 ).tag(config=True)
85 MyComponent()
88 AdvancedComponent(infile="test.foo", outfile="out.foo")
Outfile was changed to '{'name': 'outfile', 'old': None, 'new': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/stable/examples/core/out.foo'), 'owner': <__main__.AdvancedComponent object at 0x7f746c4ea1a0>, 'type': 'change'}'
TelescopeComponents need to have a subarray given to them in order
to work (since they need one to turn a TelescopeParameter into a
concrete list of values for each telescope. Here we will give a dummy
one:
99 subarray = SubarrayDescription.read(GAMMA_FILE)
100 subarray.info()
Subarray : MonteCarloArray
Num Tels : 180
Footprint: 4.92 km2
Height : 2147.00 m
Lon/Lat : 0.0 deg, 0.0 deg
Type Count Tel IDs
----------------- ----- ---------------
SST_ASTRI_CHEC 120 30-99,131-180
LST_LST_LSTCam 4 1-4
MST_MST_NectarCam 28 100-124,128-130
MST_MST_FlashCam 28 5-29,125-127
103 TelescopeWiseComponent(subarray=subarray)
This TelescopeParameters can then be set using a list of patterns like:
component.param = [(“type”, “LST*”,3.0),(“type”, “MST*”, 2.0),(id, 25, 4.0)]
These get translated into per-telescope-id values once the subarray is registered. After that one access the per-telescope id values via:
component.param.tel[tel_id]
Now create an executable Tool that contains the Components#
Note that all the components we wish to be configured via the tool must
be added to the classes attribute.
129 class MyTool(Tool):
130 name = "mytool"
131 description = "do some things and stuff"
132 aliases = dict(
133 infile="AdvancedComponent.infile",
134 outfile="AdvancedComponent.outfile",
135 iterations="MyTool.iterations",
136 )
137
138 # Which classes are registered for configuration
139 classes = [
140 MyComponent,
141 AdvancedComponent,
142 SecondaryMyComponent,
143 TelescopeWiseComponent,
144 ]
145
146 # local configuration parameters
147 iterations = Integer(5, help="Number of times to run", allow_none=False).tag(
148 config=True
149 )
150
151 def setup(self):
152 self.comp = MyComponent(parent=self)
153 self.comp2 = SecondaryMyComponent(parent=self)
154 self.comp3 = TelescopeWiseComponent(parent=self, subarray=subarray)
155 self.advanced = AdvancedComponent(parent=self)
156
157 def start(self):
158 self.log.info("Performing {} iterations...".format(self.iterations))
159 for ii in range(self.iterations):
160 self.log.info("ITERATION {}".format(ii))
161 self.comp.do_thing()
162 self.comp2.do_thing()
163 sleep(0.1)
164
165 def finish(self):
166 self.log.warning("Shutting down.")
Get Help info#
The following allows you to print the help info within a Jupyter notebook, but this same information would be displayed if the user types:
mytool --help
182 tool = MyTool()
183 tool
186 tool.print_help()
do some things and stuff
Options
=======
The options below are convenience aliases to configurable class-options,
as listed in the "Equivalent to" description-line of the aliases.
To see all configurable class-options for some <cmd>, use:
<cmd> --help-all
-q, --quiet
Disable console logging.
Equivalent to: [--Tool.quiet=True]
-v, --verbose
Set log level to DEBUG
Equivalent to: [--Tool.log_level=DEBUG]
--overwrite
Overwrite existing output files without asking
Equivalent to: [--Tool.overwrite=True]
--debug
Set log-level to debug, for the most verbose logging.
Equivalent to: [--Application.log_level=10]
--show-config
Show the application's configuration (human-readable format)
Equivalent to: [--Application.show_config=True]
--show-config-json
Show the application's configuration (json format)
Equivalent to: [--Application.show_config_json=True]
-c, --config=<list-item-1>...
Default: []
Equivalent to: [--Tool.config_files]
--log-level=<Enum>
Set the log level by value or name.
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 30
Equivalent to: [--Tool.log_level]
-l, --log-file=<Path>
Filename for the log
Default: None
Equivalent to: [--Tool.log_file]
--log-file-level=<Enum>
Logging Level for File Logging
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 'INFO'
Equivalent to: [--Tool.log_file_level]
--provenance-log=<Path>
Default: traitlets.Undefined
Equivalent to: [--Tool.provenance_log]
--infile=<Path>
input file name
Default: traitlets.Undefined
Equivalent to: [--AdvancedComponent.infile]
--outfile=<Path>
output file name
Default: traitlets.Undefined
Equivalent to: [--AdvancedComponent.outfile]
--iterations=<Long>
Number of times to run
Default: 5
Equivalent to: [--MyTool.iterations]
To see all available configurables, use `--help-all`.
The following is equivalent to the user typing mytool --help-all
193 tool.print_help(classes=True)
do some things and stuff
Options
=======
The options below are convenience aliases to configurable class-options,
as listed in the "Equivalent to" description-line of the aliases.
To see all configurable class-options for some <cmd>, use:
<cmd> --help-all
-q, --quiet
Disable console logging.
Equivalent to: [--Tool.quiet=True]
-v, --verbose
Set log level to DEBUG
Equivalent to: [--Tool.log_level=DEBUG]
--overwrite
Overwrite existing output files without asking
Equivalent to: [--Tool.overwrite=True]
--debug
Set log-level to debug, for the most verbose logging.
Equivalent to: [--Application.log_level=10]
--show-config
Show the application's configuration (human-readable format)
Equivalent to: [--Application.show_config=True]
--show-config-json
Show the application's configuration (json format)
Equivalent to: [--Application.show_config_json=True]
-c, --config=<list-item-1>...
Default: []
Equivalent to: [--Tool.config_files]
--log-level=<Enum>
Set the log level by value or name.
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 30
Equivalent to: [--Tool.log_level]
-l, --log-file=<Path>
Filename for the log
Default: None
Equivalent to: [--Tool.log_file]
--log-file-level=<Enum>
Logging Level for File Logging
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 'INFO'
Equivalent to: [--Tool.log_file_level]
--provenance-log=<Path>
Default: traitlets.Undefined
Equivalent to: [--Tool.provenance_log]
--infile=<Path>
input file name
Default: traitlets.Undefined
Equivalent to: [--AdvancedComponent.infile]
--outfile=<Path>
output file name
Default: traitlets.Undefined
Equivalent to: [--AdvancedComponent.outfile]
--iterations=<Long>
Number of times to run
Default: 5
Equivalent to: [--MyTool.iterations]
Class options
=============
The command-line option below sets the respective configurable class-parameter:
--Class.parameter=value
This line is evaluated in Python, so simple expressions are allowed.
For instance, to set `C.a=[0,1,2]`, you may type this:
--C.a='range(3)'
Application(SingletonConfigurable) options
------------------------------------------
--Application.log_datefmt=<Unicode>
The date format used by logging formatters for %(asctime)s
Default: '%Y-%m-%d %H:%M:%S'
--Application.log_format=<Unicode>
The Logging format template
Default: '[%(name)s]%(highlevel)s %(message)s'
--Application.log_level=<Enum>
Set the log level by value or name.
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 30
--Application.logging_config=<key-1>=<value-1>...
Configure additional log handlers.
The default stderr logs handler is configured by the log_level, log_datefmt
and log_format settings.
This configuration can be used to configure additional handlers (e.g. to
output the log to a file) or for finer control over the default handlers.
If provided this should be a logging configuration dictionary, for more
information see:
https://docs.python.org/3/library/logging.config.html#logging-config-
dictschema
This dictionary is merged with the base logging configuration which defines
the following:
* A logging formatter intended for interactive use called
``console``.
* A logging handler that writes to stderr called
``console`` which uses the formatter ``console``.
* A logger with the name of this application set to ``DEBUG``
level.
This example adds a new handler that writes to a file:
.. code-block:: python
c.Application.logging_config = {
"handlers": {
"file": {
"class": "logging.FileHandler",
"level": "DEBUG",
"filename": "<path/to/file>",
}
},
"loggers": {
"<application-name>": {
"level": "DEBUG",
# NOTE: if you don't list the default "console"
# handler here then it will be disabled
"handlers": ["console", "file"],
},
},
}
Default: {}
--Application.show_config=<Bool>
Instead of starting the Application, dump configuration to stdout
Default: False
--Application.show_config_json=<Bool>
Instead of starting the Application, dump configuration to stdout (as JSON)
Default: False
Tool(Application) options
-------------------------
--Tool.config_files=<list-item-1>...
Default: []
--Tool.log_config=<key-1>=<value-1>...
Default: {}
--Tool.log_datefmt=<Unicode>
The date format used by logging formatters for %(asctime)s
Default: '%Y-%m-%d %H:%M:%S'
--Tool.log_file=<Path>
Filename for the log
Default: None
--Tool.log_file_level=<Enum>
Logging Level for File Logging
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 'INFO'
--Tool.log_format=<Unicode>
The Logging format template
Default: '[%(name)s]%(highlevel)s %(message)s'
--Tool.log_level=<Enum>
Set the log level by value or name.
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 30
--Tool.logging_config=<key-1>=<value-1>...
Configure additional log handlers.
The default stderr logs handler is configured by the log_level, log_datefmt
and log_format settings.
This configuration can be used to configure additional handlers (e.g. to
output the log to a file) or for finer control over the default handlers.
If provided this should be a logging configuration dictionary, for more
information see:
https://docs.python.org/3/library/logging.config.html#logging-config-
dictschema
This dictionary is merged with the base logging configuration which defines
the following:
* A logging formatter intended for interactive use called
``console``.
* A logging handler that writes to stderr called
``console`` which uses the formatter ``console``.
* A logger with the name of this application set to ``DEBUG``
level.
This example adds a new handler that writes to a file:
.. code-block:: python
c.Application.logging_config = {
"handlers": {
"file": {
"class": "logging.FileHandler",
"level": "DEBUG",
"filename": "<path/to/file>",
}
},
"loggers": {
"<application-name>": {
"level": "DEBUG",
# NOTE: if you don't list the default "console"
# handler here then it will be disabled
"handlers": ["console", "file"],
},
},
}
Default: {}
--Tool.overwrite=<Bool>
Default: False
--Tool.provenance_log=<Path>
Default: traitlets.Undefined
--Tool.quiet=<Bool>
Default: False
--Tool.show_config=<Bool>
Instead of starting the Application, dump configuration to stdout
Default: False
--Tool.show_config_json=<Bool>
Instead of starting the Application, dump configuration to stdout (as JSON)
Default: False
MyTool(Tool) options
--------------------
--MyTool.config_files=<list-item-1>...
Default: []
--MyTool.iterations=<Long>
Number of times to run
Default: 5
--MyTool.log_config=<key-1>=<value-1>...
Default: {}
--MyTool.log_datefmt=<Unicode>
The date format used by logging formatters for %(asctime)s
Default: '%Y-%m-%d %H:%M:%S'
--MyTool.log_file=<Path>
Filename for the log
Default: None
--MyTool.log_file_level=<Enum>
Logging Level for File Logging
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 'INFO'
--MyTool.log_format=<Unicode>
The Logging format template
Default: '[%(name)s]%(highlevel)s %(message)s'
--MyTool.log_level=<Enum>
Set the log level by value or name.
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 30
--MyTool.logging_config=<key-1>=<value-1>...
Configure additional log handlers.
The default stderr logs handler is configured by the log_level, log_datefmt
and log_format settings.
This configuration can be used to configure additional handlers (e.g. to
output the log to a file) or for finer control over the default handlers.
If provided this should be a logging configuration dictionary, for more
information see:
https://docs.python.org/3/library/logging.config.html#logging-config-
dictschema
This dictionary is merged with the base logging configuration which defines
the following:
* A logging formatter intended for interactive use called
``console``.
* A logging handler that writes to stderr called
``console`` which uses the formatter ``console``.
* A logger with the name of this application set to ``DEBUG``
level.
This example adds a new handler that writes to a file:
.. code-block:: python
c.Application.logging_config = {
"handlers": {
"file": {
"class": "logging.FileHandler",
"level": "DEBUG",
"filename": "<path/to/file>",
}
},
"loggers": {
"<application-name>": {
"level": "DEBUG",
# NOTE: if you don't list the default "console"
# handler here then it will be disabled
"handlers": ["console", "file"],
},
},
}
Default: {}
--MyTool.overwrite=<Bool>
Default: False
--MyTool.provenance_log=<Path>
Default: traitlets.Undefined
--MyTool.quiet=<Bool>
Default: False
--MyTool.show_config=<Bool>
Instead of starting the Application, dump configuration to stdout
Default: False
--MyTool.show_config_json=<Bool>
Instead of starting the Application, dump configuration to stdout (as JSON)
Default: False
MyComponent(Component) options
------------------------------
--MyComponent.value=<Long>
Value to use
Default: -1
AdvancedComponent(Component) options
------------------------------------
--AdvancedComponent.infile=<Path>
input file name
Default: traitlets.Undefined
--AdvancedComponent.outfile=<Path>
output file name
Default: traitlets.Undefined
--AdvancedComponent.value1=<Long>
Value to use
Default: -1
SecondaryMyComponent(MyComponent) options
-----------------------------------------
--SecondaryMyComponent.value=<Long>
Value to use
Default: -1
TelescopeWiseComponent(TelescopeComponent) options
--------------------------------------------------
--TelescopeWiseComponent.param=<floattelescopeparameter-item-1>...
Something configurable with telescope patterns.
Default: [('type', '*', 5.0)]
Run the tool#
here we pass in argv since it is a Notebook, but if argv is not
specified it’s read from sys.argv, so the following is the same as
running:
mytool –log_level=INFO –infile gamma_test.simtel.gz –iterations=3
As Tools are intended to be executables, they are raising
SystemExit on exit. Here, we use them to demonstrate how it would
work, so we catch the SystemExit.
212 try:
213 tool.run(argv=["--infile", str(GAMMA_FILE), "--outfile", "out.csv"])
214 except SystemExit as e:
215 assert e.code == 0, f"Tool returned with error status {e}"
216
217 tool.log_format = "%(asctime)s : %(levelname)s [%(name)s %(funcName)s] %(message)s"
2024-05-15 13:13:03,654 WARNING [__main__.mytool.AdvancedComponent] (command_line_tools.on_outfile_changed): Outfile was changed to '{'name': 'outfile', 'old': traitlets.Undefined, 'new': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/stable/examples/core/out.csv'), 'owner': <__main__.AdvancedComponent object at 0x7f746b4e8190>, 'type': 'change'}'
2024-05-15 13:13:04,155 WARNING [__main__.mytool] (command_line_tools.finish): Shutting down.
221 try:
222 tool.run(
223 argv=[
224 "--log-level",
225 "INFO",
226 "--infile",
227 str(GAMMA_FILE),
228 "--outfile",
229 "out.csv",
230 "--iterations",
231 "3",
232 ]
233 )
234 except SystemExit as e:
235 assert e.code == 0, f"Tool returned with error status {e}"
2024-05-15 13:13:04,388 INFO [__main__.mytool] (tool.initialize): Loading config from '[]'
2024-05-15 13:13:04,390 INFO [__main__.mytool] (tool.initialize): ctapipe version 0.21.1
2024-05-15 13:13:04,393 WARNING [__main__.mytool.AdvancedComponent] (command_line_tools.on_outfile_changed): Outfile was changed to '{'name': 'outfile', 'old': traitlets.Undefined, 'new': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/stable/examples/core/out.csv'), 'owner': <__main__.AdvancedComponent object at 0x7f746b4ebc70>, 'type': 'change'}'
2024-05-15 13:13:04,393 INFO [__main__.mytool] (command_line_tools.start): Performing 3 iterations...
2024-05-15 13:13:04,393 INFO [__main__.mytool] (command_line_tools.start): ITERATION 0
2024-05-15 13:13:04,493 INFO [__main__.mytool] (command_line_tools.start): ITERATION 1
2024-05-15 13:13:04,594 INFO [__main__.mytool] (command_line_tools.start): ITERATION 2
2024-05-15 13:13:04,694 WARNING [__main__.mytool] (command_line_tools.finish): Shutting down.
2024-05-15 13:13:04,694 INFO [__main__.mytool] (tool.run): Finished: mytool
2024-05-15 13:13:04,696 INFO [__main__.mytool] (tool.write_provenance): Output:
2024-05-15 13:13:04,696 INFO [__main__.mytool] (tool.write_provenance): Output:
here we change the log-level to DEBUG:
242 try:
243 tool.run(
244 argv=[
245 "--log-level",
246 "DEBUG",
247 "--infile",
248 str(GAMMA_FILE),
249 "--outfile",
250 "out.csv",
251 ]
252 )
253 except SystemExit as e:
254 assert e.code == 0, f"Tool returned with error status {e}"
2024-05-15 13:13:04,700 INFO [__main__.mytool] (tool.run): Starting: mytool
2024-05-15 13:13:04,930 INFO [__main__.mytool] (tool.initialize): Loading config from '[]'
2024-05-15 13:13:04,932 INFO [__main__.mytool] (tool.initialize): ctapipe version 0.21.1
2024-05-15 13:13:04,934 WARNING [__main__.mytool.AdvancedComponent] (command_line_tools.on_outfile_changed): Outfile was changed to '{'name': 'outfile', 'old': traitlets.Undefined, 'new': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/stable/examples/core/out.csv'), 'owner': <__main__.AdvancedComponent object at 0x7f746b4eb340>, 'type': 'change'}'
2024-05-15 13:13:04,935 DEBUG [__main__.mytool] (tool.run): CONFIG: {'MyTool': {'config_files': [], 'iterations': 3, 'log_config': {}, 'log_datefmt': '%Y-%m-%d %H:%M:%S', 'log_file': None, 'log_file_level': 'INFO', 'log_format': '%(asctime)s : %(levelname)s [%(name)s %(funcName)s] %(message)s', 'log_level': 10, 'logging_config': {}, 'overwrite': False, 'provenance_log': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/stable/docs/mytool.provenance.log'), 'quiet': False, 'show_config': False, 'show_config_json': False, 'MyComponent': {'value': -1}, 'SecondaryMyComponent': {'value': -1}, 'TelescopeWiseComponent': {'param': [('type', '*', 5.0)]}, 'AdvancedComponent': {'infile': PosixPath('/home/docs/.cache/ctapipe/cccta-dataserver.in2p3.fr/data/ctapipe-test-data/v1.1.0/gamma_prod5.simtel.zst'), 'outfile': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/stable/examples/core/out.csv'), 'value1': -1, 'MyComponent': {'value': -1}}}}
2024-05-15 13:13:04,935 INFO [__main__.mytool] (command_line_tools.start): Performing 3 iterations...
2024-05-15 13:13:04,935 INFO [__main__.mytool] (command_line_tools.start): ITERATION 0
2024-05-15 13:13:04,935 DEBUG [__main__.mytool.MyComponent] (command_line_tools.do_thing): Did thing
2024-05-15 13:13:04,935 DEBUG [__main__.mytool.SecondaryMyComponent] (command_line_tools.do_thing): Did thing
2024-05-15 13:13:05,035 INFO [__main__.mytool] (command_line_tools.start): ITERATION 1
2024-05-15 13:13:05,036 DEBUG [__main__.mytool.MyComponent] (command_line_tools.do_thing): Did thing
2024-05-15 13:13:05,036 DEBUG [__main__.mytool.SecondaryMyComponent] (command_line_tools.do_thing): Did thing
2024-05-15 13:13:05,136 INFO [__main__.mytool] (command_line_tools.start): ITERATION 2
2024-05-15 13:13:05,136 DEBUG [__main__.mytool.MyComponent] (command_line_tools.do_thing): Did thing
2024-05-15 13:13:05,136 DEBUG [__main__.mytool.SecondaryMyComponent] (command_line_tools.do_thing): Did thing
2024-05-15 13:13:05,236 WARNING [__main__.mytool] (command_line_tools.finish): Shutting down.
2024-05-15 13:13:05,237 INFO [__main__.mytool] (tool.run): Finished: mytool
2024-05-15 13:13:05,238 INFO [__main__.mytool] (tool.write_provenance): Output:
2024-05-15 13:13:05,238 INFO [__main__.mytool] (tool.write_provenance): Output:
2024-05-15 13:13:05,238 INFO [__main__.mytool] (tool.write_provenance): Output:
2024-05-15 13:13:05,238 DEBUG [__main__.mytool] (tool.write_provenance): PROVENANCE: 'Details about provenance is found in /home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/stable/docs/mytool.provenance.log'
2024-05-15 13:13:05,243 DEBUG [__main__.mytool] (application.exit): Exiting application: mytool
you can also set parameters directly in the class, rather than using the argument/configfile parser. This is useful if you are calling the Tool from a script rather than the command-line
263 tool.iterations = 1
264 tool.log_level = 0
265
266 try:
267 tool.run(["--infile", str(GAMMA_FILE), "--outfile", "out.csv"])
268 except SystemExit as e:
269 assert e.code == 0, f"Tool returned with error status {e}"
2024-05-15 13:13:05,244 INFO [__main__.mytool] (tool.run): Starting: mytool
2024-05-15 13:13:05,476 WARNING [__main__.mytool.AdvancedComponent] (command_line_tools.on_outfile_changed): Outfile was changed to '{'name': 'outfile', 'old': traitlets.Undefined, 'new': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/stable/examples/core/out.csv'), 'owner': <__main__.AdvancedComponent object at 0x7f746c4e8f40>, 'type': 'change'}'
2024-05-15 13:13:05,577 WARNING [__main__.mytool] (command_line_tools.finish): Shutting down.
see what happens when a value is set that is not of the correct type:
276 try:
277 tool.iterations = "badval"
278 except TraitError as E:
279 print("bad value:", E)
280 except SystemExit as e:
281 assert e.code == 0, f"Tool returned with error status {e}"
bad value: The 'iterations' trait of a MyTool instance expected an int, not the str 'badval'.
Example of what happens when you change a parameter that is being “observed” in a class. It’s handler is called:
289 tool.advanced.outfile = "Another.txt"
2024-05-15 13:13:05,585 WARNING [__main__.mytool.AdvancedComponent] (command_line_tools.on_outfile_changed): Outfile was changed to '{'name': 'outfile', 'old': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/stable/examples/core/out.csv'), 'new': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/stable/examples/core/Another.txt'), 'owner': <__main__.AdvancedComponent object at 0x7f746c4e8f40>, 'type': 'change'}'
we see that the handler for outfile was called, and it receive a
change dict that shows the old and new values.
create a tool using a config file:
302 tool2 = MyTool()
305 try:
306 tool2.run(argv=["--config", "config.json"])
307 except SystemExit as e:
308 assert e.code == 0, f"Tool returned with error status {e}"
2024-05-15 13:13:05,817 INFO [__main__.mytool] (tool.initialize): ctapipe version 0.21.1
2024-05-15 13:13:05,819 WARNING [__main__.mytool.AdvancedComponent] (command_line_tools.on_outfile_changed): Outfile was changed to '{'name': 'outfile', 'old': traitlets.Undefined, 'new': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/stable/examples/core/foo.txt'), 'owner': <__main__.AdvancedComponent object at 0x7f746c4eb910>, 'type': 'change'}'
2024-05-15 13:13:05,820 DEBUG [__main__.mytool] (tool.run): CONFIG: {'MyTool': {'config_files': [PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/stable/examples/core/config.json')], 'iterations': 5, 'log_config': {}, 'log_datefmt': '%Y-%m-%d %H:%M:%S', 'log_file': None, 'log_file_level': 'INFO', 'log_format': '[%(name)s]%(highlevel)s %(message)s', 'log_level': 10, 'logging_config': {}, 'overwrite': False, 'provenance_log': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/stable/examples/core/mytool.provenance.log'), 'quiet': False, 'show_config': False, 'show_config_json': False, 'MyComponent': {'value': -1}, 'SecondaryMyComponent': {'value': -1}, 'TelescopeWiseComponent': {'param': [('type', '*', 5.0)]}, 'AdvancedComponent': {'infile': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/stable/examples/core/something.txt'), 'outfile': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/stable/examples/core/foo.txt'), 'value1': -1, 'MyComponent': {'value': -1}}}}
2024-05-15 13:13:05,820 INFO [__main__.mytool] (command_line_tools.start): Performing 5 iterations...
2024-05-15 13:13:05,820 INFO [__main__.mytool] (command_line_tools.start): ITERATION 0
2024-05-15 13:13:05,820 DEBUG [__main__.mytool.MyComponent] (command_line_tools.do_thing): Did thing
2024-05-15 13:13:05,820 DEBUG [__main__.mytool.SecondaryMyComponent] (command_line_tools.do_thing): Did thing
2024-05-15 13:13:05,920 INFO [__main__.mytool] (command_line_tools.start): ITERATION 1
2024-05-15 13:13:05,920 DEBUG [__main__.mytool.MyComponent] (command_line_tools.do_thing): Did thing
2024-05-15 13:13:05,920 DEBUG [__main__.mytool.SecondaryMyComponent] (command_line_tools.do_thing): Did thing
2024-05-15 13:13:06,021 INFO [__main__.mytool] (command_line_tools.start): ITERATION 2
2024-05-15 13:13:06,021 DEBUG [__main__.mytool.MyComponent] (command_line_tools.do_thing): Did thing
2024-05-15 13:13:06,021 DEBUG [__main__.mytool.SecondaryMyComponent] (command_line_tools.do_thing): Did thing
2024-05-15 13:13:06,121 INFO [__main__.mytool] (command_line_tools.start): ITERATION 3
2024-05-15 13:13:06,121 DEBUG [__main__.mytool.MyComponent] (command_line_tools.do_thing): Did thing
2024-05-15 13:13:06,121 DEBUG [__main__.mytool.SecondaryMyComponent] (command_line_tools.do_thing): Did thing
2024-05-15 13:13:06,222 INFO [__main__.mytool] (command_line_tools.start): ITERATION 4
2024-05-15 13:13:06,222 DEBUG [__main__.mytool.MyComponent] (command_line_tools.do_thing): Did thing
2024-05-15 13:13:06,222 DEBUG [__main__.mytool.SecondaryMyComponent] (command_line_tools.do_thing): Did thing
2024-05-15 13:13:06,322 WARNING [__main__.mytool] (command_line_tools.finish): Shutting down.
2024-05-15 13:13:06,322 INFO [__main__.mytool] (tool.run): Finished: mytool
2024-05-15 13:13:06,324 INFO [__main__.mytool] (tool.write_provenance): Output:
2024-05-15 13:13:06,324 INFO [__main__.mytool] (tool.write_provenance): Output:
2024-05-15 13:13:06,324 INFO [__main__.mytool] (tool.write_provenance): Output:
2024-05-15 13:13:06,324 INFO [__main__.mytool] (tool.write_provenance): Output:
2024-05-15 13:13:06,324 INFO [__main__.mytool] (tool.write_provenance): Output:
2024-05-15 13:13:06,324 DEBUG [__main__.mytool] (tool.write_provenance): PROVENANCE: 'Details about provenance is found in /home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/stable/examples/core/mytool.provenance.log'
2024-05-15 13:13:06,331 DEBUG [__main__.mytool] (application.exit): Exiting application: mytool
311 print(tool2.advanced.infile)
/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/stable/examples/core/something.txt
314 print(tool2.config)
{'MyTool': {'config_files': ['config.json'], 'log_level': 'DEBUG'}, 'AdvancedComponent': {'infile': 'something.txt', 'outfile': 'foo.txt'}}
317 tool2.is_setup
True
320 tool3 = MyTool()
323 tool3.is_setup
False
326 tool3.initialize(argv=[])
329 tool3.is_setup
False
332 tool3
335 tool.setup()
336 tool
2024-05-15 13:13:06,345 WARNING [__main__.mytool.AdvancedComponent] (command_line_tools.on_outfile_changed): Outfile was changed to '{'name': 'outfile', 'old': traitlets.Undefined, 'new': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/stable/examples/core/out.csv'), 'owner': <__main__.AdvancedComponent object at 0x7f746c4ea020>, 'type': 'change'}'
339 tool.comp2
Getting the configuration of an instance#
347 tool.get_current_config()
{'MyTool': {'config_files': [], 'iterations': 1, 'log_config': {}, 'log_datefmt': '%Y-%m-%d %H:%M:%S', 'log_file': None, 'log_file_level': 'INFO', 'log_format': '%(asctime)s : %(levelname)s [%(name)s %(funcName)s] %(message)s', 'log_level': 0, 'logging_config': {}, 'overwrite': False, 'provenance_log': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/stable/docs/mytool.provenance.log'), 'quiet': False, 'show_config': False, 'show_config_json': False, 'MyComponent': {'value': -1}, 'SecondaryMyComponent': {'value': -1}, 'TelescopeWiseComponent': {'param': [('type', '*', 5.0)]}, 'AdvancedComponent': {'infile': PosixPath('/home/docs/.cache/ctapipe/cccta-dataserver.in2p3.fr/data/ctapipe-test-data/v1.1.0/gamma_prod5.simtel.zst'), 'outfile': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/stable/examples/core/out.csv'), 'value1': -1, 'MyComponent': {'value': -1}}}}
350 tool.iterations = 12
351 tool.get_current_config()
{'MyTool': {'config_files': [], 'iterations': 12, 'log_config': {}, 'log_datefmt': '%Y-%m-%d %H:%M:%S', 'log_file': None, 'log_file_level': 'INFO', 'log_format': '%(asctime)s : %(levelname)s [%(name)s %(funcName)s] %(message)s', 'log_level': 0, 'logging_config': {}, 'overwrite': False, 'provenance_log': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/stable/docs/mytool.provenance.log'), 'quiet': False, 'show_config': False, 'show_config_json': False, 'MyComponent': {'value': -1}, 'SecondaryMyComponent': {'value': -1}, 'TelescopeWiseComponent': {'param': [('type', '*', 5.0)]}, 'AdvancedComponent': {'infile': PosixPath('/home/docs/.cache/ctapipe/cccta-dataserver.in2p3.fr/data/ctapipe-test-data/v1.1.0/gamma_prod5.simtel.zst'), 'outfile': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/stable/examples/core/out.csv'), 'value1': -1, 'MyComponent': {'value': -1}}}}
Writing a Sample Config File#
359 print(tool.generate_config_file())
# Configuration file for mytool.
c = get_config() #noqa
#------------------------------------------------------------------------------
# Application(SingletonConfigurable) configuration
#------------------------------------------------------------------------------
## This is an application.
## The date format used by logging formatters for %(asctime)s
# Default: '%Y-%m-%d %H:%M:%S'
# c.Application.log_datefmt = '%Y-%m-%d %H:%M:%S'
## The Logging format template
# Default: '[%(name)s]%(highlevel)s %(message)s'
# c.Application.log_format = '[%(name)s]%(highlevel)s %(message)s'
## Set the log level by value or name.
# Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
# Default: 30
# c.Application.log_level = 30
## Configure additional log handlers.
#
# The default stderr logs handler is configured by the log_level, log_datefmt
# and log_format settings.
#
# This configuration can be used to configure additional handlers (e.g. to
# output the log to a file) or for finer control over the default handlers.
#
# If provided this should be a logging configuration dictionary, for more
# information see:
# https://docs.python.org/3/library/logging.config.html#logging-config-
# dictschema
#
# This dictionary is merged with the base logging configuration which defines
# the following:
#
# * A logging formatter intended for interactive use called
# ``console``.
# * A logging handler that writes to stderr called
# ``console`` which uses the formatter ``console``.
# * A logger with the name of this application set to ``DEBUG``
# level.
#
# This example adds a new handler that writes to a file:
#
# .. code-block:: python
#
# c.Application.logging_config = {
# "handlers": {
# "file": {
# "class": "logging.FileHandler",
# "level": "DEBUG",
# "filename": "<path/to/file>",
# }
# },
# "loggers": {
# "<application-name>": {
# "level": "DEBUG",
# # NOTE: if you don't list the default "console"
# # handler here then it will be disabled
# "handlers": ["console", "file"],
# },
# },
# }
# Default: {}
# c.Application.logging_config = {}
## Instead of starting the Application, dump configuration to stdout
# Default: False
# c.Application.show_config = False
## Instead of starting the Application, dump configuration to stdout (as JSON)
# Default: False
# c.Application.show_config_json = False
#------------------------------------------------------------------------------
# Tool(Application) configuration
#------------------------------------------------------------------------------
## This is an application.
# Default: []
# c.Tool.config_files = []
# Default: {}
# c.Tool.log_config = {}
## The date format used by logging formatters for %(asctime)s
# See also: Application.log_datefmt
# c.Tool.log_datefmt = '%Y-%m-%d %H:%M:%S'
## Filename for the log
# Default: None
# c.Tool.log_file = None
## Logging Level for File Logging
# Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
# Default: 'INFO'
# c.Tool.log_file_level = 'INFO'
## The Logging format template
# See also: Application.log_format
# c.Tool.log_format = '[%(name)s]%(highlevel)s %(message)s'
## Set the log level by value or name.
# See also: Application.log_level
# c.Tool.log_level = 30
##
# See also: Application.logging_config
# c.Tool.logging_config = {}
# Default: False
# c.Tool.overwrite = False
# Default: traitlets.Undefined
# c.Tool.provenance_log = traitlets.Undefined
# Default: False
# c.Tool.quiet = False
## Instead of starting the Application, dump configuration to stdout
# See also: Application.show_config
# c.Tool.show_config = False
## Instead of starting the Application, dump configuration to stdout (as JSON)
# See also: Application.show_config_json
# c.Tool.show_config_json = False
#------------------------------------------------------------------------------
# MyTool(Tool) configuration
#------------------------------------------------------------------------------
# See also: Tool.config_files
# c.MyTool.config_files = []
## Number of times to run
# Default: 5
# c.MyTool.iterations = 5
# See also: Tool.log_config
# c.MyTool.log_config = {}
## The date format used by logging formatters for %(asctime)s
# See also: Application.log_datefmt
# c.MyTool.log_datefmt = '%Y-%m-%d %H:%M:%S'
## Filename for the log
# See also: Tool.log_file
# c.MyTool.log_file = None
## Logging Level for File Logging
# See also: Tool.log_file_level
# c.MyTool.log_file_level = 'INFO'
## The Logging format template
# See also: Application.log_format
# c.MyTool.log_format = '[%(name)s]%(highlevel)s %(message)s'
## Set the log level by value or name.
# See also: Application.log_level
# c.MyTool.log_level = 30
##
# See also: Application.logging_config
# c.MyTool.logging_config = {}
# See also: Tool.overwrite
# c.MyTool.overwrite = False
# See also: Tool.provenance_log
# c.MyTool.provenance_log = traitlets.Undefined
# See also: Tool.quiet
# c.MyTool.quiet = False
## Instead of starting the Application, dump configuration to stdout
# See also: Application.show_config
# c.MyTool.show_config = False
## Instead of starting the Application, dump configuration to stdout (as JSON)
# See also: Application.show_config_json
# c.MyTool.show_config_json = False
#------------------------------------------------------------------------------
# MyComponent(Component) configuration
#------------------------------------------------------------------------------
## A Component that does stuff
## Value to use
# Default: -1
# c.MyComponent.value = -1
#------------------------------------------------------------------------------
# AdvancedComponent(Component) configuration
#------------------------------------------------------------------------------
## An advanced technique
## input file name
# Default: traitlets.Undefined
# c.AdvancedComponent.infile = traitlets.Undefined
## output file name
# Default: traitlets.Undefined
# c.AdvancedComponent.outfile = traitlets.Undefined
## Value to use
# Default: -1
# c.AdvancedComponent.value1 = -1
#------------------------------------------------------------------------------
# SecondaryMyComponent(MyComponent) configuration
#------------------------------------------------------------------------------
## A second component
## Value to use
# See also: MyComponent.value
# c.SecondaryMyComponent.value = -1
#------------------------------------------------------------------------------
# TelescopeWiseComponent(TelescopeComponent) configuration
#------------------------------------------------------------------------------
## a component that contains parameters that are per-telescope configurable
## Something configurable with telescope patterns.
# Default: [('type', '*', 5.0)]
# c.TelescopeWiseComponent.param = [('type', '*', 5.0)]
Total running time of the script: (0 minutes 4.241 seconds)