Creating command-line Tools¶
[1]:
from ctapipe.core import Tool, Component, TelescopeComponent
from ctapipe.core.traits import (
Integer, Float, List, Dict, Unicode,
TraitError, observe, FloatTelescopeParameter,
Path,
)
import logging
from time import sleep
from astropy import units as u
from ctapipe.utils import get_dataset_path
[2]:
GAMMA_FILE = get_dataset_path('gamma_test.simtel.gz')
see https://github.com/ipython/traitlets/blob/master/examples/myapp.py
Setup:¶
Create a few Component
s that we will use later in a Tool
:
[3]:
class MyComponent(Component):
""" A Component that does stuff """
value = Integer(default_value=-1, help="Value to use").tag(config=True)
def do_thing(self):
self.log.debug("Did thing")
# in order to have 2 of the same components at once
class SecondaryMyComponent(MyComponent):
pass
class AdvancedComponent(Component):
""" An advanced technique """
value1 = Integer(default_value=-1, help="Value to use").tag(config=True)
infile = Path(
help="input file name",
exists=None, # set to True to require existing, False for requiring non-existing
directory_ok=False,
).tag(config=True)
outfile = Path(
help="output file name",
exists=False, directory_ok=False
).tag(config=True)
def __init__(self, config=None, parent=None, **kwargs):
super().__init__(config=config, parent=parent, **kwargs)
# components can have sub components, but these must have
# then parent=self as argument and be assigned as member
# so the full config can be received later
self.subcompent = MyComponent(parent=self)
@observe("outfile")
def on_outfile_changed(self, change):
self.log.warning("Outfile was changed to '{}'".format(change))
class TelescopeWiseComponent(TelescopeComponent):
""" a component that contains parameters that are per-telescope configurable """
param = FloatTelescopeParameter(
help="Something configurable with telescope patterns", default_value=5.0
).tag(config=True)
[4]:
MyComponent()
[4]:
A Component that does stuff
value | -1 | Value to use (default: -1) |
---|
[5]:
AdvancedComponent(infile="test.foo", outfile="out.foo")
Outfile was changed to '{'name': 'outfile', 'old': traitlets.Undefined, 'new': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.15.0/docs/examples/out.foo'), 'owner': <__main__.AdvancedComponent object at 0x7fd0f5e7d400>, 'type': 'change'}'
[5]:
An advanced technique
infile | /home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.15.0/docs/examples/test.foo | input file name (default: traitlets.Undefined) |
---|---|---|
outfile | /home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.15.0/docs/examples/out.foo | output file name (default: traitlets.Undefined) |
value1 | -1 | Value to use (default: -1) |
TelescopeComponents
need to have a subarray given to them in order to work (since they need one to turn a TelescopeParameter
into a concrete list of values for each telescope. Here we will give a dummy one:
[6]:
from ctapipe.instrument import SubarrayDescription, TelescopeDescription
subarray = SubarrayDescription(
"Junk",
tel_positions={1: (0.0, 0.0, 0.0) * u.m, 2: (1.0, 1.0, 0.0) * u.m},
tel_descriptions={
1: TelescopeDescription.from_name("LST", "LSTCam"),
2: TelescopeDescription.from_name("MST", "NectarCam"),
},
)
subarray.info()
Subarray : Junk
Num Tels : 2
Footprint: 0.00 km2
Type Count Tel IDs
----------------- ----- -------
MST_MST_NectarCam 1 2
LST_LST_LSTCam 1 1
[7]:
TelescopeWiseComponent(subarray=subarray)
[7]:
a component that contains parameters that are per-telescope configurable
param | [('type', '*', 5.0)] | Something configurable with telescope patterns (default: traitlets.Undefined) |
---|
This TelescopeParameters can then be set using a list of patterns like:
component.param = [
("type", "LST*",3.0),
("type", "MST*", 2.0),
(id, 25, 4.0)
]
These get translated into per-telescope-id values once the subarray is registered. After that one acccess the per-telescope id values via:
component.param.tel[tel_id]
Now create an executable Tool that contains the Components¶
[8]:
class MyTool(Tool):
name="mytool"
description="do some things and stuff"
aliases = dict(
infile='AdvancedComponent.infile',
outfile='AdvancedComponent.outfile',
iterations='MyTool.iterations'
)
# Which classes are registered for configuration
classes = [MyComponent, AdvancedComponent, SecondaryMyComponent, TelescopeWiseComponent]
# local configuration parameters
iterations = Integer(5,help="Number of times to run",allow_none=False).tag(config=True)
def setup(self):
self.comp = MyComponent(parent=self)
self.comp2 = SecondaryMyComponent(parent=self)
self.comp3 = TelescopeWiseComponent(parent=self, subarray=subarray)
self.advanced = AdvancedComponent(parent=self)
def start(self):
self.log.info("Performing {} iterations...".format(self.iterations))
for ii in range(self.iterations):
self.log.info("ITERATION {}".format(ii))
self.comp.do_thing()
self.comp2.do_thing()
sleep(0.1)
def finish(self):
self.log.warning("Shutting down.")
Get Help info¶
The following allows you to print the help info within a Jupyter notebook, but this same inforamtion would be displayed if the user types:
mytool --help
[9]:
tool = MyTool()
tool
[9]:
do some things and stuff
config_files | [] | (default: traitlets.Undefined) |
---|---|---|
iterations | 5 | Number of times to run (default: 5) |
log_config | {'version': 1, 'disable_existing_loggers': False, 'formatters': {'file': {'()': |
(default: traitlets.Undefined) |
log_datefmt | %Y-%m-%d %H:%M:%S | The date format used by logging formatters for %(asctime)s (default: %Y-%m-%d %H:%M:%S) |
log_file | None | Filename for the log (default: None) |
log_file_level | INFO | Logging Level for File Logging (default: INFO) |
log_format | [%(name)s]%(highlevel)s %(message)s | The Logging format template (default: [%(name)s]%(highlevel)s %(message)s) |
log_level | 30 | Set the log level by value or name. (default: 30) |
logging_config | {} |
Configure additional log handlers.
The default stderr logs handler is configured by the
log_level, log_datefmt and log_format settings.
This configuration can be used to configure additional handlers
(e.g. to output the log to a file) or for finer control over the
default handlers.
If provided this should be a logging configuration dictionary, for
more information see:
https://docs.python.org/3/library/logging.config.html#logging-config-dictschema
This dictionary is merged with the base logging configuration which
defines the following:
* A logging formatter intended for interactive use called
``console``.
* A logging handler that writes to stderr called
``console`` which uses the formatter ``console``.
* A logger with the name of this application set to ``DEBUG``
level.
This example adds a new handler that writes to a file:
.. code-block:: python
c.Application.logging_config = {
'handlers': {
'file': {
'class': 'logging.FileHandler',
'level': 'DEBUG',
'filename': ' |
provenance_log | /home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.15.0/docs/examples/mytool.provenance.log | (default: traitlets.Undefined) |
quiet | False | (default: False) |
show_config | False | Instead of starting the Application, dump configuration to stdout (default: False) |
show_config_json | False | Instead of starting the Application, dump configuration to stdout (as JSON) (default: False) |
Components: MyTool, MyComponent, AdvancedComponent, SecondaryMyComponent, TelescopeWiseComponent
[10]:
tool.print_help()
do some things and stuff
Options
=======
The options below are convenience aliases to configurable class-options,
as listed in the "Equivalent to" description-line of the aliases.
To see all configurable class-options for some <cmd>, use:
<cmd> --help-all
--debug
Set log-level to debug, for the most verbose logging.
Equivalent to: [--Application.log_level=10]
--show-config
Show the application's configuration (human-readable format)
Equivalent to: [--Application.show_config=True]
--show-config-json
Show the application's configuration (json format)
Equivalent to: [--Application.show_config_json=True]
-q, --quiet
Disable console logging.
Equivalent to: [--Tool.quiet=True]
-v, --verbose
Set log level to DEBUG
Equivalent to: [--Tool.log_level=DEBUG]
-c, --config=<list-item-1>...
Default: []
Equivalent to: [--Tool.config_files]
--log-level=<Enum>
Set the log level by value or name.
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 30
Equivalent to: [--Tool.log_level]
-l, --log-file=<Path>
Filename for the log
Default: None
Equivalent to: [--Tool.log_file]
--log-file-level=<Enum>
Logging Level for File Logging
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 'INFO'
Equivalent to: [--Tool.log_file_level]
--provenance-log=<Path>
Default: traitlets.Undefined
Equivalent to: [--Tool.provenance_log]
--infile=<Path>
input file name
Default: traitlets.Undefined
Equivalent to: [--AdvancedComponent.infile]
--outfile=<Path>
output file name
Default: traitlets.Undefined
Equivalent to: [--AdvancedComponent.outfile]
--iterations=<Int>
Number of times to run
Default: 5
Equivalent to: [--MyTool.iterations]
To see all available configurables, use `--help-all`.
The following is equivalant to the user typing mytool --help-all
[11]:
tool.print_help(classes=True)
do some things and stuff
Options
=======
The options below are convenience aliases to configurable class-options,
as listed in the "Equivalent to" description-line of the aliases.
To see all configurable class-options for some <cmd>, use:
<cmd> --help-all
--debug
Set log-level to debug, for the most verbose logging.
Equivalent to: [--Application.log_level=10]
--show-config
Show the application's configuration (human-readable format)
Equivalent to: [--Application.show_config=True]
--show-config-json
Show the application's configuration (json format)
Equivalent to: [--Application.show_config_json=True]
-q, --quiet
Disable console logging.
Equivalent to: [--Tool.quiet=True]
-v, --verbose
Set log level to DEBUG
Equivalent to: [--Tool.log_level=DEBUG]
-c, --config=<list-item-1>...
Default: []
Equivalent to: [--Tool.config_files]
--log-level=<Enum>
Set the log level by value or name.
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 30
Equivalent to: [--Tool.log_level]
-l, --log-file=<Path>
Filename for the log
Default: None
Equivalent to: [--Tool.log_file]
--log-file-level=<Enum>
Logging Level for File Logging
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 'INFO'
Equivalent to: [--Tool.log_file_level]
--provenance-log=<Path>
Default: traitlets.Undefined
Equivalent to: [--Tool.provenance_log]
--infile=<Path>
input file name
Default: traitlets.Undefined
Equivalent to: [--AdvancedComponent.infile]
--outfile=<Path>
output file name
Default: traitlets.Undefined
Equivalent to: [--AdvancedComponent.outfile]
--iterations=<Int>
Number of times to run
Default: 5
Equivalent to: [--MyTool.iterations]
Class options
=============
The command-line option below sets the respective configurable class-parameter:
--Class.parameter=value
This line is evaluated in Python, so simple expressions are allowed.
For instance, to set `C.a=[0,1,2]`, you may type this:
--C.a='range(3)'
Application(SingletonConfigurable) options
------------------------------------------
--Application.log_datefmt=<Unicode>
The date format used by logging formatters for %(asctime)s
Default: '%Y-%m-%d %H:%M:%S'
--Application.log_format=<Unicode>
The Logging format template
Default: '[%(name)s]%(highlevel)s %(message)s'
--Application.log_level=<Enum>
Set the log level by value or name.
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 30
--Application.logging_config=<key-1>=<value-1>...
Configure additional log handlers.
The default stderr logs handler is configured by the log_level, log_datefmt
and log_format settings.
This configuration can be used to configure additional handlers (e.g. to
output the log to a file) or for finer control over the default handlers.
If provided this should be a logging configuration dictionary, for more
information see:
https://docs.python.org/3/library/logging.config.html#logging-config-
dictschema
This dictionary is merged with the base logging configuration which defines
the following:
* A logging formatter intended for interactive use called
``console``.
* A logging handler that writes to stderr called
``console`` which uses the formatter ``console``.
* A logger with the name of this application set to ``DEBUG``
level.
This example adds a new handler that writes to a file:
.. code-block:: python
c.Application.logging_config = {
'handlers': {
'file': {
'class': 'logging.FileHandler',
'level': 'DEBUG',
'filename': '<path/to/file>',
}
},
'loggers': {
'<application-name>': {
'level': 'DEBUG',
# NOTE: if you don't list the default "console"
# handler here then it will be disabled
'handlers': ['console', 'file'],
},
}
}
Default: {}
--Application.show_config=<Bool>
Instead of starting the Application, dump configuration to stdout
Default: False
--Application.show_config_json=<Bool>
Instead of starting the Application, dump configuration to stdout (as JSON)
Default: False
Tool(Application) options
-------------------------
--Tool.config_files=<list-item-1>...
Default: []
--Tool.log_config=<key-1>=<value-1>...
Default: {'version': 1, 'disable_existing_loggers': False, 'formatters...
--Tool.log_datefmt=<Unicode>
The date format used by logging formatters for %(asctime)s
Default: '%Y-%m-%d %H:%M:%S'
--Tool.log_file=<Path>
Filename for the log
Default: None
--Tool.log_file_level=<Enum>
Logging Level for File Logging
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 'INFO'
--Tool.log_format=<Unicode>
The Logging format template
Default: '[%(name)s]%(highlevel)s %(message)s'
--Tool.log_level=<Enum>
Set the log level by value or name.
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 30
--Tool.logging_config=<key-1>=<value-1>...
Configure additional log handlers.
The default stderr logs handler is configured by the log_level, log_datefmt
and log_format settings.
This configuration can be used to configure additional handlers (e.g. to
output the log to a file) or for finer control over the default handlers.
If provided this should be a logging configuration dictionary, for more
information see:
https://docs.python.org/3/library/logging.config.html#logging-config-
dictschema
This dictionary is merged with the base logging configuration which defines
the following:
* A logging formatter intended for interactive use called
``console``.
* A logging handler that writes to stderr called
``console`` which uses the formatter ``console``.
* A logger with the name of this application set to ``DEBUG``
level.
This example adds a new handler that writes to a file:
.. code-block:: python
c.Application.logging_config = {
'handlers': {
'file': {
'class': 'logging.FileHandler',
'level': 'DEBUG',
'filename': '<path/to/file>',
}
},
'loggers': {
'<application-name>': {
'level': 'DEBUG',
# NOTE: if you don't list the default "console"
# handler here then it will be disabled
'handlers': ['console', 'file'],
},
}
}
Default: {}
--Tool.provenance_log=<Path>
Default: traitlets.Undefined
--Tool.quiet=<Bool>
Default: False
--Tool.show_config=<Bool>
Instead of starting the Application, dump configuration to stdout
Default: False
--Tool.show_config_json=<Bool>
Instead of starting the Application, dump configuration to stdout (as JSON)
Default: False
MyTool(Tool) options
--------------------
--MyTool.config_files=<list-item-1>...
Default: []
--MyTool.iterations=<Int>
Number of times to run
Default: 5
--MyTool.log_config=<key-1>=<value-1>...
Default: {'version': 1, 'disable_existing_loggers': False, 'formatters...
--MyTool.log_datefmt=<Unicode>
The date format used by logging formatters for %(asctime)s
Default: '%Y-%m-%d %H:%M:%S'
--MyTool.log_file=<Path>
Filename for the log
Default: None
--MyTool.log_file_level=<Enum>
Logging Level for File Logging
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 'INFO'
--MyTool.log_format=<Unicode>
The Logging format template
Default: '[%(name)s]%(highlevel)s %(message)s'
--MyTool.log_level=<Enum>
Set the log level by value or name.
Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
Default: 30
--MyTool.logging_config=<key-1>=<value-1>...
Configure additional log handlers.
The default stderr logs handler is configured by the log_level, log_datefmt
and log_format settings.
This configuration can be used to configure additional handlers (e.g. to
output the log to a file) or for finer control over the default handlers.
If provided this should be a logging configuration dictionary, for more
information see:
https://docs.python.org/3/library/logging.config.html#logging-config-
dictschema
This dictionary is merged with the base logging configuration which defines
the following:
* A logging formatter intended for interactive use called
``console``.
* A logging handler that writes to stderr called
``console`` which uses the formatter ``console``.
* A logger with the name of this application set to ``DEBUG``
level.
This example adds a new handler that writes to a file:
.. code-block:: python
c.Application.logging_config = {
'handlers': {
'file': {
'class': 'logging.FileHandler',
'level': 'DEBUG',
'filename': '<path/to/file>',
}
},
'loggers': {
'<application-name>': {
'level': 'DEBUG',
# NOTE: if you don't list the default "console"
# handler here then it will be disabled
'handlers': ['console', 'file'],
},
}
}
Default: {}
--MyTool.provenance_log=<Path>
Default: traitlets.Undefined
--MyTool.quiet=<Bool>
Default: False
--MyTool.show_config=<Bool>
Instead of starting the Application, dump configuration to stdout
Default: False
--MyTool.show_config_json=<Bool>
Instead of starting the Application, dump configuration to stdout (as JSON)
Default: False
MyComponent(Component) options
------------------------------
--MyComponent.value=<Int>
Value to use
Default: -1
AdvancedComponent(Component) options
------------------------------------
--AdvancedComponent.infile=<Path>
input file name
Default: traitlets.Undefined
--AdvancedComponent.outfile=<Path>
output file name
Default: traitlets.Undefined
--AdvancedComponent.value1=<Int>
Value to use
Default: -1
SecondaryMyComponent(MyComponent) options
-----------------------------------------
--SecondaryMyComponent.value=<Int>
Value to use
Default: -1
TelescopeWiseComponent(TelescopeComponent) options
--------------------------------------------------
--TelescopeWiseComponent.param=<floattelescopeparameter-item-1>...
Something configurable with telescope patterns
Default: [('type', '*', 5.0)]
Run the tool¶
here we pass in argv since it is a Notebook, but if argv is not specified it’s read from sys.argv
, so the following is the same as running:
mytool --log_level=INFO --infile gamma_test.simtel.gz --iterations=3
As Tools are intended to be exectutables, they are raising SystemExit
on exit. Here, we use them to demonstrate how it would work, so we catch the SystemExit
.
[12]:
try:
tool.run(argv=['--infile', str(GAMMA_FILE), '--outfile', 'out.csv'])
except SystemExit as e:
assert e.code == 0, f'Tool returned with error status {e}'
Shutting down.
[13]:
tool.log_format = "%(asctime)s : %(levelname)s [%(name)s %(funcName)s] %(message)s"
try:
tool.run(argv=['--log-level','INFO','--infile', str(GAMMA_FILE), '--outfile', 'out.csv', '--iterations','3'])
except SystemExit as e:
assert e.code == 0, f'Tool returned with error status {e}'
Shutting down.
here we change the log-level to DEBUG:
[14]:
try:
tool.run(argv=['--log-level','DEBUG','--infile', str(GAMMA_FILE), '--outfile', 'out.csv'])
except SystemExit as e:
assert e.code == 0, f'Tool returned with error status {e}'
2022-10-25 09:56:05,701 DEBUG [ctapipe.core.provenance] (provenance.start_activity): started activity: mytool
2022-10-25 09:56:05,703 DEBUG [ctapipe.core.traits] (traits.attach_subarray): argument '*' matched: ['MST_MST_NectarCam', 'LST_LST_LSTCam']
Shutting down.
2022-10-25 09:56:06,010 DEBUG [ctapipe.core.provenance] (provenance.finish_activity): finished activity: mytool
you can also set parameters directly in the class, rather than using the argument/configfile parser. This is useful if you are calling the Tool from a script rather than the command-line
[15]:
tool.iterations = 1
tool.log_level = 0
try:
tool.run(['--infile', str(GAMMA_FILE), '--outfile', 'out.csv'])
except SystemExit as e:
assert e.code == 0, f'Tool returned with error status {e}'
2022-10-25 09:56:06,048 DEBUG [ctapipe.core.provenance] (provenance.start_activity): started activity: mytool
2022-10-25 09:56:06,050 DEBUG [ctapipe.core.traits] (traits.attach_subarray): argument '*' matched: ['MST_MST_NectarCam', 'LST_LST_LSTCam']
Shutting down.
2022-10-25 09:56:06,357 DEBUG [ctapipe.core.provenance] (provenance.finish_activity): finished activity: mytool
see what happens when a value is set that is not of the correct type:
[16]:
try:
tool.iterations = "badval"
except TraitError as E:
print("bad value:",E)
except SystemExit as e:
assert e.code == 0, f'Tool returned with error status {e}'
bad value: The 'iterations' trait of a MyTool instance expected an int, not the str 'badval'.
Example of what happens when you change a parameter that is being “observed” in a class. It’s handler is called:
[17]:
tool.advanced.outfile = "Another.txt"
Outfile was changed to '{'name': 'outfile', 'old': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.15.0/docs/examples/out.csv'), 'new': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.15.0/docs/examples/Another.txt'), 'owner': <__main__.AdvancedComponent object at 0x7fd0feed43a0>, 'type': 'change'}'
we see that the handler for outfile
was called, and it receive a change dict that shows the old and new values.
create a tool using a config file:
[18]:
tool2 = MyTool()
[19]:
try:
tool2.run(argv=['--config', 'Tools.json'])
except SystemExit as e:
assert e.code == 0, f'Tool returned with error status {e}'
2022-10-25 09:56:06,423 DEBUG [ctapipe.core.provenance] (provenance.start_activity): started activity: mytool
2022-10-25 09:56:06,425 DEBUG [ctapipe.core.traits] (traits.attach_subarray): argument '*' matched: ['MST_MST_NectarCam', 'LST_LST_LSTCam']
Shutting down.
2022-10-25 09:56:06,933 DEBUG [ctapipe.core.provenance] (provenance.finish_activity): finished activity: mytool
[20]:
print(tool2.advanced.infile)
/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.15.0/docs/examples/something.txt
[21]:
print(tool2.config)
{'MyTool': {'config_files': ['Tools.json'], 'log_level': 'DEBUG'}, 'AdvancedComponent': {'infile': 'something.txt', 'outfile': 'foo.txt'}, 'attach_subarray': <LazyConfigValue {}>}
[22]:
tool2.is_setup
[22]:
True
[23]:
tool3 = MyTool()
[24]:
tool3.is_setup
[24]:
False
[25]:
tool3.initialize(argv=[])
[26]:
tool3.is_setup
[26]:
False
[27]:
tool3
[27]:
do some things and stuff
config_files | [] | (default: traitlets.Undefined) |
---|---|---|
iterations | 5 | Number of times to run (default: 5) |
log_config | {'version': 1, 'disable_existing_loggers': False, 'formatters': {'file': {'()': |
(default: traitlets.Undefined) |
log_datefmt | %Y-%m-%d %H:%M:%S | The date format used by logging formatters for %(asctime)s (default: %Y-%m-%d %H:%M:%S) |
log_file | None | Filename for the log (default: None) |
log_file_level | INFO | Logging Level for File Logging (default: INFO) |
log_format | [%(name)s]%(highlevel)s %(message)s | The Logging format template (default: [%(name)s]%(highlevel)s %(message)s) |
log_level | 30 | Set the log level by value or name. (default: 30) |
logging_config | {} |
Configure additional log handlers.
The default stderr logs handler is configured by the
log_level, log_datefmt and log_format settings.
This configuration can be used to configure additional handlers
(e.g. to output the log to a file) or for finer control over the
default handlers.
If provided this should be a logging configuration dictionary, for
more information see:
https://docs.python.org/3/library/logging.config.html#logging-config-dictschema
This dictionary is merged with the base logging configuration which
defines the following:
* A logging formatter intended for interactive use called
``console``.
* A logging handler that writes to stderr called
``console`` which uses the formatter ``console``.
* A logger with the name of this application set to ``DEBUG``
level.
This example adds a new handler that writes to a file:
.. code-block:: python
c.Application.logging_config = {
'handlers': {
'file': {
'class': 'logging.FileHandler',
'level': 'DEBUG',
'filename': ' |
provenance_log | /home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.15.0/docs/examples/mytool.provenance.log | (default: traitlets.Undefined) |
quiet | False | (default: False) |
show_config | False | Instead of starting the Application, dump configuration to stdout (default: False) |
show_config_json | False | Instead of starting the Application, dump configuration to stdout (as JSON) (default: False) |
Components: MyTool, MyComponent, AdvancedComponent, SecondaryMyComponent, TelescopeWiseComponent
[28]:
tool.setup()
tool
[28]:
do some things and stuff
config_files | [] | (default: traitlets.Undefined) |
---|---|---|
iterations | 3 | Number of times to run (default: 5) |
log_config | {'version': 1, 'disable_existing_loggers': False, 'formatters': {'file': {'()': |
(default: traitlets.Undefined) |
log_datefmt | %Y-%m-%d %H:%M:%S | The date format used by logging formatters for %(asctime)s (default: %Y-%m-%d %H:%M:%S) |
log_file | None | Filename for the log (default: None) |
log_file_level | INFO | Logging Level for File Logging (default: INFO) |
log_format | %(asctime)s : %(levelname)s [%(name)s %(funcName)s] %(message)s | The Logging format template (default: [%(name)s]%(highlevel)s %(message)s) |
log_level | 10 | Set the log level by value or name. (default: 30) |
logging_config | {} |
Configure additional log handlers.
The default stderr logs handler is configured by the
log_level, log_datefmt and log_format settings.
This configuration can be used to configure additional handlers
(e.g. to output the log to a file) or for finer control over the
default handlers.
If provided this should be a logging configuration dictionary, for
more information see:
https://docs.python.org/3/library/logging.config.html#logging-config-dictschema
This dictionary is merged with the base logging configuration which
defines the following:
* A logging formatter intended for interactive use called
``console``.
* A logging handler that writes to stderr called
``console`` which uses the formatter ``console``.
* A logger with the name of this application set to ``DEBUG``
level.
This example adds a new handler that writes to a file:
.. code-block:: python
c.Application.logging_config = {
'handlers': {
'file': {
'class': 'logging.FileHandler',
'level': 'DEBUG',
'filename': ' |
provenance_log | /home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.15.0/docs/examples/mytool.provenance.log | (default: traitlets.Undefined) |
quiet | False | (default: False) |
show_config | False | Instead of starting the Application, dump configuration to stdout (default: False) |
show_config_json | False | Instead of starting the Application, dump configuration to stdout (as JSON) (default: False) |
Components: MyTool, MyComponent, AdvancedComponent, SecondaryMyComponent, TelescopeWiseComponent
[29]:
tool.comp2
[29]:
Undocumented!
value | -1 | Value to use (default: -1) |
---|
Getting the configuration of an instance¶
[30]:
tool.get_current_config()
[30]:
{'MyTool': {'config_files': [],
'iterations': 3,
'log_config': {'version': 1,
'disable_existing_loggers': False,
'formatters': {'file': {'()': ctapipe.core.logging.PlainFormatter,
'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'},
'console': {'()': ctapipe.core.logging.ColoredFormatter,
'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'}},
'handlers': {'ctapipe-console': {'class': 'logging.StreamHandler',
'formatter': 'console',
'stream': 'ext://sys.stderr',
'level': 30},
'console': {'class': 'logging.StreamHandler',
'formatter': 'console',
'stream': 'ext://sys.stderr',
'level': 'NOTSET'}},
'loggers': {'ctapipe': {'level': 30,
'handlers': ['ctapipe-console'],
'propagate': False}}},
'log_datefmt': '%Y-%m-%d %H:%M:%S',
'log_file': None,
'log_file_level': 'INFO',
'log_format': '%(asctime)s : %(levelname)s [%(name)s %(funcName)s] %(message)s',
'log_level': 10,
'logging_config': {},
'provenance_log': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.15.0/docs/examples/mytool.provenance.log'),
'quiet': False,
'show_config': False,
'show_config_json': False,
'MyComponent': {'value': -1},
'SecondaryMyComponent': {'value': -1},
'TelescopeWiseComponent': {'param': TelescopePatternList([('type',
'*',
5.0)])},
'AdvancedComponent': {'infile': PosixPath('/home/docs/.cache/ctapipe/cccta-dataserver.in2p3.fr/data/ctapipe-extra/v0.3.4/gamma_test.simtel.gz'),
'outfile': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.15.0/docs/examples/out.csv'),
'value1': -1,
'MyComponent': {'value': -1}}}}
[31]:
tool.iterations = 12
tool.get_current_config()
[31]:
{'MyTool': {'config_files': [],
'iterations': 12,
'log_config': {'version': 1,
'disable_existing_loggers': False,
'formatters': {'file': {'()': ctapipe.core.logging.PlainFormatter,
'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'},
'console': {'()': ctapipe.core.logging.ColoredFormatter,
'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'}},
'handlers': {'ctapipe-console': {'class': 'logging.StreamHandler',
'formatter': 'console',
'stream': 'ext://sys.stderr',
'level': 30},
'console': {'class': 'logging.StreamHandler',
'formatter': 'console',
'stream': 'ext://sys.stderr',
'level': 'NOTSET'}},
'loggers': {'ctapipe': {'level': 30,
'handlers': ['ctapipe-console'],
'propagate': False}}},
'log_datefmt': '%Y-%m-%d %H:%M:%S',
'log_file': None,
'log_file_level': 'INFO',
'log_format': '%(asctime)s : %(levelname)s [%(name)s %(funcName)s] %(message)s',
'log_level': 10,
'logging_config': {},
'provenance_log': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.15.0/docs/examples/mytool.provenance.log'),
'quiet': False,
'show_config': False,
'show_config_json': False,
'MyComponent': {'value': -1},
'SecondaryMyComponent': {'value': -1},
'TelescopeWiseComponent': {'param': TelescopePatternList([('type',
'*',
5.0)])},
'AdvancedComponent': {'infile': PosixPath('/home/docs/.cache/ctapipe/cccta-dataserver.in2p3.fr/data/ctapipe-extra/v0.3.4/gamma_test.simtel.gz'),
'outfile': PosixPath('/home/docs/checkouts/readthedocs.org/user_builds/ctapipe/checkouts/v0.15.0/docs/examples/out.csv'),
'value1': -1,
'MyComponent': {'value': -1}}}}
Writing a Sample Config File¶
[32]:
print(tool.generate_config_file())
# Configuration file for mytool.
c = get_config() # noqa
#------------------------------------------------------------------------------
# Application(SingletonConfigurable) configuration
#------------------------------------------------------------------------------
## This is an application.
## The date format used by logging formatters for %(asctime)s
# Default: '%Y-%m-%d %H:%M:%S'
# c.Application.log_datefmt = '%Y-%m-%d %H:%M:%S'
## The Logging format template
# Default: '[%(name)s]%(highlevel)s %(message)s'
# c.Application.log_format = '[%(name)s]%(highlevel)s %(message)s'
## Set the log level by value or name.
# Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
# Default: 30
# c.Application.log_level = 30
## Configure additional log handlers.
#
# The default stderr logs handler is configured by the log_level, log_datefmt
# and log_format settings.
#
# This configuration can be used to configure additional handlers (e.g. to
# output the log to a file) or for finer control over the default handlers.
#
# If provided this should be a logging configuration dictionary, for more
# information see:
# https://docs.python.org/3/library/logging.config.html#logging-config-
# dictschema
#
# This dictionary is merged with the base logging configuration which defines
# the following:
#
# * A logging formatter intended for interactive use called
# ``console``.
# * A logging handler that writes to stderr called
# ``console`` which uses the formatter ``console``.
# * A logger with the name of this application set to ``DEBUG``
# level.
#
# This example adds a new handler that writes to a file:
#
# .. code-block:: python
#
# c.Application.logging_config = {
# 'handlers': {
# 'file': {
# 'class': 'logging.FileHandler',
# 'level': 'DEBUG',
# 'filename': '<path/to/file>',
# }
# },
# 'loggers': {
# '<application-name>': {
# 'level': 'DEBUG',
# # NOTE: if you don't list the default "console"
# # handler here then it will be disabled
# 'handlers': ['console', 'file'],
# },
# }
# }
# Default: {}
# c.Application.logging_config = {}
## Instead of starting the Application, dump configuration to stdout
# Default: False
# c.Application.show_config = False
## Instead of starting the Application, dump configuration to stdout (as JSON)
# Default: False
# c.Application.show_config_json = False
#------------------------------------------------------------------------------
# Tool(Application) configuration
#------------------------------------------------------------------------------
## This is an application.
# Default: []
# c.Tool.config_files = []
# Default: {'version': 1, 'disable_existing_loggers': False, 'formatters': {'file': {'()': <class 'ctapipe.core.logging.PlainFormatter'>, 'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'}, 'console': {'()': <class 'ctapipe.core.logging.ColoredFormatter'>, 'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'}}, 'handlers': {'ctapipe-console': {'class': 'logging.StreamHandler', 'formatter': 'console', 'stream': 'ext://sys.stderr', 'level': 30}, 'console': {'class': 'logging.StreamHandler', 'formatter': 'console', 'stream': 'ext://sys.stderr', 'level': 'NOTSET'}}, 'loggers': {'ctapipe': {'level': 30, 'handlers': ['ctapipe-console'], 'propagate': False}}}
# c.Tool.log_config = {'version': 1, 'disable_existing_loggers': False, 'formatters': {'file': {'()': <class 'ctapipe.core.logging.PlainFormatter'>, 'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'}, 'console': {'()': <class 'ctapipe.core.logging.ColoredFormatter'>, 'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'}}, 'handlers': {'ctapipe-console': {'class': 'logging.StreamHandler', 'formatter': 'console', 'stream': 'ext://sys.stderr', 'level': 30}, 'console': {'class': 'logging.StreamHandler', 'formatter': 'console', 'stream': 'ext://sys.stderr', 'level': 'NOTSET'}}, 'loggers': {'ctapipe': {'level': 30, 'handlers': ['ctapipe-console'], 'propagate': False}}}
## The date format used by logging formatters for %(asctime)s
# See also: Application.log_datefmt
# c.Tool.log_datefmt = '%Y-%m-%d %H:%M:%S'
## Filename for the log
# Default: None
# c.Tool.log_file = None
## Logging Level for File Logging
# Choices: any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']
# Default: 'INFO'
# c.Tool.log_file_level = 'INFO'
## The Logging format template
# See also: Application.log_format
# c.Tool.log_format = '[%(name)s]%(highlevel)s %(message)s'
## Set the log level by value or name.
# See also: Application.log_level
# c.Tool.log_level = 30
##
# See also: Application.logging_config
# c.Tool.logging_config = {}
# Default: traitlets.Undefined
# c.Tool.provenance_log = traitlets.Undefined
# Default: False
# c.Tool.quiet = False
## Instead of starting the Application, dump configuration to stdout
# See also: Application.show_config
# c.Tool.show_config = False
## Instead of starting the Application, dump configuration to stdout (as JSON)
# See also: Application.show_config_json
# c.Tool.show_config_json = False
#------------------------------------------------------------------------------
# MyTool(Tool) configuration
#------------------------------------------------------------------------------
# See also: Tool.config_files
# c.MyTool.config_files = []
## Number of times to run
# Default: 5
# c.MyTool.iterations = 5
# See also: Tool.log_config
# c.MyTool.log_config = {'version': 1, 'disable_existing_loggers': False, 'formatters': {'file': {'()': <class 'ctapipe.core.logging.PlainFormatter'>, 'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'}, 'console': {'()': <class 'ctapipe.core.logging.ColoredFormatter'>, 'fmt': '%(asctime)s %(levelname)s [%(name)s] (%(module)s.%(funcName)s): %(message)s'}}, 'handlers': {'ctapipe-console': {'class': 'logging.StreamHandler', 'formatter': 'console', 'stream': 'ext://sys.stderr', 'level': 30}, 'console': {'class': 'logging.StreamHandler', 'formatter': 'console', 'stream': 'ext://sys.stderr', 'level': 'NOTSET'}}, 'loggers': {'ctapipe': {'level': 30, 'handlers': ['ctapipe-console'], 'propagate': False}}}
## The date format used by logging formatters for %(asctime)s
# See also: Application.log_datefmt
# c.MyTool.log_datefmt = '%Y-%m-%d %H:%M:%S'
## Filename for the log
# See also: Tool.log_file
# c.MyTool.log_file = None
## Logging Level for File Logging
# See also: Tool.log_file_level
# c.MyTool.log_file_level = 'INFO'
## The Logging format template
# See also: Application.log_format
# c.MyTool.log_format = '[%(name)s]%(highlevel)s %(message)s'
## Set the log level by value or name.
# See also: Application.log_level
# c.MyTool.log_level = 30
##
# See also: Application.logging_config
# c.MyTool.logging_config = {}
# See also: Tool.provenance_log
# c.MyTool.provenance_log = traitlets.Undefined
# See also: Tool.quiet
# c.MyTool.quiet = False
## Instead of starting the Application, dump configuration to stdout
# See also: Application.show_config
# c.MyTool.show_config = False
## Instead of starting the Application, dump configuration to stdout (as JSON)
# See also: Application.show_config_json
# c.MyTool.show_config_json = False
#------------------------------------------------------------------------------
# MyComponent(Component) configuration
#------------------------------------------------------------------------------
## A Component that does stuff
## Value to use
# Default: -1
# c.MyComponent.value = -1
#------------------------------------------------------------------------------
# AdvancedComponent(Component) configuration
#------------------------------------------------------------------------------
## An advanced technique
## input file name
# Default: traitlets.Undefined
# c.AdvancedComponent.infile = traitlets.Undefined
## output file name
# Default: traitlets.Undefined
# c.AdvancedComponent.outfile = traitlets.Undefined
## Value to use
# Default: -1
# c.AdvancedComponent.value1 = -1
#------------------------------------------------------------------------------
# SecondaryMyComponent(MyComponent) configuration
#------------------------------------------------------------------------------
## Value to use
# See also: MyComponent.value
# c.SecondaryMyComponent.value = -1
#------------------------------------------------------------------------------
# TelescopeWiseComponent(TelescopeComponent) configuration
#------------------------------------------------------------------------------
## a component that contains parameters that are per-telescope configurable
## Something configurable with telescope patterns
# Default: [('type', '*', 5.0)]
# c.TelescopeWiseComponent.param = [('type', '*', 5.0)]
[ ]: