Skip to content

Logger

mlpoppyns.learning.logger.logger

Logger.

Utility functions for setting up and dealing with the logging subsystem in order to generate messages both to the console and to log useful info of the process to output files.

Authors:

Alberto Garcia Garcia (garciagarcia@ice.csic.es)

Copyright (c) MAGNESIA (ICE-CSIC)

setup_logging(log_dir, log_config_file='mlpoppyns/learning/logger/default_logger_config.json', default_level=logging.INFO)

Setup logging configuration.

Sets up the logging subsystem by reading its configuration from a JSON configuration file.

Parameters:

Name Type Description Default
log_dir str

The directory to output the log files to.

 required
log_config_file str

Path to the JSON configuration file.

 'mlpoppyns/learning/logger/default_logger_config.json'
default_level int

Default logging level.

 INFO
Source code in mlpoppyns/learning/logger/logger.py 
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
def setup_logging(
    log_dir: str,
    log_config_file: str = "mlpoppyns/learning/logger/default_logger_config.json",
    default_level: int = logging.INFO,
) -> None:
    """
    Setup logging configuration.

    Sets up the logging subsystem by reading its configuration from a JSON
    configuration file.

    Args:
        log_dir (str): The directory to output the log files to.
        log_config_file (str): Path to the JSON configuration file.
        default_level (int): Default logging level.
    """

    path_to_software = cfg["path_to_software"]
    log_config_file = pathlib.Path().joinpath(
        path_to_software, log_config_file
    )

    if log_config_file.is_file():
        log_config = json_utils.read_json(log_config_file)

        # Modify logging paths based on run configuration.
        for _, handler in log_config["handlers"].items():
            if "filename" in handler:
                if not isinstance(log_dir, pathlib.Path):
                    log_dir = pathlib.Path(log_dir)
                handler["filename"] = str(log_dir / handler["filename"])

        logging.config.dictConfig(log_config)

    else:
        print("Warning: logging configuration file is not found!")
        print("Falling back to defaults...")
        logging.basicConfig(level=default_level)

mlpoppyns.learning.logger.tensorboard_writer

Tensorboard writer.

This module provides a class for integrating Tensorboard logging functionality into your project, allowing for the visualization of metrics.

Authors:

Alberto Garcia Garcia (garciagarcia@ice.csic.es)

TensorboardWriter

A class for writing logs to Tensorboard, supporting both torch.utils.tensorboard and tensorboardX.

Attributes:

Name Type Description
writer

The Tensorboard writer object.
selected_module

The name of the selected module for writing logs.
step

The current step for logging.
mode

The current mode (e.g., "train" or "valid").
tb_writer_ftns

A set of Tensorboard writing functions.
tag_mode_exceptions

A set of functions that do not require mode tags.
timer

A datetime object to track the time between steps.
Source code in mlpoppyns/learning/logger/tensorboard_writer.py 
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
class TensorboardWriter:
    """
    A class for writing logs to Tensorboard, supporting both `torch.utils.tensorboard` and `tensorboardX`.

    Attributes:
        writer: The Tensorboard writer object.
        selected_module: The name of the selected module for writing logs.
        step: The current step for logging.
        mode: The current mode (e.g., "train" or "valid").
        tb_writer_ftns: A set of Tensorboard writing functions.
        tag_mode_exceptions: A set of functions that do not require mode tags.
        timer: A datetime object to track the time between steps.
    """

    def __init__(self, log_dir: str, logger: Logger, enabled: bool) -> None:
        """
        Initializes the TensorboardWriter with the specified log directory, logger, and enable flag.

        Args:
            log_dir (str): The directory where Tensorboard logs will be saved.
            logger (Logger): The logger for displaying warnings or errors.
            enabled (bool): Flag to enable or disable Tensorboard logging.
        """

        self.writer = None
        self.selected_module = ""

        if enabled:
            log_dir = str(log_dir)

            # Retrieve vizualization writer.
            succeeded = False

            for module in ["torch.utils.tensorboard", "tensorboardX"]:
                try:
                    self.writer = importlib.import_module(
                        module
                    ).SummaryWriter(log_dir)
                    succeeded = True
                    break

                except ImportError:
                    succeeded = False

                self.selected_module = module

            if not succeeded:
                message = (
                    "Warning: visualization (Tensorboard) is configured to use,"
                    " but currently not installed on this machine. Please "
                    " install TensorboardX with 'pip install tensorboardx', "
                    " upgrade PyTorch to version >= 1.1 to use "
                    " 'torch.utils.tensorboard' or turn off the option in the "
                    " JSON configuration file."
                )

                logger.warning(message)

        self.step = 0
        self.mode = ""

        self.tb_writer_ftns = {
            "add_scalar",
            "add_scalars",
            "add_image",
            "add_images",
            "add_audio",
            "add_text",
            "add_histogram",
            "add_pr_curve",
            "add_embedding",
        }
        self.tag_mode_exceptions = {"add_histogram", "add_embedding"}
        self.timer = datetime.datetime.now()

    def set_step(self, step: int, mode: str = "train") -> None:
        """
        Sets the current step and mode, and logs the steps per second.

        Args:
            step (int): The current step for logging.
            mode (str): The current mode (default is "train").
        """

        self.mode = mode
        self.step = step
        if step == 0:
            self.timer = datetime.datetime.now()
        else:
            duration = datetime.datetime.now() - self.timer
            self.add_scalar("steps_per_sec", 1 / duration.total_seconds())
            self.timer = datetime.datetime.now()

    def __getattr__(self, name: str) -> Union[Callable, Any]:
        """
        Provides dynamic access to Tensorboard logging methods.

        Args:
            name (str): The name of the Tensorboard method to access.

        Returns:
            (Union[Callable, Any]): A wrapped function that adds additional information (step, tag) to the Tensorboard
                log entry. If visualization is configured to use returns add_data() methods of tensorboard with
                additional information (step, tag) added. Otherwise returns a blank function handle that does nothing.
        """

        if name in self.tb_writer_ftns:
            add_data = getattr(self.writer, name, None)

            def _wrapper(
                tag: str, data: Any, *args: Any, **kwargs: Any
            ) -> None:
                """
                Wrapper function for Tensorboard logging methods.

                Adds the current mode and step information to the log entry.

                Args:
                    tag (str): The tag for the Tensorboard log entry.
                    data (Any): The data to be logged.
                    *args (Any): Additional positional arguments for the Tensorboard method.
                    **kwargs (Any): Additional keyword arguments for the Tensorboard method.
                """

                if add_data is not None:
                    # add mode(train/valid) tag
                    if name not in self.tag_mode_exceptions:
                        tag = "{}/{}".format(tag, self.mode)

                    add_data(tag, data, self.step, *args, **kwargs)

            return _wrapper

        else:
            try:
                attr = object.__getattr__(name)

            except AttributeError:
                raise AttributeError(
                    "type object '{}' has no attribute '{}'".format(
                        self.selected_module, name
                    )
                )

            return attr

__getattr__(name)

Provides dynamic access to Tensorboard logging methods.

Parameters:

Name Type Description Default
name str

The name of the Tensorboard method to access.

 required

Returns:

Type Description
Union[Callable, Any]

A wrapped function that adds additional information (step, tag) to the Tensorboard log entry. If visualization is configured to use returns add_data() methods of tensorboard with additional information (step, tag) added. Otherwise returns a blank function handle that does nothing.
Source code in mlpoppyns/learning/logger/tensorboard_writer.py 
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
def __getattr__(self, name: str) -> Union[Callable, Any]:
    """
    Provides dynamic access to Tensorboard logging methods.

    Args:
        name (str): The name of the Tensorboard method to access.

    Returns:
        (Union[Callable, Any]): A wrapped function that adds additional information (step, tag) to the Tensorboard
            log entry. If visualization is configured to use returns add_data() methods of tensorboard with
            additional information (step, tag) added. Otherwise returns a blank function handle that does nothing.
    """

    if name in self.tb_writer_ftns:
        add_data = getattr(self.writer, name, None)

        def _wrapper(
            tag: str, data: Any, *args: Any, **kwargs: Any
        ) -> None:
            """
            Wrapper function for Tensorboard logging methods.

            Adds the current mode and step information to the log entry.

            Args:
                tag (str): The tag for the Tensorboard log entry.
                data (Any): The data to be logged.
                *args (Any): Additional positional arguments for the Tensorboard method.
                **kwargs (Any): Additional keyword arguments for the Tensorboard method.
            """

            if add_data is not None:
                # add mode(train/valid) tag
                if name not in self.tag_mode_exceptions:
                    tag = "{}/{}".format(tag, self.mode)

                add_data(tag, data, self.step, *args, **kwargs)

        return _wrapper

    else:
        try:
            attr = object.__getattr__(name)

        except AttributeError:
            raise AttributeError(
                "type object '{}' has no attribute '{}'".format(
                    self.selected_module, name
                )
            )

        return attr

__init__(log_dir, logger, enabled)

Initializes the TensorboardWriter with the specified log directory, logger, and enable flag.

Parameters:

Name Type Description Default
log_dir str

The directory where Tensorboard logs will be saved.

 required
logger Logger

The logger for displaying warnings or errors.

 required
enabled bool

Flag to enable or disable Tensorboard logging.

 required
Source code in mlpoppyns/learning/logger/tensorboard_writer.py 
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
def __init__(self, log_dir: str, logger: Logger, enabled: bool) -> None:
    """
    Initializes the TensorboardWriter with the specified log directory, logger, and enable flag.

    Args:
        log_dir (str): The directory where Tensorboard logs will be saved.
        logger (Logger): The logger for displaying warnings or errors.
        enabled (bool): Flag to enable or disable Tensorboard logging.
    """

    self.writer = None
    self.selected_module = ""

    if enabled:
        log_dir = str(log_dir)

        # Retrieve vizualization writer.
        succeeded = False

        for module in ["torch.utils.tensorboard", "tensorboardX"]:
            try:
                self.writer = importlib.import_module(
                    module
                ).SummaryWriter(log_dir)
                succeeded = True
                break

            except ImportError:
                succeeded = False

            self.selected_module = module

        if not succeeded:
            message = (
                "Warning: visualization (Tensorboard) is configured to use,"
                " but currently not installed on this machine. Please "
                " install TensorboardX with 'pip install tensorboardx', "
                " upgrade PyTorch to version >= 1.1 to use "
                " 'torch.utils.tensorboard' or turn off the option in the "
                " JSON configuration file."
            )

            logger.warning(message)

    self.step = 0
    self.mode = ""

    self.tb_writer_ftns = {
        "add_scalar",
        "add_scalars",
        "add_image",
        "add_images",
        "add_audio",
        "add_text",
        "add_histogram",
        "add_pr_curve",
        "add_embedding",
    }
    self.tag_mode_exceptions = {"add_histogram", "add_embedding"}
    self.timer = datetime.datetime.now()

set_step(step, mode='train')

Sets the current step and mode, and logs the steps per second.

Parameters:

Name Type Description Default
step int

The current step for logging.

 required
mode str

The current mode (default is "train").

 'train'
Source code in mlpoppyns/learning/logger/tensorboard_writer.py 
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
def set_step(self, step: int, mode: str = "train") -> None:
    """
    Sets the current step and mode, and logs the steps per second.

    Args:
        step (int): The current step for logging.
        mode (str): The current mode (default is "train").
    """

    self.mode = mode
    self.step = step
    if step == 0:
        self.timer = datetime.datetime.now()
    else:
        duration = datetime.datetime.now() - self.timer
        self.add_scalar("steps_per_sec", 1 / duration.total_seconds())
        self.timer = datetime.datetime.now()