From 78b1428088cb47bdbea53cd236294a3e076b6c95 Mon Sep 17 00:00:00 2001 From: Ayush Thakur <31141479+ayulockin@users.noreply.github.com> Date: Wed, 20 Oct 2021 19:14:57 +0530 Subject: [PATCH 01/17] [refactor] Extend WandbLogger to log config variables, entity and kwargs (#1) ability to log config file, initialize wandb with kwargs and pass entity argument for teams account. --- mmf/configs/defaults.yaml | 8 ++++++++ mmf/trainers/callbacks/logistics.py | 10 +++++++--- mmf/utils/logger.py | 8 +++++++- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/mmf/configs/defaults.yaml b/mmf/configs/defaults.yaml index ab6c8625b..7e10c3681 100644 --- a/mmf/configs/defaults.yaml +++ b/mmf/configs/defaults.yaml @@ -45,11 +45,19 @@ training: wandb: # Whether to use Weights and Biases Logger, (Default: false) enabled: false + # An entity is a username or team name where you're sending runs. + # This is necessary if you want to log your metrics to a team account. By default + # it will log the run to your user account. + entity: null # Project name to be used while logging the experiment with wandb wandb_projectname: mmf_${oc.env:USER,} # Experiment/ run name to be used while logging the experiment # under the project with wandb wandb_runname: ${training.experiment_name} + # Specify other argument values to be used while logging the experiment + init_kwargs: + job_type: train + # Size of the batch globally. If distributed or data_parallel # is used, this will be divided equally among GPUs diff --git a/mmf/trainers/callbacks/logistics.py b/mmf/trainers/callbacks/logistics.py index 70b2ef05d..bb2e7a332 100644 --- a/mmf/trainers/callbacks/logistics.py +++ b/mmf/trainers/callbacks/logistics.py @@ -58,11 +58,15 @@ def __init__(self, config, trainer): if env_wandb_logdir: log_dir = env_wandb_logdir - wandb_projectname = config.training.wandb.wandb_projectname - wandb_runname = config.training.wandb.wandb_runname + wandb_init_kwargs = config.training.wandb.init_kwargs self.wandb_logger = WandbLogger( - name=wandb_runname, save_dir=log_dir, project=wandb_projectname + entity=config.training.wandb.entity, + project=config.training.wandb.wandb_projectname, + config=config, + name=config.training.wandb.wandb_runname, + save_dir=log_dir, + **wandb_init_kwargs, ) def on_train_start(self): diff --git a/mmf/utils/logger.py b/mmf/utils/logger.py index a82696f97..d484b396f 100644 --- a/mmf/utils/logger.py +++ b/mmf/utils/logger.py @@ -395,9 +395,11 @@ class WandbLogger: Log using `Weights and Biases`. Args: + entity: An entity is a username or team name where you're sending runs. name: Display name for the run. save_dir: Path where data is saved (./save/logs/wandb/ by default). project: Display name for the project. + config: Configuration for the run. **init_kwargs: Arguments passed to :func:`wandb.init`. Raises: @@ -406,9 +408,11 @@ class WandbLogger: def __init__( self, + entity: Optional[str] = None, name: Optional[str] = None, save_dir: Optional[str] = None, project: Optional[str] = None, + config: Optional[Dict] = None, **init_kwargs, ): try: @@ -421,7 +425,9 @@ def __init__( self._wandb = wandb - self._wandb_init = dict(name=name, project=project, dir=save_dir) + self._wandb_init = dict( + entity=entity, name=name, project=project, dir=save_dir, config=config + ) self._wandb_init.update(**init_kwargs) From a0decd2e195a40dfafbd9973900cc8f9c5e5ee1d Mon Sep 17 00:00:00 2001 From: Ayush Thakur Date: Mon, 25 Oct 2021 22:20:30 +0530 Subject: [PATCH 02/17] cleaned passing of kwargs, added wandb_logger to write validation metrics, log lr --- mmf/configs/defaults.yaml | 8 +++++--- mmf/trainers/callbacks/logistics.py | 4 +--- mmf/utils/logger.py | 21 +++++++++++++++------ 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/mmf/configs/defaults.yaml b/mmf/configs/defaults.yaml index 7e10c3681..5c02c0a59 100644 --- a/mmf/configs/defaults.yaml +++ b/mmf/configs/defaults.yaml @@ -54,9 +54,11 @@ training: # Experiment/ run name to be used while logging the experiment # under the project with wandb wandb_runname: ${training.experiment_name} - # Specify other argument values to be used while logging the experiment - init_kwargs: - job_type: train + # Specify other argument values that you want to pass to wandb.init(). Check out the documentation + # at https://docs.wandb.ai/ref/python/init to see what arguments are available. + # job_type: 'train' + # tags: ['tag1', 'tag2'] + # Size of the batch globally. If distributed or data_parallel diff --git a/mmf/trainers/callbacks/logistics.py b/mmf/trainers/callbacks/logistics.py index bb2e7a332..977a0c97b 100644 --- a/mmf/trainers/callbacks/logistics.py +++ b/mmf/trainers/callbacks/logistics.py @@ -58,15 +58,12 @@ def __init__(self, config, trainer): if env_wandb_logdir: log_dir = env_wandb_logdir - wandb_init_kwargs = config.training.wandb.init_kwargs - self.wandb_logger = WandbLogger( entity=config.training.wandb.entity, project=config.training.wandb.wandb_projectname, config=config, name=config.training.wandb.wandb_runname, save_dir=log_dir, - **wandb_init_kwargs, ) def on_train_start(self): @@ -157,6 +154,7 @@ def on_test_end(self, **kwargs): meter=kwargs["meter"], should_print=prefix, tb_writer=self.tb_writer, + wandb_logger=self.wandb_logger, ) logger.info(f"Finished run in {self.total_timer.get_time_since_start()}") diff --git a/mmf/utils/logger.py b/mmf/utils/logger.py index d484b396f..37c746fbe 100644 --- a/mmf/utils/logger.py +++ b/mmf/utils/logger.py @@ -2,6 +2,7 @@ import collections import functools +import itertools import json import logging import os @@ -231,7 +232,11 @@ def summarize_report( if wandb_logger: metrics = meter.get_scalar_dict() - wandb_logger.log_metrics({**metrics, "trainer/global_step": current_iteration}) + wandb_logger.log_metrics({**metrics, "trainer/global_step": current_iteration}, commit=False) + + # Log the learning rate if available + if wandb_logger and 'lr' in extra.keys(): + wandb_logger.log_metrics({"train/learning_rate": float(extra["lr"])}) if not should_print: return @@ -400,7 +405,6 @@ class WandbLogger: save_dir: Path where data is saved (./save/logs/wandb/ by default). project: Display name for the project. config: Configuration for the run. - **init_kwargs: Arguments passed to :func:`wandb.init`. Raises: ImportError: If wandb package is not installed. @@ -413,7 +417,6 @@ def __init__( save_dir: Optional[str] = None, project: Optional[str] = None, config: Optional[Dict] = None, - **init_kwargs, ): try: import wandb @@ -429,6 +432,11 @@ def __init__( entity=entity, name=name, project=project, dir=save_dir, config=config ) + init_kwargs = dict( + itertools.islice( + config.training.wandb.items(), 4, len(config.training.wandb) + ) + ) self._wandb_init.update(**init_kwargs) self.setup() @@ -459,14 +467,15 @@ def _should_log_wandb(self): else: return True - def log_metrics(self, metrics: Dict[str, float]): + def log_metrics(self, metrics: Dict[str, float], commit=True): """ Log the monitored metrics to the wand dashboard. Args: - metrics (Dict[str, float]): [description] + metrics (Dict[str, float]): A dictionary of metrics to log. + commit (bool): Save the metrics dict to the wandb server and increment the step. (default: True) """ if not self._should_log_wandb(): return - self._wandb.log(metrics) + self._wandb.log(metrics, commit=commit) From a97cf2a02df0c35ab654fad0e888366830f97742 Mon Sep 17 00:00:00 2001 From: Ayush Thakur Date: Tue, 26 Oct 2021 02:24:49 +0530 Subject: [PATCH 03/17] update docs --- website/docs/notes/logging.md | 57 +++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 12 deletions(-) diff --git a/website/docs/notes/logging.md b/website/docs/notes/logging.md index ebec33902..28b7a42a6 100644 --- a/website/docs/notes/logging.md +++ b/website/docs/notes/logging.md @@ -1,19 +1,27 @@ --- -id: concepts -title: Terminology and Concepts -sidebar_label: Terminology and Concepts +id: logger +title: Weights and Biases Logging +sidebar_label: Weights and Biases Logging --- ## Weights and Biases Logger -MMF has a `WandbLogger` class which lets the user to log their model's progress using [Weights and Biases](https://gitbook-docs.wandb.ai/). +MMF now has a `WandbLogger` class which lets the user to log their model's progress using [Weights and Biases](https://wandb.ai/site). Enable this logger to automatically log the training/validation metrics, system (GPU and CPU) metrics and configuration parameters. + +## First time setup To set up wandb, run the following: ``` pip install wandb +``` +In order to log anything to the W&B server you need to authenticate the machine with W&B **API key**. You can create a new account by going to https://wandb.ai/signup which will generate an API key. If you are an existing user you can retrieve your key from https://wandb.ai/authorize. You only need to supply your key once, and then it is remembered on the same device. + +``` wandb login ``` +## W&B config parameters + The following options are available in config to enable and customize the wandb logging: ```yaml training: @@ -21,22 +29,47 @@ training: wandb: # Whether to use Weights and Biases Logger, (Default: false) enabled: false + # An entity is a username or team name where you're sending runs. + # This is necessary if you want to log your metrics to a team account. By default + # it will log the run to your user account. + entity: null # Project name to be used while logging the experiment with wandb - wandb_projectname: mmf_${oc.env:USER} + wandb_projectname: mmf_${oc.env:USER,} # Experiment/ run name to be used while logging the experiment # under the project with wandb wandb_runname: ${training.experiment_name} + # Specify other argument values that you want to pass to wandb.init(). Check out the documentation + # at https://docs.wandb.ai/ref/python/init to see what arguments are available. + # job_type: 'train' + # tags: ['tag1', 'tag2'] env: wandb_logdir: ${env:MMF_WANDB_LOGDIR,} -``` -To enable wandb logger the user needs to change the following option in the config. +``` + +* To enable wandb logger the user needs to change the following option in the config. + + `training.wandb.enabled=True` + +* To give the `entity` which is the name of the team or the username, the user needs to change the following option in the config. In case no `entity` is provided, the data will be logged to the `entity` set as default in the user's settings. + + `training.wandb.entity=` + +* To give the current experiment a project and run name, user should add these config options. + + `training.wandb.wandb_projectname=`
+ `training.wandb.wandb_runname=` + +* To change the path to the directory where wandb metadata would be stored (Default: `env.log_dir`): -`training.wandb.enabled=True` + `env.wandb_logdir=` -To give the current experiment a project and run name, user should add these config options. +* To provide extra arguments to `wandb.init()`, the user just needs to define them in the config file. Check out the documentation at https://docs.wandb.ai/ref/python/init to see what arguments are available. An example is shown in the config parameter shown above. -`training.wandb.wandb_projectname= training.wandb.wandb_runname=` +## Current features -To change the path to the directory where wandb metadata would be stored (Default: `env.log_dir`): +The following features are currently supported by the `WandbLogger`: -`env.wandb_logdir=` +* Training & Validation metrics +* Learning Rate over time +* GPU: Type, GPU Utilization, power, temperature, CUDA memory usage +* Log configuration parameters From 5cc98b84836a13841304c748b084ac1914072587 Mon Sep 17 00:00:00 2001 From: Ayush Thakur <31141479+ayulockin@users.noreply.github.com> Date: Tue, 26 Oct 2021 17:56:29 +0530 Subject: [PATCH 04/17] init kwargs (#3) --- mmf/configs/defaults.yaml | 4 +-- mmf/trainers/callbacks/logistics.py | 4 +-- mmf/utils/logger.py | 42 ++++++++++++++--------------- website/docs/notes/logging.md | 22 +++++++-------- 4 files changed, 35 insertions(+), 37 deletions(-) diff --git a/mmf/configs/defaults.yaml b/mmf/configs/defaults.yaml index 5c02c0a59..fc6d3abb3 100644 --- a/mmf/configs/defaults.yaml +++ b/mmf/configs/defaults.yaml @@ -50,10 +50,10 @@ training: # it will log the run to your user account. entity: null # Project name to be used while logging the experiment with wandb - wandb_projectname: mmf_${oc.env:USER,} + project: mmf # Experiment/ run name to be used while logging the experiment # under the project with wandb - wandb_runname: ${training.experiment_name} + name: ${training.experiment_name} # Specify other argument values that you want to pass to wandb.init(). Check out the documentation # at https://docs.wandb.ai/ref/python/init to see what arguments are available. # job_type: 'train' diff --git a/mmf/trainers/callbacks/logistics.py b/mmf/trainers/callbacks/logistics.py index 977a0c97b..0804f5218 100644 --- a/mmf/trainers/callbacks/logistics.py +++ b/mmf/trainers/callbacks/logistics.py @@ -60,10 +60,8 @@ def __init__(self, config, trainer): self.wandb_logger = WandbLogger( entity=config.training.wandb.entity, - project=config.training.wandb.wandb_projectname, config=config, - name=config.training.wandb.wandb_runname, - save_dir=log_dir, + project=config.training.wandb.project, ) def on_train_start(self): diff --git a/mmf/utils/logger.py b/mmf/utils/logger.py index 37c746fbe..c3b611f71 100644 --- a/mmf/utils/logger.py +++ b/mmf/utils/logger.py @@ -2,7 +2,6 @@ import collections import functools -import itertools import json import logging import os @@ -11,12 +10,14 @@ from functools import wraps from typing import Any, Callable, Dict, Optional, Union +import omegaconf import torch from mmf.common.registry import registry from mmf.utils.configuration import get_mmf_env from mmf.utils.distributed import get_rank, is_main, is_xla from mmf.utils.file_io import PathManager from mmf.utils.timer import Timer +from omegaconf import OmegaConf from termcolor import colored @@ -226,17 +227,19 @@ def summarize_report( if not is_main() and not is_xla(): return + # Log the learning rate if available + if wandb_logger and "lr" in extra.keys(): + wandb_logger.log_metrics( + {"train/learning_rate": float(extra["lr"])}, commit=False + ) + if tb_writer: scalar_dict = meter.get_scalar_dict() tb_writer.add_scalars(scalar_dict, current_iteration) if wandb_logger: metrics = meter.get_scalar_dict() - wandb_logger.log_metrics({**metrics, "trainer/global_step": current_iteration}, commit=False) - - # Log the learning rate if available - if wandb_logger and 'lr' in extra.keys(): - wandb_logger.log_metrics({"train/learning_rate": float(extra["lr"])}) + wandb_logger.log_metrics({**metrics, "trainer/global_step": current_iteration}) if not should_print: return @@ -401,10 +404,8 @@ class WandbLogger: Args: entity: An entity is a username or team name where you're sending runs. - name: Display name for the run. - save_dir: Path where data is saved (./save/logs/wandb/ by default). - project: Display name for the project. config: Configuration for the run. + project: Name of the W&B project. Raises: ImportError: If wandb package is not installed. @@ -413,10 +414,8 @@ class WandbLogger: def __init__( self, entity: Optional[str] = None, - name: Optional[str] = None, - save_dir: Optional[str] = None, - project: Optional[str] = None, config: Optional[Dict] = None, + project: Optional[str] = None, ): try: import wandb @@ -428,15 +427,15 @@ def __init__( self._wandb = wandb - self._wandb_init = dict( - entity=entity, name=name, project=project, dir=save_dir, config=config - ) + self._wandb_init = dict(entity=entity, config=config, project=project) - init_kwargs = dict( - itertools.islice( - config.training.wandb.items(), 4, len(config.training.wandb) - ) - ) + wandb_params = config.training.wandb + with omegaconf.open_dict(wandb_params): + wandb_params.pop("enabled") + wandb_params.pop("entity") + wandb_params.pop("project") + + init_kwargs = OmegaConf.to_container(wandb_params, resolve=True) self._wandb_init.update(**init_kwargs) self.setup() @@ -473,7 +472,8 @@ def log_metrics(self, metrics: Dict[str, float], commit=True): Args: metrics (Dict[str, float]): A dictionary of metrics to log. - commit (bool): Save the metrics dict to the wandb server and increment the step. (default: True) + commit (bool): Save the metrics dict to the wandb server and + increment the step. (default: True) """ if not self._should_log_wandb(): return diff --git a/website/docs/notes/logging.md b/website/docs/notes/logging.md index 28b7a42a6..3dde92270 100644 --- a/website/docs/notes/logging.md +++ b/website/docs/notes/logging.md @@ -8,13 +8,13 @@ sidebar_label: Weights and Biases Logging MMF now has a `WandbLogger` class which lets the user to log their model's progress using [Weights and Biases](https://wandb.ai/site). Enable this logger to automatically log the training/validation metrics, system (GPU and CPU) metrics and configuration parameters. -## First time setup +## First time setup To set up wandb, run the following: ``` pip install wandb ``` -In order to log anything to the W&B server you need to authenticate the machine with W&B **API key**. You can create a new account by going to https://wandb.ai/signup which will generate an API key. If you are an existing user you can retrieve your key from https://wandb.ai/authorize. You only need to supply your key once, and then it is remembered on the same device. +In order to log anything to the W&B server you need to authenticate the machine with W&B **API key**. You can create a new account by going to https://wandb.ai/signup which will generate an API key. If you are an existing user you can retrieve your key from https://wandb.ai/authorize. You only need to supply your key once, and then it is remembered on the same device. ``` wandb login @@ -28,42 +28,42 @@ training: # Weights and Biases control, by default Weights and Biases (wandb) is disabled wandb: # Whether to use Weights and Biases Logger, (Default: false) - enabled: false + enabled: true # An entity is a username or team name where you're sending runs. # This is necessary if you want to log your metrics to a team account. By default # it will log the run to your user account. entity: null # Project name to be used while logging the experiment with wandb - wandb_projectname: mmf_${oc.env:USER,} + project: mmf # Experiment/ run name to be used while logging the experiment # under the project with wandb - wandb_runname: ${training.experiment_name} + name: ${training.experiment_name} # Specify other argument values that you want to pass to wandb.init(). Check out the documentation # at https://docs.wandb.ai/ref/python/init to see what arguments are available. # job_type: 'train' # tags: ['tag1', 'tag2'] env: wandb_logdir: ${env:MMF_WANDB_LOGDIR,} -``` +``` * To enable wandb logger the user needs to change the following option in the config. `training.wandb.enabled=True` -* To give the `entity` which is the name of the team or the username, the user needs to change the following option in the config. In case no `entity` is provided, the data will be logged to the `entity` set as default in the user's settings. +* To give the `entity` which is the name of the team or the username, the user needs to change the following option in the config. In case no `entity` is provided, the data will be logged to the `entity` set as default in the user's settings. `training.wandb.entity=` -* To give the current experiment a project and run name, user should add these config options. +* To give the current experiment a project and run name, user should add these config options. The default project name is `mmf` and the default run name is `${training.experiment_name}`. - `training.wandb.wandb_projectname=`
- `training.wandb.wandb_runname=` + `training.wandb.project=`
+ `training.wandb.name=` * To change the path to the directory where wandb metadata would be stored (Default: `env.log_dir`): `env.wandb_logdir=` -* To provide extra arguments to `wandb.init()`, the user just needs to define them in the config file. Check out the documentation at https://docs.wandb.ai/ref/python/init to see what arguments are available. An example is shown in the config parameter shown above. +* To provide extra arguments to `wandb.init()`, the user just needs to define them in the config file. Check out the documentation at https://docs.wandb.ai/ref/python/init to see what arguments are available. An example is shown in the config parameter shown above. Make sure to use the same key name in the config file as defined in the documentation. ## Current features From 5d4ee99c6c6bb3fccaac4a2e7272e4dbd36a341d Mon Sep 17 00:00:00 2001 From: Ayush Thakur Date: Wed, 27 Oct 2021 11:15:06 +0530 Subject: [PATCH 05/17] wandb checkpointing --- mmf/configs/defaults.yaml | 5 ++++- mmf/utils/checkpoint.py | 13 +++++++++++++ mmf/utils/logger.py | 22 +++++++++++++++++++++- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/mmf/configs/defaults.yaml b/mmf/configs/defaults.yaml index fc6d3abb3..68ed25f8b 100644 --- a/mmf/configs/defaults.yaml +++ b/mmf/configs/defaults.yaml @@ -44,7 +44,7 @@ training: # Weights and Biases control, by default Weights and Biases (wandb) is disabled wandb: # Whether to use Weights and Biases Logger, (Default: false) - enabled: false + enabled: true # An entity is a username or team name where you're sending runs. # This is necessary if you want to log your metrics to a team account. By default # it will log the run to your user account. @@ -54,6 +54,9 @@ training: # Experiment/ run name to be used while logging the experiment # under the project with wandb name: ${training.experiment_name} + # You can save your model checkpoints as W&B Artifacts for model versioning. + # Set the value to `true` to enable this feature. + log_checkpoint: true # Specify other argument values that you want to pass to wandb.init(). Check out the documentation # at https://docs.wandb.ai/ref/python/init to see what arguments are available. # job_type: 'train' diff --git a/mmf/utils/checkpoint.py b/mmf/utils/checkpoint.py index 5c3dd3944..a7b70bba3 100644 --- a/mmf/utils/checkpoint.py +++ b/mmf/utils/checkpoint.py @@ -522,6 +522,7 @@ def save(self, update, iteration=None, update_best=False): best_metric = ( self.trainer.early_stop_callback.early_stopping.best_monitored_value ) + model = self.trainer.model data_parallel = registry.get("data_parallel") or registry.get("distributed") fp16_scaler = getattr(self.trainer, "scaler", None) @@ -574,6 +575,18 @@ def save(self, update, iteration=None, update_best=False): with open_if_main(current_ckpt_filepath, "wb") as f: self.save_func(ckpt, f) + # Save the current checkpoint as W&B artifacts for model versioning. + if ( + self.config.training.wandb.enabled + and self.config.training.wandb.log_checkpoint + ): + logger.info( + "Saving current checkpoint as W&B Artifacts for model versioning" + ) + self.trainer.logistics_callback.wandb_logger.log_model_checkpoint( + current_ckpt_filepath, ckpt + ) + # Remove old checkpoints if max_to_keep is set # In XLA, only delete checkpoint files in main process if self.max_to_keep > 0 and is_main(): diff --git a/mmf/utils/logger.py b/mmf/utils/logger.py index c3b611f71..fd4b1865a 100644 --- a/mmf/utils/logger.py +++ b/mmf/utils/logger.py @@ -1,6 +1,7 @@ # Copyright (c) Facebook, Inc. and its affiliates. import collections +import copy import functools import json import logging @@ -429,11 +430,12 @@ def __init__( self._wandb_init = dict(entity=entity, config=config, project=project) - wandb_params = config.training.wandb + wandb_params = copy.copy(config.training.wandb) with omegaconf.open_dict(wandb_params): wandb_params.pop("enabled") wandb_params.pop("entity") wandb_params.pop("project") + wandb_params.pop("log_checkpoint") init_kwargs = OmegaConf.to_container(wandb_params, resolve=True) self._wandb_init.update(**init_kwargs) @@ -479,3 +481,21 @@ def log_metrics(self, metrics: Dict[str, float], commit=True): return self._wandb.log(metrics, commit=commit) + + def log_model_checkpoint(self, model_path, ckpt_dict): + """ + Log the model checkpoint to the wandb dashboard. + + Args: + model_path (str): Path to the model file. + ckpt_dict (Dict[str, Any]): Checkpoint dictionary. + """ + if not self._should_log_wandb(): + return + + model_artifact = self._wandb.Artifact( + "run_" + self._wandb.run.id + "_model", type="model" + ) + + model_artifact.add_file(model_path, name="current.pt") + self._wandb.log_artifact(model_artifact, aliases=["latest"]) From aec0bd7c7daa6b4cf9e98034600d378c04e49768 Mon Sep 17 00:00:00 2001 From: Ayush Thakur Date: Wed, 27 Oct 2021 12:30:20 +0530 Subject: [PATCH 06/17] wandb default false --- mmf/configs/defaults.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mmf/configs/defaults.yaml b/mmf/configs/defaults.yaml index 68ed25f8b..de2267325 100644 --- a/mmf/configs/defaults.yaml +++ b/mmf/configs/defaults.yaml @@ -44,7 +44,7 @@ training: # Weights and Biases control, by default Weights and Biases (wandb) is disabled wandb: # Whether to use Weights and Biases Logger, (Default: false) - enabled: true + enabled: false # An entity is a username or team name where you're sending runs. # This is necessary if you want to log your metrics to a team account. By default # it will log the run to your user account. @@ -56,7 +56,7 @@ training: name: ${training.experiment_name} # You can save your model checkpoints as W&B Artifacts for model versioning. # Set the value to `true` to enable this feature. - log_checkpoint: true + log_checkpoint: false # Specify other argument values that you want to pass to wandb.init(). Check out the documentation # at https://docs.wandb.ai/ref/python/init to see what arguments are available. # job_type: 'train' From 5dbbbd8dcc3c2f38da0230a86dec5850b37ce38d Mon Sep 17 00:00:00 2001 From: Evan Smothers Date: Mon, 1 Nov 2021 16:35:58 +0000 Subject: [PATCH 07/17] replace usage of open_dict --- mmf/utils/logger.py | 22 ++++++++-------------- website/docs/notes/logging.md | 10 +++++----- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/mmf/utils/logger.py b/mmf/utils/logger.py index c3b611f71..aa2ac522c 100644 --- a/mmf/utils/logger.py +++ b/mmf/utils/logger.py @@ -7,17 +7,16 @@ import os import sys import time +from copy import deepcopy from functools import wraps from typing import Any, Callable, Dict, Optional, Union -import omegaconf import torch from mmf.common.registry import registry from mmf.utils.configuration import get_mmf_env from mmf.utils.distributed import get_rank, is_main, is_xla from mmf.utils.file_io import PathManager from mmf.utils.timer import Timer -from omegaconf import OmegaConf from termcolor import colored @@ -228,7 +227,7 @@ def summarize_report( return # Log the learning rate if available - if wandb_logger and "lr" in extra.keys(): + if wandb_logger and "lr" in extra: wandb_logger.log_metrics( {"train/learning_rate": float(extra["lr"])}, commit=False ) @@ -426,17 +425,12 @@ def __init__( ) self._wandb = wandb - - self._wandb_init = dict(entity=entity, config=config, project=project) - - wandb_params = config.training.wandb - with omegaconf.open_dict(wandb_params): - wandb_params.pop("enabled") - wandb_params.pop("entity") - wandb_params.pop("project") - - init_kwargs = OmegaConf.to_container(wandb_params, resolve=True) - self._wandb_init.update(**init_kwargs) + self._wandb_init = dict(entity=entity, project=project) + wandb_kwargs = deepcopy(config.training.wandb) + wandb_kwargs.pop("enabled") + wandb_kwargs.pop("entity") + wandb_kwargs.pop("project") + self._wandb_init.update(**wandb_kwargs) self.setup() diff --git a/website/docs/notes/logging.md b/website/docs/notes/logging.md index 3dde92270..dee7f2828 100644 --- a/website/docs/notes/logging.md +++ b/website/docs/notes/logging.md @@ -48,20 +48,20 @@ env: * To enable wandb logger the user needs to change the following option in the config. - `training.wandb.enabled=True` + `training.wandb.enabled=True` * To give the `entity` which is the name of the team or the username, the user needs to change the following option in the config. In case no `entity` is provided, the data will be logged to the `entity` set as default in the user's settings. - `training.wandb.entity=` + `training.wandb.entity=` * To give the current experiment a project and run name, user should add these config options. The default project name is `mmf` and the default run name is `${training.experiment_name}`. - `training.wandb.project=`
- `training.wandb.name=` + `training.wandb.project=`
+ `training.wandb.name=` * To change the path to the directory where wandb metadata would be stored (Default: `env.log_dir`): - `env.wandb_logdir=` + `env.wandb_logdir=` * To provide extra arguments to `wandb.init()`, the user just needs to define them in the config file. Check out the documentation at https://docs.wandb.ai/ref/python/init to see what arguments are available. An example is shown in the config parameter shown above. Make sure to use the same key name in the config file as defined in the documentation. From 23addf034df2c99a14b700eded6fba331623de21 Mon Sep 17 00:00:00 2001 From: Evan Smothers Date: Mon, 1 Nov 2021 18:35:19 +0000 Subject: [PATCH 08/17] Add config back to WandB init --- mmf/utils/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmf/utils/logger.py b/mmf/utils/logger.py index aa2ac522c..ad819c498 100644 --- a/mmf/utils/logger.py +++ b/mmf/utils/logger.py @@ -425,7 +425,7 @@ def __init__( ) self._wandb = wandb - self._wandb_init = dict(entity=entity, project=project) + self._wandb_init = dict(entity=entity, config=config, project=project) wandb_kwargs = deepcopy(config.training.wandb) wandb_kwargs.pop("enabled") wandb_kwargs.pop("entity") From 2571772821a5b5ffa3da4f360c91b58756d18aac Mon Sep 17 00:00:00 2001 From: Ayush Thakur <31141479+ayulockin@users.noreply.github.com> Date: Wed, 10 Nov 2021 02:34:10 +0530 Subject: [PATCH 09/17] minor change to correct the error it was throwing (#4) --- mmf/utils/logger.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mmf/utils/logger.py b/mmf/utils/logger.py index ad819c498..cc06283c6 100644 --- a/mmf/utils/logger.py +++ b/mmf/utils/logger.py @@ -7,7 +7,6 @@ import os import sys import time -from copy import deepcopy from functools import wraps from typing import Any, Callable, Dict, Optional, Union @@ -426,7 +425,7 @@ def __init__( self._wandb = wandb self._wandb_init = dict(entity=entity, config=config, project=project) - wandb_kwargs = deepcopy(config.training.wandb) + wandb_kwargs = dict(config.training.wandb) wandb_kwargs.pop("enabled") wandb_kwargs.pop("entity") wandb_kwargs.pop("project") From 8f4a55e6cfc3ca996aecc51a91c36e54483cc344 Mon Sep 17 00:00:00 2001 From: Ayush Thakur Date: Wed, 10 Nov 2021 03:18:47 +0530 Subject: [PATCH 10/17] update checkpointing --- mmf/utils/checkpoint.py | 2 +- mmf/utils/logger.py | 22 +++++++--------------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/mmf/utils/checkpoint.py b/mmf/utils/checkpoint.py index a7b70bba3..f0f5ce658 100644 --- a/mmf/utils/checkpoint.py +++ b/mmf/utils/checkpoint.py @@ -584,7 +584,7 @@ def save(self, update, iteration=None, update_best=False): "Saving current checkpoint as W&B Artifacts for model versioning" ) self.trainer.logistics_callback.wandb_logger.log_model_checkpoint( - current_ckpt_filepath, ckpt + current_ckpt_filepath ) # Remove old checkpoints if max_to_keep is set diff --git a/mmf/utils/logger.py b/mmf/utils/logger.py index fd4b1865a..36cb73ce7 100644 --- a/mmf/utils/logger.py +++ b/mmf/utils/logger.py @@ -1,7 +1,6 @@ # Copyright (c) Facebook, Inc. and its affiliates. import collections -import copy import functools import json import logging @@ -11,14 +10,12 @@ from functools import wraps from typing import Any, Callable, Dict, Optional, Union -import omegaconf import torch from mmf.common.registry import registry from mmf.utils.configuration import get_mmf_env from mmf.utils.distributed import get_rank, is_main, is_xla from mmf.utils.file_io import PathManager from mmf.utils.timer import Timer -from omegaconf import OmegaConf from termcolor import colored @@ -429,16 +426,12 @@ def __init__( self._wandb = wandb self._wandb_init = dict(entity=entity, config=config, project=project) - - wandb_params = copy.copy(config.training.wandb) - with omegaconf.open_dict(wandb_params): - wandb_params.pop("enabled") - wandb_params.pop("entity") - wandb_params.pop("project") - wandb_params.pop("log_checkpoint") - - init_kwargs = OmegaConf.to_container(wandb_params, resolve=True) - self._wandb_init.update(**init_kwargs) + wandb_kwargs = dict(config.training.wandb) + wandb_kwargs.pop("enabled") + wandb_kwargs.pop("entity") + wandb_kwargs.pop("project") + wandb_kwargs.pop("log_checkpoint") + self._wandb_init.update(**wandb_kwargs) self.setup() @@ -482,13 +475,12 @@ def log_metrics(self, metrics: Dict[str, float], commit=True): self._wandb.log(metrics, commit=commit) - def log_model_checkpoint(self, model_path, ckpt_dict): + def log_model_checkpoint(self, model_path): """ Log the model checkpoint to the wandb dashboard. Args: model_path (str): Path to the model file. - ckpt_dict (Dict[str, Any]): Checkpoint dictionary. """ if not self._should_log_wandb(): return From 2c5e2409e888db6a17d30afc29e6b9057f8d00a8 Mon Sep 17 00:00:00 2001 From: Ayush Thakur Date: Wed, 10 Nov 2021 06:36:45 +0530 Subject: [PATCH 11/17] remove extra condition to check --- mmf/utils/checkpoint.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mmf/utils/checkpoint.py b/mmf/utils/checkpoint.py index f0f5ce658..e583f8394 100644 --- a/mmf/utils/checkpoint.py +++ b/mmf/utils/checkpoint.py @@ -576,10 +576,7 @@ def save(self, update, iteration=None, update_best=False): self.save_func(ckpt, f) # Save the current checkpoint as W&B artifacts for model versioning. - if ( - self.config.training.wandb.enabled - and self.config.training.wandb.log_checkpoint - ): + if self.config.training.wandb.log_checkpoint: logger.info( "Saving current checkpoint as W&B Artifacts for model versioning" ) From 443f8c2f9ce03514479611612a916b5cadc115dd Mon Sep 17 00:00:00 2001 From: Ayush Thakur Date: Wed, 27 Oct 2021 11:15:06 +0530 Subject: [PATCH 12/17] wandb checkpointing --- mmf/configs/defaults.yaml | 5 ++++- mmf/utils/checkpoint.py | 13 +++++++++++++ mmf/utils/logger.py | 19 +++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/mmf/configs/defaults.yaml b/mmf/configs/defaults.yaml index fc6d3abb3..68ed25f8b 100644 --- a/mmf/configs/defaults.yaml +++ b/mmf/configs/defaults.yaml @@ -44,7 +44,7 @@ training: # Weights and Biases control, by default Weights and Biases (wandb) is disabled wandb: # Whether to use Weights and Biases Logger, (Default: false) - enabled: false + enabled: true # An entity is a username or team name where you're sending runs. # This is necessary if you want to log your metrics to a team account. By default # it will log the run to your user account. @@ -54,6 +54,9 @@ training: # Experiment/ run name to be used while logging the experiment # under the project with wandb name: ${training.experiment_name} + # You can save your model checkpoints as W&B Artifacts for model versioning. + # Set the value to `true` to enable this feature. + log_checkpoint: true # Specify other argument values that you want to pass to wandb.init(). Check out the documentation # at https://docs.wandb.ai/ref/python/init to see what arguments are available. # job_type: 'train' diff --git a/mmf/utils/checkpoint.py b/mmf/utils/checkpoint.py index 5c3dd3944..a7b70bba3 100644 --- a/mmf/utils/checkpoint.py +++ b/mmf/utils/checkpoint.py @@ -522,6 +522,7 @@ def save(self, update, iteration=None, update_best=False): best_metric = ( self.trainer.early_stop_callback.early_stopping.best_monitored_value ) + model = self.trainer.model data_parallel = registry.get("data_parallel") or registry.get("distributed") fp16_scaler = getattr(self.trainer, "scaler", None) @@ -574,6 +575,18 @@ def save(self, update, iteration=None, update_best=False): with open_if_main(current_ckpt_filepath, "wb") as f: self.save_func(ckpt, f) + # Save the current checkpoint as W&B artifacts for model versioning. + if ( + self.config.training.wandb.enabled + and self.config.training.wandb.log_checkpoint + ): + logger.info( + "Saving current checkpoint as W&B Artifacts for model versioning" + ) + self.trainer.logistics_callback.wandb_logger.log_model_checkpoint( + current_ckpt_filepath, ckpt + ) + # Remove old checkpoints if max_to_keep is set # In XLA, only delete checkpoint files in main process if self.max_to_keep > 0 and is_main(): diff --git a/mmf/utils/logger.py b/mmf/utils/logger.py index cc06283c6..89063c0eb 100644 --- a/mmf/utils/logger.py +++ b/mmf/utils/logger.py @@ -1,6 +1,7 @@ # Copyright (c) Facebook, Inc. and its affiliates. import collections +import copy import functools import json import logging @@ -472,3 +473,21 @@ def log_metrics(self, metrics: Dict[str, float], commit=True): return self._wandb.log(metrics, commit=commit) + + def log_model_checkpoint(self, model_path, ckpt_dict): + """ + Log the model checkpoint to the wandb dashboard. + + Args: + model_path (str): Path to the model file. + ckpt_dict (Dict[str, Any]): Checkpoint dictionary. + """ + if not self._should_log_wandb(): + return + + model_artifact = self._wandb.Artifact( + "run_" + self._wandb.run.id + "_model", type="model" + ) + + model_artifact.add_file(model_path, name="current.pt") + self._wandb.log_artifact(model_artifact, aliases=["latest"]) From dd9db1d1bed657f24ca20c2e05c32146a3663ec6 Mon Sep 17 00:00:00 2001 From: Ayush Thakur Date: Wed, 27 Oct 2021 12:30:20 +0530 Subject: [PATCH 13/17] wandb default false --- mmf/configs/defaults.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mmf/configs/defaults.yaml b/mmf/configs/defaults.yaml index 68ed25f8b..de2267325 100644 --- a/mmf/configs/defaults.yaml +++ b/mmf/configs/defaults.yaml @@ -44,7 +44,7 @@ training: # Weights and Biases control, by default Weights and Biases (wandb) is disabled wandb: # Whether to use Weights and Biases Logger, (Default: false) - enabled: true + enabled: false # An entity is a username or team name where you're sending runs. # This is necessary if you want to log your metrics to a team account. By default # it will log the run to your user account. @@ -56,7 +56,7 @@ training: name: ${training.experiment_name} # You can save your model checkpoints as W&B Artifacts for model versioning. # Set the value to `true` to enable this feature. - log_checkpoint: true + log_checkpoint: false # Specify other argument values that you want to pass to wandb.init(). Check out the documentation # at https://docs.wandb.ai/ref/python/init to see what arguments are available. # job_type: 'train' From 2d0e08e575e89c4e08b49f7bcff559f5608acea3 Mon Sep 17 00:00:00 2001 From: Ayush Thakur Date: Wed, 10 Nov 2021 03:18:47 +0530 Subject: [PATCH 14/17] update checkpointing --- mmf/utils/checkpoint.py | 2 +- mmf/utils/logger.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/mmf/utils/checkpoint.py b/mmf/utils/checkpoint.py index a7b70bba3..f0f5ce658 100644 --- a/mmf/utils/checkpoint.py +++ b/mmf/utils/checkpoint.py @@ -584,7 +584,7 @@ def save(self, update, iteration=None, update_best=False): "Saving current checkpoint as W&B Artifacts for model versioning" ) self.trainer.logistics_callback.wandb_logger.log_model_checkpoint( - current_ckpt_filepath, ckpt + current_ckpt_filepath ) # Remove old checkpoints if max_to_keep is set diff --git a/mmf/utils/logger.py b/mmf/utils/logger.py index 89063c0eb..b4b7acbc2 100644 --- a/mmf/utils/logger.py +++ b/mmf/utils/logger.py @@ -1,7 +1,6 @@ # Copyright (c) Facebook, Inc. and its affiliates. import collections -import copy import functools import json import logging @@ -430,6 +429,7 @@ def __init__( wandb_kwargs.pop("enabled") wandb_kwargs.pop("entity") wandb_kwargs.pop("project") + wandb_kwargs.pop("log_checkpoint") self._wandb_init.update(**wandb_kwargs) self.setup() @@ -474,13 +474,12 @@ def log_metrics(self, metrics: Dict[str, float], commit=True): self._wandb.log(metrics, commit=commit) - def log_model_checkpoint(self, model_path, ckpt_dict): + def log_model_checkpoint(self, model_path): """ Log the model checkpoint to the wandb dashboard. Args: model_path (str): Path to the model file. - ckpt_dict (Dict[str, Any]): Checkpoint dictionary. """ if not self._should_log_wandb(): return From 00593021029b2a7cee06350561572e5b3d621c60 Mon Sep 17 00:00:00 2001 From: Ayush Thakur Date: Wed, 10 Nov 2021 06:36:45 +0530 Subject: [PATCH 15/17] remove extra condition to check --- mmf/utils/checkpoint.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mmf/utils/checkpoint.py b/mmf/utils/checkpoint.py index f0f5ce658..e583f8394 100644 --- a/mmf/utils/checkpoint.py +++ b/mmf/utils/checkpoint.py @@ -576,10 +576,7 @@ def save(self, update, iteration=None, update_best=False): self.save_func(ckpt, f) # Save the current checkpoint as W&B artifacts for model versioning. - if ( - self.config.training.wandb.enabled - and self.config.training.wandb.log_checkpoint - ): + if self.config.training.wandb.log_checkpoint: logger.info( "Saving current checkpoint as W&B Artifacts for model versioning" ) From d2b1f4c3d2483337ee420dba18a0a7c8d51d6873 Mon Sep 17 00:00:00 2001 From: Ayush Thakur Date: Mon, 22 Nov 2021 09:44:53 +0530 Subject: [PATCH 16/17] wandb tables 1 --- mmf/configs/defaults.yaml | 2 ++ mmf/trainers/core/evaluation_loop.py | 6 ++++++ mmf/utils/logger.py | 18 ++++++++++++++++++ 3 files changed, 26 insertions(+) diff --git a/mmf/configs/defaults.yaml b/mmf/configs/defaults.yaml index de2267325..73d63fab9 100644 --- a/mmf/configs/defaults.yaml +++ b/mmf/configs/defaults.yaml @@ -57,6 +57,8 @@ training: # You can save your model checkpoints as W&B Artifacts for model versioning. # Set the value to `true` to enable this feature. log_checkpoint: false + # Set the evaluation prediction report as W&B Tables. + log_tables: false # Specify other argument values that you want to pass to wandb.init(). Check out the documentation # at https://docs.wandb.ai/ref/python/init to see what arguments are available. # job_type: 'train' diff --git a/mmf/trainers/core/evaluation_loop.py b/mmf/trainers/core/evaluation_loop.py index 17d3f554a..cfd063e80 100644 --- a/mmf/trainers/core/evaluation_loop.py +++ b/mmf/trainers/core/evaluation_loop.py @@ -144,6 +144,12 @@ def prediction_loop(self, dataset_type: str) -> None: reporter.postprocess_dataset_report() + # Log the prediction report as W&B Tables + if self.config.training.wandb.log_tables: + self.logistics_callback.wandb_logger.log_prediction_report( + reporter.report, reporter.current_datamodule.dataset_name + ) + logger.info(f"Finished predicting. Loaded {loaded_batches}") logger.info(f" -- skipped {skipped_batches} batches.") self.model.train() diff --git a/mmf/utils/logger.py b/mmf/utils/logger.py index b4b7acbc2..36c3101d4 100644 --- a/mmf/utils/logger.py +++ b/mmf/utils/logger.py @@ -490,3 +490,21 @@ def log_model_checkpoint(self, model_path): model_artifact.add_file(model_path, name="current.pt") self._wandb.log_artifact(model_artifact, aliases=["latest"]) + + def log_prediction_report(self, report, dataset_name): + """ + Log the prediction report as W&B Tables for better comparison. + Args: + report: Prediction report to log. + dataset_name: Name of the dataset. + """ + if not self._should_log_wandb(): + return + + columns = list(report[0].keys()) + data_at = self._wandb.Table(columns=columns) + + for item in report: + data_at.add_data(*item.values()) + + self._wandb.log({f"pred_table_{dataset_name}": data_at}) From 39b936a27a1b07a6d86df3bdaaec2608553e86eb Mon Sep 17 00:00:00 2001 From: Ayush Thakur Date: Mon, 22 Nov 2021 13:30:25 +0530 Subject: [PATCH 17/17] minor issue fix --- mmf/utils/logger.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mmf/utils/logger.py b/mmf/utils/logger.py index 36c3101d4..100174c75 100644 --- a/mmf/utils/logger.py +++ b/mmf/utils/logger.py @@ -430,6 +430,7 @@ def __init__( wandb_kwargs.pop("entity") wandb_kwargs.pop("project") wandb_kwargs.pop("log_checkpoint") + wandb_kwargs.pop("log_tables") self._wandb_init.update(**wandb_kwargs) self.setup()