diff --git a/Dockerfile b/Dockerfile
index 5b3592b4..4584b7ed 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,37 +1,51 @@
# YOLOv3 🚀 by Ultralytics, GPL-3.0 license
+# Builds ultralytics/yolov3:latest image on DockerHub https://hub.docker.com/r/ultralytics/yolov3
+# Image is CUDA-optimized for YOLOv3 single/multi-GPU training and inference
-# Start FROM Nvidia PyTorch image https://ngc.nvidia.com/catalog/containers/nvidia:pytorch
-FROM nvcr.io/nvidia/pytorch:21.10-py3
+# Start FROM NVIDIA PyTorch image https://ngc.nvidia.com/catalog/containers/nvidia:pytorch
+# FROM docker.io/pytorch/pytorch:latest
+FROM pytorch/pytorch:latest
+
+# Downloads to user config dir
+ADD https://ultralytics.com/assets/Arial.ttf https://ultralytics.com/assets/Arial.Unicode.ttf /root/.config/Ultralytics/
# Install linux packages
-RUN apt update && apt install -y zip htop screen libgl1-mesa-glx
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt update
+RUN TZ=Etc/UTC apt install -y tzdata
+RUN apt install --no-install-recommends -y gcc git zip curl htop libgl1-mesa-glx libglib2.0-0 libpython3-dev gnupg
+# RUN alias python=python3
-# Install python dependencies
-COPY requirements.txt .
-RUN python -m pip install --upgrade pip
-RUN pip uninstall -y nvidia-tensorboard nvidia-tensorboard-plugin-dlprof
-RUN pip install --no-cache -r requirements.txt coremltools onnx gsutil notebook
-RUN pip install --no-cache -U torch torchvision numpy Pillow
-# RUN pip install --no-cache torch==1.10.0+cu113 torchvision==0.11.1+cu113 -f https://download.pytorch.org/whl/cu113/torch_stable.html
+# Security updates
+# https://security.snyk.io/vuln/SNYK-UBUNTU1804-OPENSSL-3314796
+RUN apt upgrade --no-install-recommends -y openssl
# Create working directory
-RUN mkdir -p /usr/src/app
+RUN rm -rf /usr/src/app && mkdir -p /usr/src/app
WORKDIR /usr/src/app
# Copy contents
-COPY . /usr/src/app
+# COPY . /usr/src/app (issues as not a .git directory)
+RUN git clone https://github.com/ultralytics/yolov3 /usr/src/app
-# Downloads to user config dir
-ADD https://ultralytics.com/assets/Arial.ttf /root/.config/Ultralytics/
+# Install pip packages
+COPY requirements.txt .
+RUN python3 -m pip install --upgrade pip wheel
+RUN pip install --no-cache -r requirements.txt albumentations comet gsutil notebook \
+ coremltools onnx onnx-simplifier onnxruntime 'openvino-dev>=2022.3'
+ # tensorflow tensorflowjs \
# Set environment variables
-# ENV HOME=/usr/src/app
+ENV OMP_NUM_THREADS=1
+
+# Cleanup
+ENV DEBIAN_FRONTEND teletype
# Usage Examples -------------------------------------------------------------------------------------------------------
# Build and Push
-# t=ultralytics/yolov3:latest && sudo docker build -t $t . && sudo docker push $t
+# t=ultralytics/yolov3:latest && sudo docker build -f utils/docker/Dockerfile -t $t . && sudo docker push $t
# Pull and Run
# t=ultralytics/yolov3:latest && sudo docker pull $t && sudo docker run -it --ipc=host --gpus all $t
@@ -45,17 +59,17 @@ ADD https://ultralytics.com/assets/Arial.ttf /root/.config/Ultralytics/
# Kill all image-based
# sudo docker kill $(sudo docker ps -qa --filter ancestor=ultralytics/yolov3:latest)
-# Bash into running container
-# sudo docker exec -it 5a9b5863d93d bash
-
-# Bash into stopped container
-# id=$(sudo docker ps -qa) && sudo docker start $id && sudo docker exec -it $id bash
+# DockerHub tag update
+# t=ultralytics/yolov3:latest tnew=ultralytics/yolov3:v6.2 && sudo docker pull $t && sudo docker tag $t $tnew && sudo docker push $tnew
# Clean up
-# docker system prune -a --volumes
+# sudo docker system prune -a --volumes
# Update Ubuntu drivers
# https://www.maketecheasier.com/install-nvidia-drivers-ubuntu/
# DDP test
# python -m torch.distributed.run --nproc_per_node 2 --master_port 1 train.py --epochs 3
+
+# GCP VM from Image
+# docker.io/ultralytics/yolov3:latest
diff --git a/utils/loggers/wandb/README.md b/utils/loggers/wandb/README.md
deleted file mode 100644
index bae57bda..00000000
--- a/utils/loggers/wandb/README.md
+++ /dev/null
@@ -1,147 +0,0 @@
-📚 This guide explains how to use **Weights & Biases** (W&B) with YOLOv3 🚀. UPDATED 29 September 2021.
-* [About Weights & Biases](#about-weights-&-biases)
-* [First-Time Setup](#first-time-setup)
-* [Viewing runs](#viewing-runs)
-* [Advanced Usage: Dataset Versioning and Evaluation](#advanced-usage)
-* [Reports: Share your work with the world!](#reports)
-
-## About Weights & Biases
-Think of [W&B](https://wandb.ai/site?utm_campaign=repo_yolo_wandbtutorial) like GitHub for machine learning models. With a few lines of code, save everything you need to debug, compare and reproduce your models — architecture, hyperparameters, git commits, model weights, GPU usage, and even datasets and predictions.
-
-Used by top researchers including teams at OpenAI, Lyft, Github, and MILA, W&B is part of the new standard of best practices for machine learning. How W&B can help you optimize your machine learning workflows:
-
- * [Debug](https://wandb.ai/wandb/getting-started/reports/Visualize-Debug-Machine-Learning-Models--VmlldzoyNzY5MDk#Free-2) model performance in real time
- * [GPU usage](https://wandb.ai/wandb/getting-started/reports/Visualize-Debug-Machine-Learning-Models--VmlldzoyNzY5MDk#System-4) visualized automatically
- * [Custom charts](https://wandb.ai/wandb/customizable-charts/reports/Powerful-Custom-Charts-To-Debug-Model-Peformance--VmlldzoyNzY4ODI) for powerful, extensible visualization
- * [Share insights](https://wandb.ai/wandb/getting-started/reports/Visualize-Debug-Machine-Learning-Models--VmlldzoyNzY5MDk#Share-8) interactively with collaborators
- * [Optimize hyperparameters](https://docs.wandb.com/sweeps) efficiently
- * [Track](https://docs.wandb.com/artifacts) datasets, pipelines, and production models
-
-## First-Time Setup
-
- Toggle Details
-When you first train, W&B will prompt you to create a new account and will generate an **API key** for you. If you are an existing user you can retrieve your key from https://wandb.ai/authorize. This key is used to tell W&B where to log your data. You only need to supply your key once, and then it is remembered on the same device.
-
-W&B will create a cloud **project** (default is 'YOLOv3') for your training runs, and each new training run will be provided a unique run **name** within that project as project/name. You can also manually set your project and run name as:
-
- ```shell
- $ python train.py --project ... --name ...
- ```
-
-YOLOv3 notebook example: 
-
-
-
-
-
-## Viewing Runs
-
- Toggle Details
-Run information streams from your environment to the W&B cloud console as you train. This allows you to monitor and even cancel runs in realtime . All important information is logged:
-
- * Training & Validation losses
- * Metrics: Precision, Recall, mAP@0.5, mAP@0.5:0.95
- * Learning Rate over time
- * A bounding box debugging panel, showing the training progress over time
- * GPU: Type, **GPU Utilization**, power, temperature, **CUDA memory usage**
- * System: Disk I/0, CPU utilization, RAM memory usage
- * Your trained model as W&B Artifact
- * Environment: OS and Python types, Git repository and state, **training command**
-
-
-
-
-
-
-## Advanced Usage
-You can leverage W&B artifacts and Tables integration to easily visualize and manage your datasets, models and training evaluations. Here are some quick examples to get you started.
-
- 1. Visualize and Version Datasets
- Log, visualize, dynamically query, and understand your data with W&B Tables. You can use the following command to log your dataset as a W&B Table. This will generate a {dataset}_wandb.yaml file which can be used to train from dataset artifact.
-
- Usage
- Code $ python utils/logger/wandb/log_dataset.py --project ... --name ... --data ..
-
- 
-
-
- 2: Train and Log Evaluation simultaneousy
- This is an extension of the previous section, but it'll also training after uploading the dataset. This also evaluation Table
- Evaluation table compares your predictions and ground truths across the validation set for each epoch. It uses the references to the already uploaded datasets,
- so no images will be uploaded from your system more than once.
-
- Usage
- Code $ python utils/logger/wandb/log_dataset.py --data .. --upload_data
-
-
-
-
- 3: Train using dataset artifact
- When you upload a dataset as described in the first section, you get a new config file with an added `_wandb` to its name. This file contains the information that
- can be used to train a model directly from the dataset artifact. This also logs evaluation
-
- Usage
- Code $ python utils/logger/wandb/log_dataset.py --data {data}_wandb.yaml
-
-
-
-
- 4: Save model checkpoints as artifacts
- To enable saving and versioning checkpoints of your experiment, pass `--save_period n` with the base cammand, where `n` represents checkpoint interval.
- You can also log both the dataset and model checkpoints simultaneously. If not passed, only the final model will be logged
-
-
- Usage
- Code $ python train.py --save_period 1
-
-
-
-
-
-
- 5: Resume runs from checkpoint artifacts.
-Any run can be resumed using artifacts if the --resume argument starts with wandb-artifact:// prefix followed by the run path, i.e, wandb-artifact://username/project/runid . This doesn't require the model checkpoint to be present on the local system.
-
-
- Usage
- Code $ python train.py --resume wandb-artifact://{run_path}
-
-
-
-
- 6: Resume runs from dataset artifact & checkpoint artifacts.
- Local dataset or model checkpoints are not required. This can be used to resume runs directly on a different device
- The syntax is same as the previous section, but you'll need to lof both the dataset and model checkpoints as artifacts, i.e, set bot --upload_dataset or
- train from _wandb.yaml file and set --save_period
-
-
- Usage
- Code $ python train.py --resume wandb-artifact://{run_path}
-
-
-
-
-
-
-
- Reports
-W&B Reports can be created from your saved runs for sharing online. Once a report is created you will receive a link you can use to publically share your results. Here is an example report created from the COCO128 tutorial trainings of all YOLOv5 models ([link](https://wandb.ai/glenn-jocher/yolov5_tutorial/reports/YOLOv5-COCO128-Tutorial-Results--VmlldzozMDI5OTY)).
-
-
-
-
-## Environments
-
-YOLOv3 may be run in any of the following up-to-date verified environments (with all dependencies including [CUDA](https://developer.nvidia.com/cuda)/[CUDNN](https://developer.nvidia.com/cudnn), [Python](https://www.python.org/) and [PyTorch](https://pytorch.org/) preinstalled):
-
-- **Google Colab and Kaggle** notebooks with free GPU:
-- **Google Cloud** Deep Learning VM. See [GCP Quickstart Guide](https://github.com/ultralytics/yolov3/wiki/GCP-Quickstart)
-- **Amazon** Deep Learning AMI. See [AWS Quickstart Guide](https://github.com/ultralytics/yolov3/wiki/AWS-Quickstart)
-- **Docker Image**. See [Docker Quickstart Guide](https://github.com/ultralytics/yolov3/wiki/Docker-Quickstart) 
-
-
-## Status
-
-
-
-If this badge is green, all [YOLOv3 GitHub Actions](https://github.com/ultralytics/yolov3/actions) Continuous Integration (CI) tests are currently passing. CI tests verify correct operation of YOLOv3 training ([train.py](https://github.com/ultralytics/yolov3/blob/master/train.py)), validation ([val.py](https://github.com/ultralytics/yolov3/blob/master/val.py)), inference ([detect.py](https://github.com/ultralytics/yolov3/blob/master/detect.py)) and export ([export.py](https://github.com/ultralytics/yolov3/blob/master/export.py)) on MacOS, Windows, and Ubuntu every 24 hours and on every commit.
diff --git a/utils/loggers/wandb/__init__.py b/utils/loggers/wandb/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/utils/loggers/wandb/log_dataset.py b/utils/loggers/wandb/log_dataset.py
deleted file mode 100644
index d3c77430..00000000
--- a/utils/loggers/wandb/log_dataset.py
+++ /dev/null
@@ -1,27 +0,0 @@
-import argparse
-
-from wandb_utils import WandbLogger
-
-from utils.general import LOGGER
-
-WANDB_ARTIFACT_PREFIX = 'wandb-artifact://'
-
-
-def create_dataset_artifact(opt):
- logger = WandbLogger(opt, None, job_type='Dataset Creation') # TODO: return value unused
- if not logger.wandb:
- LOGGER.info("install wandb using `pip install wandb` to log the dataset")
-
-
-if __name__ == '__main__':
- parser = argparse.ArgumentParser()
- parser.add_argument('--data', type=str, default='data/coco128.yaml', help='data.yaml path')
- parser.add_argument('--single-cls', action='store_true', help='train as single-class dataset')
- parser.add_argument('--project', type=str, default='YOLOv3', help='name of W&B Project')
- parser.add_argument('--entity', default=None, help='W&B entity')
- parser.add_argument('--name', type=str, default='log dataset', help='name of W&B run')
-
- opt = parser.parse_args()
- opt.resume = False # Explicitly disallow resume check for dataset upload job
-
- create_dataset_artifact(opt)
diff --git a/utils/loggers/wandb/sweep.py b/utils/loggers/wandb/sweep.py
deleted file mode 100644
index 5e24f96e..00000000
--- a/utils/loggers/wandb/sweep.py
+++ /dev/null
@@ -1,41 +0,0 @@
-import sys
-from pathlib import Path
-
-import wandb
-
-FILE = Path(__file__).resolve()
-ROOT = FILE.parents[3] # root directory
-if str(ROOT) not in sys.path:
- sys.path.append(str(ROOT)) # add ROOT to PATH
-
-from train import parse_opt, train
-from utils.callbacks import Callbacks
-from utils.general import increment_path
-from utils.torch_utils import select_device
-
-
-def sweep():
- wandb.init()
- # Get hyp dict from sweep agent
- hyp_dict = vars(wandb.config).get("_items")
-
- # Workaround: get necessary opt args
- opt = parse_opt(known=True)
- opt.batch_size = hyp_dict.get("batch_size")
- opt.save_dir = str(increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok or opt.evolve))
- opt.epochs = hyp_dict.get("epochs")
- opt.nosave = True
- opt.data = hyp_dict.get("data")
- opt.weights = str(opt.weights)
- opt.cfg = str(opt.cfg)
- opt.data = str(opt.data)
- opt.hyp = str(opt.hyp)
- opt.project = str(opt.project)
- device = select_device(opt.device, batch_size=opt.batch_size)
-
- # train
- train(hyp_dict, opt, device, callbacks=Callbacks())
-
-
-if __name__ == "__main__":
- sweep()
diff --git a/utils/loggers/wandb/sweep.yaml b/utils/loggers/wandb/sweep.yaml
deleted file mode 100644
index c7790d75..00000000
--- a/utils/loggers/wandb/sweep.yaml
+++ /dev/null
@@ -1,143 +0,0 @@
-# Hyperparameters for training
-# To set range-
-# Provide min and max values as:
-# parameter:
-#
-# min: scalar
-# max: scalar
-# OR
-#
-# Set a specific list of search space-
-# parameter:
-# values: [scalar1, scalar2, scalar3...]
-#
-# You can use grid, bayesian and hyperopt search strategy
-# For more info on configuring sweeps visit - https://docs.wandb.ai/guides/sweeps/configuration
-
-program: utils/loggers/wandb/sweep.py
-method: random
-metric:
- name: metrics/mAP_0.5
- goal: maximize
-
-parameters:
- # hyperparameters: set either min, max range or values list
- data:
- value: "data/coco128.yaml"
- batch_size:
- values: [64]
- epochs:
- values: [10]
-
- lr0:
- distribution: uniform
- min: 1e-5
- max: 1e-1
- lrf:
- distribution: uniform
- min: 0.01
- max: 1.0
- momentum:
- distribution: uniform
- min: 0.6
- max: 0.98
- weight_decay:
- distribution: uniform
- min: 0.0
- max: 0.001
- warmup_epochs:
- distribution: uniform
- min: 0.0
- max: 5.0
- warmup_momentum:
- distribution: uniform
- min: 0.0
- max: 0.95
- warmup_bias_lr:
- distribution: uniform
- min: 0.0
- max: 0.2
- box:
- distribution: uniform
- min: 0.02
- max: 0.2
- cls:
- distribution: uniform
- min: 0.2
- max: 4.0
- cls_pw:
- distribution: uniform
- min: 0.5
- max: 2.0
- obj:
- distribution: uniform
- min: 0.2
- max: 4.0
- obj_pw:
- distribution: uniform
- min: 0.5
- max: 2.0
- iou_t:
- distribution: uniform
- min: 0.1
- max: 0.7
- anchor_t:
- distribution: uniform
- min: 2.0
- max: 8.0
- fl_gamma:
- distribution: uniform
- min: 0.0
- max: 0.1
- hsv_h:
- distribution: uniform
- min: 0.0
- max: 0.1
- hsv_s:
- distribution: uniform
- min: 0.0
- max: 0.9
- hsv_v:
- distribution: uniform
- min: 0.0
- max: 0.9
- degrees:
- distribution: uniform
- min: 0.0
- max: 45.0
- translate:
- distribution: uniform
- min: 0.0
- max: 0.9
- scale:
- distribution: uniform
- min: 0.0
- max: 0.9
- shear:
- distribution: uniform
- min: 0.0
- max: 10.0
- perspective:
- distribution: uniform
- min: 0.0
- max: 0.001
- flipud:
- distribution: uniform
- min: 0.0
- max: 1.0
- fliplr:
- distribution: uniform
- min: 0.0
- max: 1.0
- mosaic:
- distribution: uniform
- min: 0.0
- max: 1.0
- mixup:
- distribution: uniform
- min: 0.0
- max: 1.0
- copy_paste:
- distribution: uniform
- min: 0.0
- max: 1.0
diff --git a/utils/loggers/wandb/wandb_utils.py b/utils/loggers/wandb/wandb_utils.py
deleted file mode 100644
index 7087e4e9..00000000
--- a/utils/loggers/wandb/wandb_utils.py
+++ /dev/null
@@ -1,532 +0,0 @@
-"""Utilities and tools for tracking runs with Weights & Biases."""
-
-import logging
-import os
-import sys
-from contextlib import contextmanager
-from pathlib import Path
-from typing import Dict
-
-import pkg_resources as pkg
-import yaml
-from tqdm import tqdm
-
-FILE = Path(__file__).resolve()
-ROOT = FILE.parents[3] # root directory
-if str(ROOT) not in sys.path:
- sys.path.append(str(ROOT)) # add ROOT to PATH
-
-from utils.datasets import LoadImagesAndLabels, img2label_paths
-from utils.general import LOGGER, check_dataset, check_file
-
-try:
- import wandb
-
- assert hasattr(wandb, '__version__') # verify package import not local dir
-except (ImportError, AssertionError):
- wandb = None
-
-RANK = int(os.getenv('RANK', -1))
-WANDB_ARTIFACT_PREFIX = 'wandb-artifact://'
-
-
-def remove_prefix(from_string, prefix=WANDB_ARTIFACT_PREFIX):
- return from_string[len(prefix):]
-
-
-def check_wandb_config_file(data_config_file):
- wandb_config = '_wandb.'.join(data_config_file.rsplit('.', 1)) # updated data.yaml path
- if Path(wandb_config).is_file():
- return wandb_config
- return data_config_file
-
-
-def check_wandb_dataset(data_file):
- is_trainset_wandb_artifact = False
- is_valset_wandb_artifact = False
- if check_file(data_file) and data_file.endswith('.yaml'):
- with open(data_file, errors='ignore') as f:
- data_dict = yaml.safe_load(f)
- is_trainset_wandb_artifact = (isinstance(data_dict['train'], str) and
- data_dict['train'].startswith(WANDB_ARTIFACT_PREFIX))
- is_valset_wandb_artifact = (isinstance(data_dict['val'], str) and
- data_dict['val'].startswith(WANDB_ARTIFACT_PREFIX))
- if is_trainset_wandb_artifact or is_valset_wandb_artifact:
- return data_dict
- else:
- return check_dataset(data_file)
-
-
-def get_run_info(run_path):
- run_path = Path(remove_prefix(run_path, WANDB_ARTIFACT_PREFIX))
- run_id = run_path.stem
- project = run_path.parent.stem
- entity = run_path.parent.parent.stem
- model_artifact_name = 'run_' + run_id + '_model'
- return entity, project, run_id, model_artifact_name
-
-
-def check_wandb_resume(opt):
- process_wandb_config_ddp_mode(opt) if RANK not in [-1, 0] else None
- if isinstance(opt.resume, str):
- if opt.resume.startswith(WANDB_ARTIFACT_PREFIX):
- if RANK not in [-1, 0]: # For resuming DDP runs
- entity, project, run_id, model_artifact_name = get_run_info(opt.resume)
- api = wandb.Api()
- artifact = api.artifact(entity + '/' + project + '/' + model_artifact_name + ':latest')
- modeldir = artifact.download()
- opt.weights = str(Path(modeldir) / "last.pt")
- return True
- return None
-
-
-def process_wandb_config_ddp_mode(opt):
- with open(check_file(opt.data), errors='ignore') as f:
- data_dict = yaml.safe_load(f) # data dict
- train_dir, val_dir = None, None
- if isinstance(data_dict['train'], str) and data_dict['train'].startswith(WANDB_ARTIFACT_PREFIX):
- api = wandb.Api()
- train_artifact = api.artifact(remove_prefix(data_dict['train']) + ':' + opt.artifact_alias)
- train_dir = train_artifact.download()
- train_path = Path(train_dir) / 'data/images/'
- data_dict['train'] = str(train_path)
-
- if isinstance(data_dict['val'], str) and data_dict['val'].startswith(WANDB_ARTIFACT_PREFIX):
- api = wandb.Api()
- val_artifact = api.artifact(remove_prefix(data_dict['val']) + ':' + opt.artifact_alias)
- val_dir = val_artifact.download()
- val_path = Path(val_dir) / 'data/images/'
- data_dict['val'] = str(val_path)
- if train_dir or val_dir:
- ddp_data_path = str(Path(val_dir) / 'wandb_local_data.yaml')
- with open(ddp_data_path, 'w') as f:
- yaml.safe_dump(data_dict, f)
- opt.data = ddp_data_path
-
-
-class WandbLogger():
- """Log training runs, datasets, models, and predictions to Weights & Biases.
-
- This logger sends information to W&B at wandb.ai. By default, this information
- includes hyperparameters, system configuration and metrics, model metrics,
- and basic data metrics and analyses.
-
- By providing additional command line arguments to train.py, datasets,
- models and predictions can also be logged.
-
- For more on how this logger is used, see the Weights & Biases documentation:
- https://docs.wandb.com/guides/integrations/yolov5
- """
-
- def __init__(self, opt, run_id=None, job_type='Training'):
- """
- - Initialize WandbLogger instance
- - Upload dataset if opt.upload_dataset is True
- - Setup trainig processes if job_type is 'Training'
-
- arguments:
- opt (namespace) -- Commandline arguments for this run
- run_id (str) -- Run ID of W&B run to be resumed
- job_type (str) -- To set the job_type for this run
-
- """
- # Pre-training routine --
- self.job_type = job_type
- self.wandb, self.wandb_run = wandb, None if not wandb else wandb.run
- self.val_artifact, self.train_artifact = None, None
- self.train_artifact_path, self.val_artifact_path = None, None
- self.result_artifact = None
- self.val_table, self.result_table = None, None
- self.bbox_media_panel_images = []
- self.val_table_path_map = None
- self.max_imgs_to_log = 16
- self.wandb_artifact_data_dict = None
- self.data_dict = None
- # It's more elegant to stick to 1 wandb.init call,
- # but useful config data is overwritten in the WandbLogger's wandb.init call
- if isinstance(opt.resume, str): # checks resume from artifact
- if opt.resume.startswith(WANDB_ARTIFACT_PREFIX):
- entity, project, run_id, model_artifact_name = get_run_info(opt.resume)
- model_artifact_name = WANDB_ARTIFACT_PREFIX + model_artifact_name
- assert wandb, 'install wandb to resume wandb runs'
- # Resume wandb-artifact:// runs here| workaround for not overwriting wandb.config
- self.wandb_run = wandb.init(id=run_id,
- project=project,
- entity=entity,
- resume='allow',
- allow_val_change=True)
- opt.resume = model_artifact_name
- elif self.wandb:
- self.wandb_run = wandb.init(config=opt,
- resume="allow",
- project='YOLOv3' if opt.project == 'runs/train' else Path(opt.project).stem,
- entity=opt.entity,
- name=opt.name if opt.name != 'exp' else None,
- job_type=job_type,
- id=run_id,
- allow_val_change=True) if not wandb.run else wandb.run
- if self.wandb_run:
- if self.job_type == 'Training':
- if opt.upload_dataset:
- if not opt.resume:
- self.wandb_artifact_data_dict = self.check_and_upload_dataset(opt)
-
- if opt.resume:
- # resume from artifact
- if isinstance(opt.resume, str) and opt.resume.startswith(WANDB_ARTIFACT_PREFIX):
- self.data_dict = dict(self.wandb_run.config.data_dict)
- else: # local resume
- self.data_dict = check_wandb_dataset(opt.data)
- else:
- self.data_dict = check_wandb_dataset(opt.data)
- self.wandb_artifact_data_dict = self.wandb_artifact_data_dict or self.data_dict
-
- # write data_dict to config. useful for resuming from artifacts. Do this only when not resuming.
- self.wandb_run.config.update({'data_dict': self.wandb_artifact_data_dict},
- allow_val_change=True)
- self.setup_training(opt)
-
- if self.job_type == 'Dataset Creation':
- self.data_dict = self.check_and_upload_dataset(opt)
-
- def check_and_upload_dataset(self, opt):
- """
- Check if the dataset format is compatible and upload it as W&B artifact
-
- arguments:
- opt (namespace)-- Commandline arguments for current run
-
- returns:
- Updated dataset info dictionary where local dataset paths are replaced by WAND_ARFACT_PREFIX links.
- """
- assert wandb, 'Install wandb to upload dataset'
- config_path = self.log_dataset_artifact(opt.data,
- opt.single_cls,
- 'YOLOv3' if opt.project == 'runs/train' else Path(opt.project).stem)
- LOGGER.info(f"Created dataset config file {config_path}")
- with open(config_path, errors='ignore') as f:
- wandb_data_dict = yaml.safe_load(f)
- return wandb_data_dict
-
- def setup_training(self, opt):
- """
- Setup the necessary processes for training YOLO models:
- - Attempt to download model checkpoint and dataset artifacts if opt.resume stats with WANDB_ARTIFACT_PREFIX
- - Update data_dict, to contain info of previous run if resumed and the paths of dataset artifact if downloaded
- - Setup log_dict, initialize bbox_interval
-
- arguments:
- opt (namespace) -- commandline arguments for this run
-
- """
- self.log_dict, self.current_epoch = {}, 0
- self.bbox_interval = opt.bbox_interval
- if isinstance(opt.resume, str):
- modeldir, _ = self.download_model_artifact(opt)
- if modeldir:
- self.weights = Path(modeldir) / "last.pt"
- config = self.wandb_run.config
- opt.weights, opt.save_period, opt.batch_size, opt.bbox_interval, opt.epochs, opt.hyp = str(
- self.weights), config.save_period, config.batch_size, config.bbox_interval, config.epochs, \
- config.hyp
- data_dict = self.data_dict
- if self.val_artifact is None: # If --upload_dataset is set, use the existing artifact, don't download
- self.train_artifact_path, self.train_artifact = self.download_dataset_artifact(data_dict.get('train'),
- opt.artifact_alias)
- self.val_artifact_path, self.val_artifact = self.download_dataset_artifact(data_dict.get('val'),
- opt.artifact_alias)
-
- if self.train_artifact_path is not None:
- train_path = Path(self.train_artifact_path) / 'data/images/'
- data_dict['train'] = str(train_path)
- if self.val_artifact_path is not None:
- val_path = Path(self.val_artifact_path) / 'data/images/'
- data_dict['val'] = str(val_path)
-
- if self.val_artifact is not None:
- self.result_artifact = wandb.Artifact("run_" + wandb.run.id + "_progress", "evaluation")
- self.result_table = wandb.Table(["epoch", "id", "ground truth", "prediction", "avg_confidence"])
- self.val_table = self.val_artifact.get("val")
- if self.val_table_path_map is None:
- self.map_val_table_path()
- if opt.bbox_interval == -1:
- self.bbox_interval = opt.bbox_interval = (opt.epochs // 10) if opt.epochs > 10 else 1
- train_from_artifact = self.train_artifact_path is not None and self.val_artifact_path is not None
- # Update the the data_dict to point to local artifacts dir
- if train_from_artifact:
- self.data_dict = data_dict
-
- def download_dataset_artifact(self, path, alias):
- """
- download the model checkpoint artifact if the path starts with WANDB_ARTIFACT_PREFIX
-
- arguments:
- path -- path of the dataset to be used for training
- alias (str)-- alias of the artifact to be download/used for training
-
- returns:
- (str, wandb.Artifact) -- path of the downladed dataset and it's corresponding artifact object if dataset
- is found otherwise returns (None, None)
- """
- if isinstance(path, str) and path.startswith(WANDB_ARTIFACT_PREFIX):
- artifact_path = Path(remove_prefix(path, WANDB_ARTIFACT_PREFIX) + ":" + alias)
- dataset_artifact = wandb.use_artifact(artifact_path.as_posix().replace("\\", "/"))
- assert dataset_artifact is not None, "'Error: W&B dataset artifact doesn\'t exist'"
- datadir = dataset_artifact.download()
- return datadir, dataset_artifact
- return None, None
-
- def download_model_artifact(self, opt):
- """
- download the model checkpoint artifact if the resume path starts with WANDB_ARTIFACT_PREFIX
-
- arguments:
- opt (namespace) -- Commandline arguments for this run
- """
- if opt.resume.startswith(WANDB_ARTIFACT_PREFIX):
- model_artifact = wandb.use_artifact(remove_prefix(opt.resume, WANDB_ARTIFACT_PREFIX) + ":latest")
- assert model_artifact is not None, 'Error: W&B model artifact doesn\'t exist'
- modeldir = model_artifact.download()
- epochs_trained = model_artifact.metadata.get('epochs_trained')
- total_epochs = model_artifact.metadata.get('total_epochs')
- is_finished = total_epochs is None
- assert not is_finished, 'training is finished, can only resume incomplete runs.'
- return modeldir, model_artifact
- return None, None
-
- def log_model(self, path, opt, epoch, fitness_score, best_model=False):
- """
- Log the model checkpoint as W&B artifact
-
- arguments:
- path (Path) -- Path of directory containing the checkpoints
- opt (namespace) -- Command line arguments for this run
- epoch (int) -- Current epoch number
- fitness_score (float) -- fitness score for current epoch
- best_model (boolean) -- Boolean representing if the current checkpoint is the best yet.
- """
- model_artifact = wandb.Artifact('run_' + wandb.run.id + '_model', type='model', metadata={
- 'original_url': str(path),
- 'epochs_trained': epoch + 1,
- 'save period': opt.save_period,
- 'project': opt.project,
- 'total_epochs': opt.epochs,
- 'fitness_score': fitness_score
- })
- model_artifact.add_file(str(path / 'last.pt'), name='last.pt')
- wandb.log_artifact(model_artifact,
- aliases=['latest', 'last', 'epoch ' + str(self.current_epoch), 'best' if best_model else ''])
- LOGGER.info(f"Saving model artifact on epoch {epoch + 1}")
-
- def log_dataset_artifact(self, data_file, single_cls, project, overwrite_config=False):
- """
- Log the dataset as W&B artifact and return the new data file with W&B links
-
- arguments:
- data_file (str) -- the .yaml file with information about the dataset like - path, classes etc.
- single_class (boolean) -- train multi-class data as single-class
- project (str) -- project name. Used to construct the artifact path
- overwrite_config (boolean) -- overwrites the data.yaml file if set to true otherwise creates a new
- file with _wandb postfix. Eg -> data_wandb.yaml
-
- returns:
- the new .yaml file with artifact links. it can be used to start training directly from artifacts
- """
- self.data_dict = check_dataset(data_file) # parse and check
- data = dict(self.data_dict)
- nc, names = (1, ['item']) if single_cls else (int(data['nc']), data['names'])
- names = {k: v for k, v in enumerate(names)} # to index dictionary
- self.train_artifact = self.create_dataset_table(LoadImagesAndLabels(
- data['train'], rect=True, batch_size=1), names, name='train') if data.get('train') else None
- self.val_artifact = self.create_dataset_table(LoadImagesAndLabels(
- data['val'], rect=True, batch_size=1), names, name='val') if data.get('val') else None
- if data.get('train'):
- data['train'] = WANDB_ARTIFACT_PREFIX + str(Path(project) / 'train')
- if data.get('val'):
- data['val'] = WANDB_ARTIFACT_PREFIX + str(Path(project) / 'val')
- path = Path(data_file).stem
- path = (path if overwrite_config else path + '_wandb') + '.yaml' # updated data.yaml path
- data.pop('download', None)
- data.pop('path', None)
- with open(path, 'w') as f:
- yaml.safe_dump(data, f)
-
- if self.job_type == 'Training': # builds correct artifact pipeline graph
- self.wandb_run.use_artifact(self.val_artifact)
- self.wandb_run.use_artifact(self.train_artifact)
- self.val_artifact.wait()
- self.val_table = self.val_artifact.get('val')
- self.map_val_table_path()
- else:
- self.wandb_run.log_artifact(self.train_artifact)
- self.wandb_run.log_artifact(self.val_artifact)
- return path
-
- def map_val_table_path(self):
- """
- Map the validation dataset Table like name of file -> it's id in the W&B Table.
- Useful for - referencing artifacts for evaluation.
- """
- self.val_table_path_map = {}
- LOGGER.info("Mapping dataset")
- for i, data in enumerate(tqdm(self.val_table.data)):
- self.val_table_path_map[data[3]] = data[0]
-
- def create_dataset_table(self, dataset: LoadImagesAndLabels, class_to_id: Dict[int,str], name: str = 'dataset'):
- """
- Create and return W&B artifact containing W&B Table of the dataset.
-
- arguments:
- dataset -- instance of LoadImagesAndLabels class used to iterate over the data to build Table
- class_to_id -- hash map that maps class ids to labels
- name -- name of the artifact
-
- returns:
- dataset artifact to be logged or used
- """
- # TODO: Explore multiprocessing to slpit this loop parallely| This is essential for speeding up the the logging
- artifact = wandb.Artifact(name=name, type="dataset")
- img_files = tqdm([dataset.path]) if isinstance(dataset.path, str) and Path(dataset.path).is_dir() else None
- img_files = tqdm(dataset.img_files) if not img_files else img_files
- for img_file in img_files:
- if Path(img_file).is_dir():
- artifact.add_dir(img_file, name='data/images')
- labels_path = 'labels'.join(dataset.path.rsplit('images', 1))
- artifact.add_dir(labels_path, name='data/labels')
- else:
- artifact.add_file(img_file, name='data/images/' + Path(img_file).name)
- label_file = Path(img2label_paths([img_file])[0])
- artifact.add_file(str(label_file),
- name='data/labels/' + label_file.name) if label_file.exists() else None
- table = wandb.Table(columns=["id", "train_image", "Classes", "name"])
- class_set = wandb.Classes([{'id': id, 'name': name} for id, name in class_to_id.items()])
- for si, (img, labels, paths, shapes) in enumerate(tqdm(dataset)):
- box_data, img_classes = [], {}
- for cls, *xywh in labels[:, 1:].tolist():
- cls = int(cls)
- box_data.append({"position": {"middle": [xywh[0], xywh[1]], "width": xywh[2], "height": xywh[3]},
- "class_id": cls,
- "box_caption": "%s" % (class_to_id[cls])})
- img_classes[cls] = class_to_id[cls]
- boxes = {"ground_truth": {"box_data": box_data, "class_labels": class_to_id}} # inference-space
- table.add_data(si, wandb.Image(paths, classes=class_set, boxes=boxes), list(img_classes.values()),
- Path(paths).name)
- artifact.add(table, name)
- return artifact
-
- def log_training_progress(self, predn, path, names):
- """
- Build evaluation Table. Uses reference from validation dataset table.
-
- arguments:
- predn (list): list of predictions in the native space in the format - [xmin, ymin, xmax, ymax, confidence, class]
- path (str): local path of the current evaluation image
- names (dict(int, str)): hash map that maps class ids to labels
- """
- class_set = wandb.Classes([{'id': id, 'name': name} for id, name in names.items()])
- box_data = []
- total_conf = 0
- for *xyxy, conf, cls in predn.tolist():
- if conf >= 0.25:
- box_data.append(
- {"position": {"minX": xyxy[0], "minY": xyxy[1], "maxX": xyxy[2], "maxY": xyxy[3]},
- "class_id": int(cls),
- "box_caption": f"{names[cls]} {conf:.3f}",
- "scores": {"class_score": conf},
- "domain": "pixel"})
- total_conf += conf
- boxes = {"predictions": {"box_data": box_data, "class_labels": names}} # inference-space
- id = self.val_table_path_map[Path(path).name]
- self.result_table.add_data(self.current_epoch,
- id,
- self.val_table.data[id][1],
- wandb.Image(self.val_table.data[id][1], boxes=boxes, classes=class_set),
- total_conf / max(1, len(box_data))
- )
-
- def val_one_image(self, pred, predn, path, names, im):
- """
- Log validation data for one image. updates the result Table if validation dataset is uploaded and log bbox media panel
-
- arguments:
- pred (list): list of scaled predictions in the format - [xmin, ymin, xmax, ymax, confidence, class]
- predn (list): list of predictions in the native space - [xmin, ymin, xmax, ymax, confidence, class]
- path (str): local path of the current evaluation image
- """
- if self.val_table and self.result_table: # Log Table if Val dataset is uploaded as artifact
- self.log_training_progress(predn, path, names)
-
- if len(self.bbox_media_panel_images) < self.max_imgs_to_log and self.current_epoch > 0:
- if self.current_epoch % self.bbox_interval == 0:
- box_data = [{"position": {"minX": xyxy[0], "minY": xyxy[1], "maxX": xyxy[2], "maxY": xyxy[3]},
- "class_id": int(cls),
- "box_caption": f"{names[cls]} {conf:.3f}",
- "scores": {"class_score": conf},
- "domain": "pixel"} for *xyxy, conf, cls in pred.tolist()]
- boxes = {"predictions": {"box_data": box_data, "class_labels": names}} # inference-space
- self.bbox_media_panel_images.append(wandb.Image(im, boxes=boxes, caption=path.name))
-
- def log(self, log_dict):
- """
- save the metrics to the logging dictionary
-
- arguments:
- log_dict (Dict) -- metrics/media to be logged in current step
- """
- if self.wandb_run:
- for key, value in log_dict.items():
- self.log_dict[key] = value
-
- def end_epoch(self, best_result=False):
- """
- commit the log_dict, model artifacts and Tables to W&B and flush the log_dict.
-
- arguments:
- best_result (boolean): Boolean representing if the result of this evaluation is best or not
- """
- if self.wandb_run:
- with all_logging_disabled():
- if self.bbox_media_panel_images:
- self.log_dict["BoundingBoxDebugger"] = self.bbox_media_panel_images
- try:
- wandb.log(self.log_dict)
- except BaseException as e:
- LOGGER.info(f"An error occurred in wandb logger. The training will proceed without interruption. More info\n{e}")
- self.wandb_run.finish()
- self.wandb_run = None
-
- self.log_dict = {}
- self.bbox_media_panel_images = []
- if self.result_artifact:
- self.result_artifact.add(self.result_table, 'result')
- wandb.log_artifact(self.result_artifact, aliases=['latest', 'last', 'epoch ' + str(self.current_epoch),
- ('best' if best_result else '')])
-
- wandb.log({"evaluation": self.result_table})
- self.result_table = wandb.Table(["epoch", "id", "ground truth", "prediction", "avg_confidence"])
- self.result_artifact = wandb.Artifact("run_" + wandb.run.id + "_progress", "evaluation")
-
- def finish_run(self):
- """
- Log metrics if any and finish the current W&B run
- """
- if self.wandb_run:
- if self.log_dict:
- with all_logging_disabled():
- wandb.log(self.log_dict)
- wandb.run.finish()
-
-
-@contextmanager
-def all_logging_disabled(highest_level=logging.CRITICAL):
- """ source - https://gist.github.com/simon-weber/7853144
- A context manager that will prevent any logging messages triggered during the body from being processed.
- :param highest_level: the maximum logging level in use.
- This would only need to be changed if a custom level greater than CRITICAL is defined.
- """
- previous_level = logging.root.manager.disable
- logging.disable(highest_level)
- try:
- yield
- finally:
- logging.disable(previous_level)