Skip to content

Commit

Permalink
ultralytics 8.0.158 add benchmarks to coverage (ultralytics#4432)
Browse files Browse the repository at this point in the history
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Yonghye Kwon <[email protected]>
  • Loading branch information
3 people authored Aug 20, 2023
1 parent 4958065 commit 87ce15d
Showing 51 changed files with 352 additions and 482 deletions.
27 changes: 17 additions & 10 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
@@ -106,11 +106,7 @@ jobs:
shell: bash # for Windows compatibility
run: |
python -m pip install --upgrade pip wheel
if [ "${{ matrix.os }}" == "macos-latest" ]; then
pip install -e ".[export]" --extra-index-url https://download.pytorch.org/whl/cpu
else
pip install -e ".[export]" --extra-index-url https://download.pytorch.org/whl/cpu
fi
pip install -e ".[export]" coverage --extra-index-url https://download.pytorch.org/whl/cpu
yolo export format=tflite imgsz=32 || true
- name: Check environment
run: |
@@ -125,16 +121,25 @@ jobs:
pip list
- name: Benchmark DetectionModel
shell: bash
run: yolo benchmark model='path with spaces/${{ matrix.model }}.pt' imgsz=160 verbose=0.26
run: coverage run -a --source=ultralytics -m ultralytics.cfg.__init__ benchmark model='path with spaces/${{ matrix.model }}.pt' imgsz=160 verbose=0.26
- name: Benchmark SegmentationModel
shell: bash
run: yolo benchmark model='path with spaces/${{ matrix.model }}-seg.pt' imgsz=160 verbose=0.30
run: coverage run -a --source=ultralytics -m ultralytics.cfg.__init__ benchmark model='path with spaces/${{ matrix.model }}-seg.pt' imgsz=160 verbose=0.30
- name: Benchmark ClassificationModel
shell: bash
run: yolo benchmark model='path with spaces/${{ matrix.model }}-cls.pt' imgsz=160 verbose=0.36
run: coverage run -a --source=ultralytics -m ultralytics.cfg.__init__ benchmark model='path with spaces/${{ matrix.model }}-cls.pt' imgsz=160 verbose=0.36
- name: Benchmark PoseModel
shell: bash
run: yolo benchmark model='path with spaces/${{ matrix.model }}-pose.pt' imgsz=160 verbose=0.17
run: coverage run -a --source=ultralytics -m ultralytics.cfg.__init__ benchmark model='path with spaces/${{ matrix.model }}-pose.pt' imgsz=160 verbose=0.17
- name: Merge Coverage Reports
run: |
coverage xml -o coverage-benchmarks.xml
- name: Upload Coverage Reports to CodeCov
uses: codecov/codecov-action@v3
with:
flags: Benchmarks
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
- name: Benchmark Summary
run: |
cat benchmarks.log
@@ -183,9 +188,11 @@ jobs:
- name: Pytest tests
shell: bash # for Windows compatibility
run: pytest --cov=ultralytics/ --cov-report xml tests/
- name: Upload coverage reports to Codecov
- name: Upload Coverage Reports to CodeCov
if: github.repository == 'ultralytics/ultralytics' && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11'
uses: codecov/codecov-action@v3
with:
flags: Tests
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -118,6 +118,9 @@ venv.bak/
.spyderproject
.spyproject

# VSCode project settings
.vscode/

# Rope project settings
.ropeproject

10 changes: 3 additions & 7 deletions docs/reference/engine/exporter.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
---
description: Explore the exporter functionality of Ultralytics. Learn about exporting formats, iOSDetectModel, and try exporting with examples.
keywords: Ultralytics, Exporter, iOSDetectModel, Export Formats, Try export
description: Explore the exporter functionality of Ultralytics. Learn about exporting formats, IOSDetectModel, and try exporting with examples.
keywords: Ultralytics, Exporter, IOSDetectModel, Export Formats, Try export
---

# Reference for `ultralytics/engine/exporter.py`
@@ -14,7 +14,7 @@ keywords: Ultralytics, Exporter, iOSDetectModel, Export Formats, Try export
<br><br>

---
## ::: ultralytics.engine.exporter.iOSDetectModel
## ::: ultralytics.engine.exporter.IOSDetectModel
<br><br>

---
@@ -28,7 +28,3 @@ keywords: Ultralytics, Exporter, iOSDetectModel, Export Formats, Try export
---
## ::: ultralytics.engine.exporter.try_export
<br><br>

---
## ::: ultralytics.engine.exporter.export
<br><br>
4 changes: 0 additions & 4 deletions docs/reference/models/rtdetr/train.md
Original file line number Diff line number Diff line change
@@ -12,7 +12,3 @@ keywords: Ultralytics, RTDETRTrainer, model training, Ultralytics models, PyTorc
---
## ::: ultralytics.models.rtdetr.train.RTDETRTrainer
<br><br>

---
## ::: ultralytics.models.rtdetr.train.train
<br><br>
4 changes: 0 additions & 4 deletions docs/reference/models/yolo/classify/predict.md
Original file line number Diff line number Diff line change
@@ -12,7 +12,3 @@ keywords: Ultralytics, classification predictor, predict, YOLO, AI models, model
---
## ::: ultralytics.models.yolo.classify.predict.ClassificationPredictor
<br><br>

---
## ::: ultralytics.models.yolo.classify.predict.predict
<br><br>
4 changes: 0 additions & 4 deletions docs/reference/models/yolo/classify/train.md
Original file line number Diff line number Diff line change
@@ -12,7 +12,3 @@ keywords: Ultralytics, YOLO, Classification Trainer, deep learning, training pro
---
## ::: ultralytics.models.yolo.classify.train.ClassificationTrainer
<br><br>

---
## ::: ultralytics.models.yolo.classify.train.train
<br><br>
4 changes: 0 additions & 4 deletions docs/reference/models/yolo/classify/val.md
Original file line number Diff line number Diff line change
@@ -12,7 +12,3 @@ keywords: Ultralytics, YOLO, ClassificationValidator, model validation, model fi
---
## ::: ultralytics.models.yolo.classify.val.ClassificationValidator
<br><br>

---
## ::: ultralytics.models.yolo.classify.val.val
<br><br>
4 changes: 0 additions & 4 deletions docs/reference/models/yolo/detect/predict.md
Original file line number Diff line number Diff line change
@@ -12,7 +12,3 @@ keywords: Ultralytics, YOLO, DetectionPredictor, detect, predict, object detecti
---
## ::: ultralytics.models.yolo.detect.predict.DetectionPredictor
<br><br>

---
## ::: ultralytics.models.yolo.detect.predict.predict
<br><br>
4 changes: 0 additions & 4 deletions docs/reference/models/yolo/detect/train.md
Original file line number Diff line number Diff line change
@@ -12,7 +12,3 @@ keywords: Ultralytics YOLO, YOLO, Detection Trainer, Model Training, Machine Lea
---
## ::: ultralytics.models.yolo.detect.train.DetectionTrainer
<br><br>

---
## ::: ultralytics.models.yolo.detect.train.train
<br><br>
4 changes: 0 additions & 4 deletions docs/reference/models/yolo/detect/val.md
Original file line number Diff line number Diff line change
@@ -12,7 +12,3 @@ keywords: Ultralytics, YOLO, Detection Validator, model valuation, precision, re
---
## ::: ultralytics.models.yolo.detect.val.DetectionValidator
<br><br>

---
## ::: ultralytics.models.yolo.detect.val.val
<br><br>
4 changes: 0 additions & 4 deletions docs/reference/models/yolo/pose/predict.md
Original file line number Diff line number Diff line change
@@ -12,7 +12,3 @@ keywords: Ultralytics, YOLO, PosePredictor, machine learning, AI, predictive mod
---
## ::: ultralytics.models.yolo.pose.predict.PosePredictor
<br><br>

---
## ::: ultralytics.models.yolo.pose.predict.predict
<br><br>
4 changes: 0 additions & 4 deletions docs/reference/models/yolo/pose/train.md
Original file line number Diff line number Diff line change
@@ -12,7 +12,3 @@ keywords: Ultralytics, YOLO, PoseTrainer, pose training, AI modeling, custom dat
---
## ::: ultralytics.models.yolo.pose.train.PoseTrainer
<br><br>

---
## ::: ultralytics.models.yolo.pose.train.train
<br><br>
4 changes: 0 additions & 4 deletions docs/reference/models/yolo/pose/val.md
Original file line number Diff line number Diff line change
@@ -12,7 +12,3 @@ keywords: PoseValidator, Ultralytics, YOLO, Object detection, Pose validation
---
## ::: ultralytics.models.yolo.pose.val.PoseValidator
<br><br>

---
## ::: ultralytics.models.yolo.pose.val.val
<br><br>
4 changes: 0 additions & 4 deletions docs/reference/models/yolo/segment/predict.md
Original file line number Diff line number Diff line change
@@ -12,7 +12,3 @@ keywords: YOLO, Ultralytics, object detection, segmentation predictor
---
## ::: ultralytics.models.yolo.segment.predict.SegmentationPredictor
<br><br>

---
## ::: ultralytics.models.yolo.segment.predict.predict
<br><br>
2 changes: 1 addition & 1 deletion docs/usage/engine.md
Original file line number Diff line number Diff line change
@@ -14,7 +14,7 @@ the required functions or operations as long the as correct formats are followed
custom model and dataloader by just overriding these functions:

* `get_model(cfg, weights)` - The function that builds the model to be trained
* `get_dataloder()` - The function that builds the dataloader
* `get_dataloader()` - The function that builds the dataloader
More details and source code can be found in [`BaseTrainer` Reference](../reference/engine/trainer.md)

## DetectionTrainer
1 change: 1 addition & 0 deletions mkdocs.yml
Original file line number Diff line number Diff line change
@@ -401,6 +401,7 @@ plugins:
handlers:
python:
options:
docstring_style: google
show_root_heading: true
show_source: true
- ultralytics:
2 changes: 1 addition & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Ultralytics requirements
# Usage: pip install -r requirements.txt
# Example: pip install -r requirements.txt

# Base ----------------------------------------
matplotlib>=3.2.2
8 changes: 8 additions & 0 deletions tests/test_cli.py
Original file line number Diff line number Diff line change
@@ -40,6 +40,14 @@ def test_train(task, model, data):

@pytest.mark.parametrize('task,model,data', TASK_ARGS)
def test_val(task, model, data):
# Download annotations to run pycocotools eval
# from ultralytics.utils import SETTINGS, Path
# from ultralytics.utils.downloads import download
# url = 'https://github.com/ultralytics/assets/releases/download/v0.0.0/'
# download(f'{url}instances_val2017.json', dir=Path(SETTINGS['datasets_dir']) / 'coco8/annotations')
# download(f'{url}person_keypoints_val2017.json', dir=Path(SETTINGS['datasets_dir']) / 'coco8-pose/annotations')

# Validate
run(f'yolo val {task} model={WEIGHTS_DIR / model}.pt data={data} imgsz=32 save_txt save_json')


15 changes: 12 additions & 3 deletions tests/test_python.py
Original file line number Diff line number Diff line change
@@ -132,13 +132,13 @@ def test_val():

def test_train_scratch():
model = YOLO(CFG)
model.train(data='coco8.yaml', epochs=1, imgsz=32, cache='disk', batch=-1) # test disk caching with AutoBatch
model.train(data='coco8.yaml', epochs=2, imgsz=32, cache='disk', batch=-1, close_mosaic=1)
model(SOURCE)


def test_train_pretrained():
model = YOLO(WEIGHTS_DIR / 'yolov8n-seg.pt')
model.train(data='coco8-seg.yaml', epochs=1, imgsz=32, cache='ram', copy_paste=0.5, mixup=0.5) # test RAM caching
model.train(data='coco8-seg.yaml', epochs=1, imgsz=32, cache='ram', copy_paste=0.5, mixup=0.5)
model(SOURCE)


@@ -283,6 +283,12 @@ def test_data_converter():
coco80_to_coco91_class()


def test_data_annotator():
from ultralytics.data.annotator import auto_annotate

auto_annotate(ASSETS, det_model='yolov8n.pt', sam_model='mobile_sam.pt', output_dir=TMP / 'auto_annotate_labels')


def test_events():
# Test event sending
from ultralytics.hub.utils import Events
@@ -304,12 +310,15 @@ def test_utils_init():


def test_utils_checks():
from ultralytics.utils.checks import check_requirements, check_yolov5u_filename, git_describe
from ultralytics.utils.checks import (check_imgsz, check_requirements, check_yolov5u_filename, git_describe,
print_args)

check_yolov5u_filename('yolov5n.pt')
# check_imshow(warn=True)
git_describe(ROOT)
check_requirements() # check requirements.txt
check_imgsz([600, 600], max_dim=1)
print_args()


def test_utils_benchmarks():
2 changes: 1 addition & 1 deletion ultralytics/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Ultralytics YOLO 🚀, AGPL-3.0 license

__version__ = '8.0.157'
__version__ = '8.0.158'

from ultralytics.hub import start
from ultralytics.models import RTDETR, SAM, YOLO
11 changes: 10 additions & 1 deletion ultralytics/data/annotator.py
Original file line number Diff line number Diff line change
@@ -8,19 +8,28 @@
def auto_annotate(data, det_model='yolov8x.pt', sam_model='sam_b.pt', device='', output_dir=None):
"""
Automatically annotates images using a YOLO object detection model and a SAM segmentation model.
Args:
data (str): Path to a folder containing images to be annotated.
det_model (str, optional): Pre-trained YOLO detection model. Defaults to 'yolov8x.pt'.
sam_model (str, optional): Pre-trained SAM segmentation model. Defaults to 'sam_b.pt'.
device (str, optional): Device to run the models on. Defaults to an empty string (CPU or GPU, if available).
output_dir (str | None | optional): Directory to save the annotated results.
Defaults to a 'labels' folder in the same directory as 'data'.
Example:
```python
from ultralytics.data.annotator import auto_annotate
auto_annotate(data='ultralytics/assets', det_model='yolov8n.pt', sam_model='mobile_sam.pt')
```
"""
det_model = YOLO(det_model)
sam_model = SAM(sam_model)

data = Path(data)
if not output_dir:
output_dir = Path(str(data)).parent / 'labels'
output_dir = data.parent / f'{data.stem}_auto_annotate_labels'
Path(output_dir).mkdir(exist_ok=True, parents=True)

det_results = det_model(data, stream=True, device=device)
2 changes: 1 addition & 1 deletion ultralytics/data/augment.py
Original file line number Diff line number Diff line change
@@ -402,7 +402,7 @@ def apply_keypoints(self, keypoints, M):
keypoints (ndarray): keypoints, [N, 17, 3].
M (ndarray): affine matrix.
Return:
Returns:
new_keypoints (ndarray): keypoints after affine, [N, 17, 3].
"""
n, nkpt = keypoints.shape[:2]
35 changes: 8 additions & 27 deletions ultralytics/engine/exporter.py
Original file line number Diff line number Diff line change
@@ -484,7 +484,7 @@ def export_coreml(self, prefix=colorstr('CoreML:')):
classifier_config = ct.ClassifierConfig(list(self.model.names.values())) if self.args.nms else None
model = self.model
elif self.model.task == 'detect':
model = iOSDetectModel(self.model, self.im) if self.args.nms else self.model
model = IOSDetectModel(self.model, self.im) if self.args.nms else self.model
else:
if self.args.nms:
LOGGER.warning(f"{prefix} WARNING ⚠️ 'nms=True' is only available for Detect models like 'yolov8n.pt'.")
@@ -846,12 +846,11 @@ def _pipeline_coreml(self, model, weights_dir=None, prefix=colorstr('CoreML Pipe
out0, out1 = iter(spec.description.output)
if MACOS:
from PIL import Image
img = Image.new('RGB', (w, h)) # img(192 width, 320 height)
# img = torch.zeros((*opt.img_size, 3)).numpy() # img size(320,192,3) iDetection
img = Image.new('RGB', (w, h)) # w=192, h=320
out = model.predict({'image': img})
out0_shape = out[out0.name].shape
out1_shape = out[out1.name].shape
else: # linux and windows can not run model.predict(), get sizes from pytorch output y
out0_shape = out[out0.name].shape # (3780, 80)
out1_shape = out[out1.name].shape # (3780, 4)
else: # linux and windows can not run model.predict(), get sizes from PyTorch model output y
out0_shape = self.output_shape[2], self.output_shape[1] - 4 # (3780, 80)
out1_shape = self.output_shape[2], 4 # (3780, 4)

@@ -963,11 +962,11 @@ def run_callbacks(self, event: str):
callback(self)


class iOSDetectModel(torch.nn.Module):
"""Wrap an Ultralytics YOLO model for iOS export."""
class IOSDetectModel(torch.nn.Module):
"""Wrap an Ultralytics YOLO model for Apple iOS CoreML export."""

def __init__(self, model, im):
"""Initialize the iOSDetectModel class with a YOLO model and example image."""
"""Initialize the IOSDetectModel class with a YOLO model and example image."""
super().__init__()
b, c, h, w = im.shape # batch, channel, height, width
self.model = model
@@ -981,21 +980,3 @@ def forward(self, x):
"""Normalize predictions of object detection model with input size-dependent factors."""
xywh, cls = self.model(x)[0].transpose(0, 1).split((4, self.nc), 1)
return cls, xywh * self.normalize # confidence (3780, 80), coordinates (3780, 4)


def export(cfg=DEFAULT_CFG):
"""Export a YOLOv model to a specific format."""
cfg.model = cfg.model or 'yolov8n.yaml'
cfg.format = cfg.format or 'torchscript'

from ultralytics import YOLO
model = YOLO(cfg.model)
model.export(**vars(cfg))


if __name__ == '__main__':
"""
CLI:
yolo mode=export model=yolov8n.yaml format=onnx
"""
export()
6 changes: 4 additions & 2 deletions ultralytics/engine/predictor.py
Original file line number Diff line number Diff line change
@@ -138,12 +138,14 @@ def inference(self, im, *args, **kwargs):
return self.model(im, augment=self.args.augment, visualize=visualize)

def pre_transform(self, im):
"""Pre-transform input image before inference.
"""
Pre-transform input image before inference.
Args:
im (List(np.ndarray)): (N, 3, h, w) for tensor, [(h, w, 3) x N] for list.
Return: A list of transformed imgs.
Returns:
(list): A list of transformed images.
"""
same_shapes = all(x.shape == im[0].shape for x in im)
auto = same_shapes and self.model.pt
Loading

0 comments on commit 87ce15d

Please sign in to comment.