Merge branch 'dev' into 'master'

Release 0.1.0 See merge request kkirchheim/pytorch-ood!42
tla93 · Mar 24, 2023 · 9dc1b88 · 9dc1b88
2 parents 58c6419 + ced2426
commit 9dc1b88
Show file tree

Hide file tree

Showing 31 changed files with 1,072 additions and 214 deletions.
diff --git a/README.rst b/README.rst
@@ -136,6 +136,8 @@ The package can be installed via PyPI:
 +-----------------------------+------------------------------------------------------------------------------------------------+------+--------------------+
 | Energy-Based OOD Detection  | Implements the Energy Score of *Energy-based Out-of-distribution Detection*.                   | 2020 | [#EnergyBasedOOD]_ |
 +-----------------------------+------------------------------------------------------------------------------------------------+------+--------------------+
+| Entropy                     | Uses entropy to detect OOD inputs.                                                             | 2021 | [#MaxEntropy]_     |
++-----------------------------+------------------------------------------------------------------------------------------------+------+--------------------+
 | Maximum Logit               | Implements the MaxLogit method.                                                                | 2022 | [#StreeHaz]_       |
 +-----------------------------+------------------------------------------------------------------------------------------------+------+--------------------+
 | KL-Matching                 | Implements the KL-Matching method for Multi-Class classification.                              | 2022 | [#StreeHaz]_       |
@@ -164,12 +166,15 @@ The package can be installed via PyPI:
 | CAC Loss                   | Class Anchor Clustering Loss from *Class Anchor Clustering: a Distance-based Loss for Training   | 2021 | [#CACLoss]_        |
 |                            | Open Set Classifiers*                                                                            |      |                    |
 +----------------------------+--------------------------------------------------------------------------------------------------+------+--------------------+
+| Entropy Maximization       | Entropy maximization and meta classification for OOD in semantic segmentation                    | 2021 | [#MaxEntropy]_     |
++----------------------------+--------------------------------------------------------------------------------------------------+------+--------------------+
 | II Loss                    | Implementation of II Loss function from *Learning a neural network-based representation for      | 2022 | [#IILoss]_         |
 |                            | open set recognition*.                                                                           |      |                    |
 +----------------------------+--------------------------------------------------------------------------------------------------+------+--------------------+
 | MCHAD Loss                 | Implementation of the MCHAD Loss friom the paper *Multi Class Hypersphere Anomaly Detection*.    | 2022 | [#MCHAD]_          |
 +----------------------------+--------------------------------------------------------------------------------------------------+------+--------------------+
 
+
 **Image Datasets**:
 
 +-----------------------+-----------------------------------------------------------------------------------------------------------------+------+---------------+
@@ -212,6 +217,9 @@ The package can be installed via PyPI:
 +-------------+---------------------------------------------------------------------------------------------------------------------------+------+-----------------+
 | WikiText103 | Texts from the wikipedia often used as auxiliary OOD training data.                                                       | 2016 | [#WikiText2]_   |
 +-------------+---------------------------------------------------------------------------------------------------------------------------+------+-----------------+
+| NewsGroup20 | Textx from different newsgroups, as used by Hendrycks et al. in the OOD baseline paper.                                   |      |                 |
++-------------+---------------------------------------------------------------------------------------------------------------------------+------+-----------------+
+
 
 
 
@@ -224,7 +232,7 @@ or check the existing implementations for bugs.
 📝 Citing
 ^^^^^^^^^^
 
-``pytorch-ood`` was presented on a CVPR Workshop in 2022.
+``pytorch-ood`` was presented at a CVPR Workshop in 2022.
 If you use it in a scientific publication, please consider citing::
 
     @InProceedings{kirchheim2022pytorch,
@@ -296,3 +304,5 @@ The legal implications of using pre-trained models in commercial services are, t
 .. [#ViM] Wang, H., Li, Z., Feng, L., Zhang, W. (2022) ViM: Out-Of-Distribution with Virtual-logit Matching. CVPR
 
 .. [#PixMix] Hendrycks, D, Zou, A,  et al. (2022) PixMix: Dreamlike Pictures Comprehensively Improve Safety Measures. CVPR
+
+.. [#MaxEntropy] Chan R,  et al. (2021) Entropy maximization and meta classification for out-of-distribution detection in semantic segmentation. CVPR
diff --git a/docs/conf.py b/docs/conf.py
@@ -35,7 +35,7 @@
 # -- Project information -----------------------------------------------------
 
 project = "pytorch-ood"
-copyright = "2022, K. Kirchheim"
+copyright = "2023, K. Kirchheim"
 author = "Konstantin Kirchheim"
 
 

diff --git a/docs/utils.rst b/docs/utils.rst
@@ -9,7 +9,8 @@ Utilities
       contains_unknown,
       contains_known_and_unknown,
       calc_openness,
-      TensorBuffer
+      TensorBuffer,
+      fix_random_seed
    :undoc-members:
 
 

diff --git a/examples/benchmark/README.rst → examples/benchmarks/README.rst b/examples/benchmark/README.rst → examples/benchmarks/README.rst
@@ -1,6 +1,6 @@
+Image Benchmarks
+====================
 
-Benchmarks
-==================
 The goal of this section is to describe how to quickly obtain some baseline results to compare against.
 
 The following examples reproduces a common benchmark with 7 OOD detectors.
@@ -20,4 +20,4 @@ as additional dependencies:
 
 .. code-block:: shell
 
-    pip install pandas scikit-learn
+    pip install pandas scikit-learn pandas
diff --git a/examples/benchmark/cifar100_baseline.py → examples/benchmarks/cifar100_baseline.py b/examples/benchmark/cifar100_baseline.py → examples/benchmarks/cifar100_baseline.py
@@ -10,21 +10,20 @@
 +=============+=======+=========+==========+==========+
 | MaxSoftmax  | 79.10 | 73.75   | 82.92    | 58.16    |
 +-------------+-------+---------+----------+----------+
-| KLMatching  | 80.14 | 75.26   | 82.48    | 60.73    |
+| KLMatching  | 80.47 | 75.53   | 82.70    | 57.91    |
 +-------------+-------+---------+----------+----------+
-| ODIN        | 81.46 | 76.67   | 84.88    | 55.55    |
+| ODIN        | 81.46 | 76.65   | 84.88    | 55.57    |
 +-------------+-------+---------+----------+----------+
-| Mahalanobis | 83.91 | 79.86   | 86.08    | 47.03    |
+| Entropy     | 81.52 | 77.09   | 84.38    | 57.12    |
 +-------------+-------+---------+----------+----------+
-| MaxLogit    | 84.64 | 79.95   | 87.11    | 48.51    |
+| Mahalanobis | 83.91 | 79.86   | 86.08    | 46.99    |
 +-------------+-------+---------+----------+----------+
-| EnergyBased | 84.90 | 80.27   | 87.32    | 47.84    |
+| MaxLogit    | 84.64 | 79.95   | 87.25    | 48.51    |
 +-------------+-------+---------+----------+----------+
-| ViM         | 85.86 | 81.18   | 88.81    | 41.82    |
+| EnergyBased | 84.90 | 80.27   | 87.46    | 47.85    |
++-------------+-------+---------+----------+----------+
+| ViM         | 85.87 | 81.18   | 88.81    | 41.83    |
 +-------------+-------+---------+----------+----------+
-
-
-
 
 """
 import pandas as pd  # additional dependency, used here for convenience
@@ -43,6 +42,7 @@
 from pytorch_ood.detector import (
     ODIN,
     EnergyBased,
+    Entropy,
     KLMatching,
     Mahalanobis,
     MaxLogit,
@@ -85,6 +85,7 @@
 # Stage 2: Create OOD detector
 print("STAGE 2: Creating OOD Detectors")
 detectors = {}
+detectors["Entropy"] = Entropy(model)
 detectors["ViM"] = ViM(model.features, d=64, w=model.fc.weight, b=model.fc.bias)
 detectors["Mahalanobis"] = Mahalanobis(model.features, norm_std=std, eps=0.002)
 detectors["KLMatching"] = KLMatching(model)

diff --git a/examples/benchmark/cifar10_baseline.py → examples/benchmarks/cifar10_baseline.py b/examples/benchmark/cifar10_baseline.py → examples/benchmarks/cifar10_baseline.py
@@ -8,18 +8,23 @@
 +-------------+-------+---------+----------+----------+
 | Detector    | AUROC | AUPR-IN | AUPR-OUT | FPR95TPR |
 +=============+=======+=========+==========+==========+
-| KLMatching  | 88.76 | 86.95   | 85.17    | 58.55    |
+| KLMatching  | 88.73 | 86.95   | 85.10    | 58.74    |
 +-------------+-------+---------+----------+----------+
-| MaxSoftmax  | 91.85 | 88.55   | 93.57    | 28.42    |
+| MaxSoftmax  | 91.85 | 88.55   | 93.57    | 28.43    |
 +-------------+-------+---------+----------+----------+
-| MaxLogit    | 93.06 | 91.44   | 93.73    | 31.16    |
+| Entropy     | 92.48 | 90.16   | 93.87    | 28.29    |
 +-------------+-------+---------+----------+----------+
-| EnergyBased | 93.11 | 91.51   | 93.76    | 31.02    |
+| MaxLogit    | 93.06 | 91.44   | 93.74    | 31.18    |
 +-------------+-------+---------+----------+----------+
-| ODIN        | 93.20 | 92.12   | 93.94    | 31.64    |
+| EnergyBased | 93.10 | 91.51   | 93.78    | 31.05    |
 +-------------+-------+---------+----------+----------+
-| ViM         | 94.49 | 93.43   | 95.33    | 23.49    |
+| ODIN        | 93.20 | 92.12   | 93.94    | 31.65    |
 +-------------+-------+---------+----------+----------+
+| ViM         | 94.49 | 93.42   | 95.34    | 23.48    |
++-------------+-------+---------+----------+----------+
+| Mahalanobis | 94.87 | 93.69   | 95.79    | 21.00    |
++-------------+-------+---------+----------+----------+
+
 
 """
 import pandas as pd  # additional dependency, used here for convenience
@@ -38,6 +43,7 @@
 from pytorch_ood.detector import (
     ODIN,
     EnergyBased,
+    Entropy,
     KLMatching,
     Mahalanobis,
     MaxLogit,
@@ -83,6 +89,7 @@
 # **Stage 2**: Create OOD detector
 print("STAGE 2: Creating OOD Detectors")
 detectors = {}
+detectors["Entropy"] = Entropy(model)
 detectors["ViM"] = ViM(model.features, d=64, w=model.fc.weight, b=model.fc.bias)
 detectors["Mahalanobis"] = Mahalanobis(model.features, norm_std=std, eps=0.002)
 detectors["KLMatching"] = KLMatching(model)

diff --git a/examples/segmentation/README.rst b/examples/segmentation/README.rst
@@ -1,5 +1,5 @@
 
-Segmentation
+Image Segmentation
 ==================
 
 Examples for anomaly segmentation.

diff --git a/examples/segmentation/street.py b/examples/segmentation/street.py
@@ -10,8 +10,9 @@
 .. note :: Training with a batch-size of 4 requires slightly more than 12 GB of GPU memory.
     However, the models tend to also converge to reasonable performance with a smaller batch-size.
 
-.. warning :: The evaluation will allocate :math:`\\approx` 100GB of memory. Make sure to configure sufficient
-    SWAP space. This is because the calculation of the metrics requires that all predictions are available.
+.. warning :: The results produced by this script vary. It is impossible to ensure the
+    reproducibility of the exact numerical values at the moment, because the model includes operations for
+    which no deterministic implementation exists at the time of writing.
 
 .. note :: The license of the model originally used for the street hazards dataset
     is not compatible with ``pytorch-ood``. This prevents us from re-using the implementation
@@ -27,12 +28,17 @@
 
 from pytorch_ood.dataset.img import StreetHazards
 from pytorch_ood.detector import EnergyBased
-from pytorch_ood.utils import OODMetrics
+from pytorch_ood.utils import OODMetrics, fix_random_seed
 
 device = "cuda:0"
 batch_size = 4
 num_epochs = 1
 
+fix_random_seed(12345)
+g = torch.Generator()
+g.manual_seed(0)
+
+
 # %%
 # Setup preprocessing
 preprocess_input = get_preprocessing_fn("resnet50", pretrained="imagenet")
@@ -62,14 +68,21 @@ def my_transform(img, target):
     encoder_name="resnet50",
     encoder_weights="imagenet",
     in_channels=3,
-    classes=14,
+    classes=13,
 ).to(device)
 
 # %%
 # Train model for some epochs
 criterion = smp.losses.DiceLoss(mode="multiclass")
 optimizer = torch.optim.Adam(params=model.parameters(), lr=0.0001)
-loader = DataLoader(dataset, batch_size=batch_size, shuffle=True, num_workers=10)
+loader = DataLoader(
+    dataset,
+    batch_size=batch_size,
+    shuffle=True,
+    num_workers=10,
+    worker_init_fn=fix_random_seed,
+    generator=g,
+)
 
 ious = []
 loss_ema = 0
@@ -89,7 +102,7 @@ def my_transform(img, target):
             y_hat.softmax(dim=1).max(dim=1).indices.long(),
             y.long(),
             mode="multiclass",
-            num_classes=14,
+            num_classes=13,
         )
         iou = iou_score(tp, fp, fn, tn)
 
@@ -105,9 +118,9 @@ def my_transform(img, target):
 # Evaluate
 print("Evaluating")
 model.eval()
-loader = DataLoader(dataset_test, batch_size=4)
+loader = DataLoader(dataset_test, batch_size=4, worker_init_fn=fix_random_seed, generator=g)
 detector = EnergyBased(model)
-metrics = OODMetrics()
+metrics = OODMetrics(mode="segmentation")
 
 with torch.no_grad():
     for n, (x, y) in enumerate(loader):

diff --git a/examples/segmentation/street_entropic.py b/examples/segmentation/street_entropic.py
@@ -0,0 +1,139 @@
+"""
+StreetHazards with Entropic Loss
+-------------------------------------
+
+We train a Feature Pyramid Segmentation model
+with a ResNet-50 backbone pre-trained on the ImageNet
+on the :class:`StreetHazards<pytorch_ood.dataset.img.StreetHazards>` **test set** using
+the supervised :class:`EntropicOpenSetLoss<pytorch_ood.loss.EntropicOpenSetLoss>`.
+
+We then use the :class:`Entropy<pytorch_ood.detector.Entropy>` OOD detector.
+
+This setup is merely made to demonstrate how to train a supervised anomaly segmentation model with
+this loss function.
+
+.. warning :: We train on the test set, as it contains examples of anomalies.
+    The results will not be meaningful.
+
+
+.. note :: Training with a batch-size of 4 requires slightly more than 12 GB of GPU memory.
+    However, the models tend to also converge to reasonable performance with a smaller batch-size.
+
+"""
+import segmentation_models_pytorch as smp
+import torch
+from segmentation_models_pytorch.encoders import get_preprocessing_fn
+from segmentation_models_pytorch.metrics import iou_score
+from torch.utils.data import DataLoader
+from torchvision.transforms.functional import pad, to_tensor
+
+from pytorch_ood.dataset.img import StreetHazards
+from pytorch_ood.detector import Entropy, MaxSoftmax
+from pytorch_ood.loss import EntropicOpenSetLoss
+from pytorch_ood.utils import OODMetrics, fix_random_seed
+
+device = "cuda:0"
+batch_size = 4
+num_epochs = 1
+
+fix_random_seed(12345)
+g = torch.Generator()
+g.manual_seed(0)
+
+
+# %%
+# Setup preprocessing
+preprocess_input = get_preprocessing_fn("resnet50", pretrained="imagenet")
+
+
+def my_transform(img, target):
+    img = to_tensor(img)[:3, :, :]  # drop 4th channel
+    img = torch.moveaxis(img, 0, -1)
+    img = preprocess_input(img)
+    img = torch.moveaxis(img, -1, 0)
+
+    # size must be divisible by 32, so we pad the image.
+    img = pad(img, [0, 8]).float()
+    target = pad(target, [0, 8])
+    return img, target
+
+
+# %%
+# Setup datasets, train on ood images for demonstration purposes.
+dataset = StreetHazards(root="data", subset="test", transform=my_transform, download=True)
+dataset_test = StreetHazards(root="data", subset="test", transform=my_transform, download=True)
+
+
+# %%
+# Setup model
+model = smp.FPN(
+    encoder_name="resnet50",
+    encoder_weights="imagenet",
+    in_channels=3,
+    classes=13,
+).to(device)
+
+# %%
+# Train model for some epochs
+
+criterion = EntropicOpenSetLoss()
+optimizer = torch.optim.Adam(params=model.parameters(), lr=0.0001)
+loader = DataLoader(
+    dataset,
+    batch_size=batch_size,
+    shuffle=True,
+    num_workers=10,
+    worker_init_fn=fix_random_seed,
+    generator=g,
+)
+
+ious = []
+loss_ema = 0
+ioe_ema = 0
+
+for epoch in range(num_epochs):
+    for n, (x, y) in enumerate(loader):
+        optimizer.zero_grad()
+        y, x = y.to(device), x.to(device)
+
+        y_hat = model(x)
+        loss = criterion(y_hat, y)
+        loss.backward()
+        optimizer.step()
+
+        tp, fp, fn, tn = smp.metrics.get_stats(
+            y_hat.softmax(dim=1).max(dim=1).indices.long(),
+            y.long(),
+            mode="multiclass",
+            num_classes=13,
+        )
+        iou = iou_score(tp, fp, fn, tn)
+
+        loss_ema = 0.8 * loss_ema + 0.2 * loss.item()
+        ioe_ema = 0.8 * ioe_ema + 0.2 * iou.mean().item()
+
+        if n % 10 == 0:
+            print(
+                f"Epoch {epoch:03d} [{n:05d}/{len(loader):05d}] \t Loss: {loss_ema:02.2f} \t IoU: {ioe_ema:02.2f}"
+            )
+
+# %%
+# Evaluate
+print("Evaluating")
+model.eval()
+loader = DataLoader(dataset_test, batch_size=4, worker_init_fn=fix_random_seed, generator=g)
+detector = Entropy(model)
+metrics = OODMetrics(mode="segmentation")
+
+with torch.no_grad():
+    for n, (x, y) in enumerate(loader):
+        y, x = y.to(device), x.to(device)
+        o = detector(x)
+
+        # undo padding
+        o = pad(o, [-8, -8])
+        y = pad(y, [-8, -8])
+
+        metrics.update(o, y)
+
+print(metrics.compute())