use per-project pathmanager

Reviewed By: vkhalidov

Differential Revision: D24616106

fbshipit-source-id: 3f6a7a9fa33f5f9b11ab2625b7309f0e405cbbda

detectron2/checkpoint/catalog.py

@@ -1,6 +1,7 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 import logging
-from fvcore.common.file_io import PathHandler, PathManager
+
+from detectron2.utils.file_io import PathHandler, PathManager
 
 
 class ModelCatalog(object):
@@ -111,24 +112,4 @@ def _open(self, path, mode="r", **kwargs):
         return PathManager.open(self._get_local_path(path), mode, **kwargs)
 
 
-class Detectron2Handler(PathHandler):
-    """
-    Resolve anything that's in Detectron2 model zoo.
-    """
-
-    PREFIX = "detectron2://"
-    S3_DETECTRON2_PREFIX = "https://dl.fbaipublicfiles.com/detectron2/"
-
-    def _get_supported_prefixes(self):
-        return [self.PREFIX]
-
-    def _get_local_path(self, path):
-        name = path[len(self.PREFIX) :]
-        return PathManager.get_local_path(self.S3_DETECTRON2_PREFIX + name)
-
-    def _open(self, path, mode="r", **kwargs):
-        return PathManager.open(self._get_local_path(path), mode, **kwargs)
-
-
 PathManager.register_handler(ModelCatalogHandler())
-PathManager.register_handler(Detectron2Handler())

detectron2/checkpoint/detection_checkpoint.py

@@ -1,9 +1,9 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 import pickle
 from fvcore.common.checkpoint import Checkpointer
-from fvcore.common.file_io import PathManager
 
 import detectron2.utils.comm as comm
+from detectron2.utils.file_io import PathManager
 
 from .c2_model_loading import align_and_update_state_dicts
 
@@ -22,6 +22,18 @@ def __init__(self, model, save_dir="", *, save_to_disk=None, **checkpointables):
             save_to_disk=is_main_process if save_to_disk is None else save_to_disk,
             **checkpointables,
         )
+        if hasattr(self, "path_manager"):
+            self.path_manager = PathManager
+        else:
+            # This could only happen for open source
+            # TODO remove after upgrading fvcore
+            from fvcore.common.file_io import PathManager as g_PathManager
+
+            for handler in PathManager._path_handlers.values():
+                try:
+                    g_PathManager.register_handler(handler)
+                except KeyError:
+                    pass
 
     def _load_file(self, filename):
         if filename.endswith(".pkl"):

detectron2/config/config.py

@@ -5,7 +5,8 @@
 import inspect
 import logging
 from fvcore.common.config import CfgNode as _CfgNode
-from fvcore.common.file_io import PathManager
+
+from detectron2.utils.file_io import PathManager
 
 
 class CfgNode(_CfgNode):
@@ -20,10 +21,14 @@ class CfgNode(_CfgNode):
        When attempting to merge an old config, it will convert the old config automatically.
     """
 
+    @classmethod
+    def _open_cfg(cls, filename):
+        return PathManager.open(filename, "r")
+
     # Note that the default value of allow_unsafe is changed to True
     def merge_from_file(self, cfg_filename: str, allow_unsafe: bool = True) -> None:
         assert PathManager.isfile(cfg_filename), f"Config file '{cfg_filename}' does not exist!"
-        loaded_cfg = _CfgNode.load_yaml_with_base(cfg_filename, allow_unsafe=allow_unsafe)
+        loaded_cfg = self.load_yaml_with_base(cfg_filename, allow_unsafe=allow_unsafe)
         loaded_cfg = type(self)(loaded_cfg)
 
         # defaults.py needs to import CfgNode

detectron2/data/build.py

@@ -5,13 +5,13 @@
 import operator
 import pickle
 import torch.utils.data
-from fvcore.common.file_io import PathManager
 from tabulate import tabulate
 from termcolor import colored
 
 from detectron2.structures import BoxMode
 from detectron2.utils.comm import get_world_size
 from detectron2.utils.env import seed_all_rng
+from detectron2.utils.file_io import PathManager
 from detectron2.utils.logger import log_first_n
 
 from .catalog import DatasetCatalog, MetadataCatalog

detectron2/data/datasets/cityscapes.py

@@ -7,11 +7,11 @@
 import os
 from itertools import chain
 import pycocotools.mask as mask_util
-from fvcore.common.file_io import PathManager
 from PIL import Image
 
 from detectron2.structures import BoxMode
 from detectron2.utils.comm import get_world_size
+from detectron2.utils.file_io import PathManager
 from detectron2.utils.logger import setup_logger
 
 try:

detectron2/data/datasets/cityscapes_panoptic.py

@@ -2,10 +2,10 @@
 import json
 import logging
 import os
-from fvcore.common.file_io import PathManager
 
 from detectron2.data import DatasetCatalog, MetadataCatalog
 from detectron2.data.datasets.builtin_meta import CITYSCAPES_CATEGORIES
+from detectron2.utils.file_io import PathManager
 
 """
 This file contains functions to register the Cityscapes panoptic dataset to the DatasetCatalog.

detectron2/data/datasets/coco.py

@@ -8,11 +8,12 @@
 import os
 import shutil
 import pycocotools.mask as mask_util
-from fvcore.common.file_io import PathManager, file_lock
+from fvcore.common.file_io import file_lock
 from fvcore.common.timer import Timer
 from PIL import Image
 
 from detectron2.structures import Boxes, BoxMode, PolygonMasks
+from detectron2.utils.file_io import PathManager
 
 from .. import DatasetCatalog, MetadataCatalog
 

detectron2/data/datasets/coco_panoptic.py

@@ -2,9 +2,9 @@
 import copy
 import json
 import os
-from fvcore.common.file_io import PathManager
 
 from detectron2.data import DatasetCatalog, MetadataCatalog
+from detectron2.utils.file_io import PathManager
 
 from .coco import load_coco_json, load_sem_seg
 

detectron2/data/datasets/lvis.py

@@ -1,11 +1,11 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 import logging
 import os
-from fvcore.common.file_io import PathManager
 from fvcore.common.timer import Timer
 
 from detectron2.data import DatasetCatalog, MetadataCatalog
 from detectron2.structures import BoxMode
+from detectron2.utils.file_io import PathManager
 
 from .builtin_meta import _get_coco_instances_meta
 from .lvis_v0_5_categories import LVIS_CATEGORIES as LVIS_V0_5_CATEGORIES

detectron2/data/datasets/pascal_voc.py

@@ -5,10 +5,10 @@
 import os
 import xml.etree.ElementTree as ET
 from typing import List, Tuple, Union
-from fvcore.common.file_io import PathManager
 
 from detectron2.data import DatasetCatalog, MetadataCatalog
 from detectron2.structures import BoxMode
+from detectron2.utils.file_io import PathManager
 
 __all__ = ["load_voc_instances", "register_pascal_voc"]
 

detectron2/data/detection_utils.py

@@ -9,7 +9,6 @@
 import numpy as np
 import pycocotools.mask as mask_util
 import torch
-from fvcore.common.file_io import PathManager
 from PIL import Image
 
 from detectron2.structures import (
@@ -22,6 +21,7 @@
     RotatedBoxes,
     polygons_to_bitmask,
 )
+from detectron2.utils.file_io import PathManager
 
 from . import transforms as T
 from .catalog import MetadataCatalog

detectron2/engine/defaults.py

@@ -15,7 +15,6 @@
 import sys
 from collections import OrderedDict
 import torch
-from fvcore.common.file_io import PathManager
 from fvcore.nn.precise_bn import get_bn_modules
 from torch.nn.parallel import DistributedDataParallel
 
@@ -38,6 +37,7 @@
 from detectron2.utils.collect_env import collect_env_info
 from detectron2.utils.env import seed_all_rng
 from detectron2.utils.events import CommonMetricPrinter, JSONWriter, TensorboardXWriter
+from detectron2.utils.file_io import PathManager
 from detectron2.utils.logger import setup_logger
 
 from . import hooks

detectron2/engine/hooks.py

@@ -10,13 +10,13 @@
 from collections import Counter
 import torch
 from fvcore.common.checkpoint import PeriodicCheckpointer as _PeriodicCheckpointer
-from fvcore.common.file_io import PathManager
 from fvcore.common.timer import Timer
 from fvcore.nn.precise_bn import get_bn_modules, update_bn_stats
 
 import detectron2.utils.comm as comm
 from detectron2.evaluation.testing import flatten_results_dict
 from detectron2.utils.events import EventStorage, EventWriter
+from detectron2.utils.file_io import PathManager
 
 from .train_loop import HookBase
 

detectron2/evaluation/cityscapes_evaluation.py

@@ -6,11 +6,11 @@
 import tempfile
 from collections import OrderedDict
 import torch
-from fvcore.common.file_io import PathManager
 from PIL import Image
 
 from detectron2.data import MetadataCatalog
 from detectron2.utils import comm
+from detectron2.utils.file_io import PathManager
 
 from .evaluator import DatasetEvaluator
 

detectron2/evaluation/coco_evaluation.py

@@ -11,7 +11,6 @@
 from collections import OrderedDict
 import pycocotools.mask as mask_util
 import torch
-from fvcore.common.file_io import PathManager
 from pycocotools.coco import COCO
 from pycocotools.cocoeval import COCOeval
 from tabulate import tabulate
@@ -22,6 +21,7 @@
 from detectron2.data.datasets.coco import convert_to_coco_json
 from detectron2.evaluation.fast_eval_api import COCOeval_opt
 from detectron2.structures import Boxes, BoxMode, pairwise_iou
+from detectron2.utils.file_io import PathManager
 from detectron2.utils.logger import create_small_table
 
 from .evaluator import DatasetEvaluator

detectron2/evaluation/lvis_evaluation.py

@@ -7,11 +7,11 @@
 import pickle
 from collections import OrderedDict
 import torch
-from fvcore.common.file_io import PathManager
 
 import detectron2.utils.comm as comm
 from detectron2.data import MetadataCatalog
 from detectron2.structures import Boxes, BoxMode, pairwise_iou
+from detectron2.utils.file_io import PathManager
 from detectron2.utils.logger import create_small_table
 
 from .coco_evaluation import instances_to_coco_json

detectron2/evaluation/panoptic_evaluation.py

@@ -8,12 +8,12 @@
 import os
 import tempfile
 from collections import OrderedDict
-from fvcore.common.file_io import PathManager
 from PIL import Image
 from tabulate import tabulate
 
 from detectron2.data import MetadataCatalog
 from detectron2.utils import comm
+from detectron2.utils.file_io import PathManager
 
 from .evaluator import DatasetEvaluator
 

detectron2/evaluation/pascal_voc_evaluation.py

@@ -9,10 +9,10 @@
 from collections import OrderedDict, defaultdict
 from functools import lru_cache
 import torch
-from fvcore.common.file_io import PathManager
 
 from detectron2.data import MetadataCatalog
 from detectron2.utils import comm
+from detectron2.utils.file_io import PathManager
 
 from .evaluator import DatasetEvaluator
 

detectron2/evaluation/rotated_coco_evaluation.py

@@ -4,10 +4,10 @@
 import numpy as np
 import os
 import torch
-from fvcore.common.file_io import PathManager
 from pycocotools.cocoeval import COCOeval, maskUtils
 
 from detectron2.structures import BoxMode, RotatedBoxes, pairwise_iou_rotated
+from detectron2.utils.file_io import PathManager
 
 from .coco_evaluation import COCOEvaluator
 

detectron2/evaluation/sem_seg_evaluation.py

@@ -8,10 +8,10 @@
 import PIL.Image as Image
 import pycocotools.mask as mask_util
 import torch
-from fvcore.common.file_io import PathManager
 
 from detectron2.data import DatasetCatalog, MetadataCatalog
 from detectron2.utils.comm import all_gather, is_main_process, synchronize
+from detectron2.utils.file_io import PathManager
 
 from .evaluator import DatasetEvaluator
 

detectron2/export/api.py

@@ -4,10 +4,10 @@
 import os
 import torch
 from caffe2.proto import caffe2_pb2
-from fvcore.common.file_io import PathManager
 from torch import nn
 
 from detectron2.config import CfgNode as CN
+from detectron2.utils.file_io import PathManager
 
 from .caffe2_inference import ProtobufDetectionModel
 from .caffe2_modeling import META_ARCH_CAFFE2_EXPORT_TYPE_MAP, convert_batched_inputs_to_c2_format

detectron2/export/torchscript.py

@@ -2,7 +2,8 @@
 
 import os
 import torch
-from fvcore.common.file_io import PathManager
+
+from detectron2.utils.file_io import PathManager
 
 from .torchscript_patch import patch_instances
 

detectron2/utils/events.py

@@ -7,9 +7,10 @@
 from collections import defaultdict
 from contextlib import contextmanager
 import torch
-from fvcore.common.file_io import PathManager
 from fvcore.common.history_buffer import HistoryBuffer
 
+from detectron2.utils.file_io import PathManager
+
 __all__ = [
     "get_event_storage",
     "JSONWriter",

detectron2/utils/file_io.py

@@ -0,0 +1,35 @@
+from fvcore.common.file_io import HTTPURLHandler, OneDrivePathHandler, PathHandler, PathManagerBase
+
+__all__ = ["PathManager", "PathHandler"]
+
+
+PathManager = PathManagerBase()
+"""
+This is a detectron2 project-specific PathManager.
+We try to stay away from global PathManager in fvcore as it
+introduces potential conflicts among other libraries.
+"""
+
+
+class Detectron2Handler(PathHandler):
+    """
+    Resolve anything that's hosted under detectron2's namespace.
+    """
+
+    PREFIX = "detectron2://"
+    S3_DETECTRON2_PREFIX = "https://dl.fbaipublicfiles.com/detectron2/"
+
+    def _get_supported_prefixes(self):
+        return [self.PREFIX]
+
+    def _get_local_path(self, path):
+        name = path[len(self.PREFIX) :]
+        return PathManager.get_local_path(self.S3_DETECTRON2_PREFIX + name)
+
+    def _open(self, path, mode="r", **kwargs):
+        return PathManager.open(self._get_local_path(path), mode, **kwargs)
+
+
+PathManager.register_handler(HTTPURLHandler())
+PathManager.register_handler(OneDrivePathHandler())
+PathManager.register_handler(Detectron2Handler())