Move pytorch distributed tests to separate folder for contbuild. (pytorch#30445)

Summary:
Pull Request resolved: pytorch#30445

Create distributed and rpc directories under caffe/test for better management
of unit tests.

Differential Revision: D18702786

fbshipit-source-id: e9daeed0cfb846ef68806f6decfcb57c0e0e3606

Author: pritamdamania

.jenkins/pytorch/multigpu-test.sh

@@ -31,7 +31,7 @@ fi
 
 python tools/download_mnist.py --quiet -d test/cpp/api/mnist
 OMP_NUM_THREADS=2 TORCH_CPP_TEST_MNIST_PATH="test/cpp/api/mnist" build/bin/test_api
-time python test/run_test.py --verbose -i distributed
-time python test/run_test.py --verbose -i c10d
-time python test/run_test.py --verbose -i c10d_spawn
+time python test/run_test.py --verbose -i distributed/test_distributed
+time python test/run_test.py --verbose -i distributed/test_c10d
+time python test/run_test.py --verbose -i distributed/test_c10d_spawn
 assert_git_not_dirty

.jenkins/pytorch/test.sh

@@ -131,22 +131,22 @@ elif [[ "${BUILD_ENVIRONMENT}" == *-NO_AVX2-* ]]; then
 fi
 
 test_python_nn() {
-  time python test/run_test.py --include nn --verbose
+  time python test/run_test.py --include test_nn --verbose
   assert_git_not_dirty
 }
 
 test_python_ge_config_simple() {
-  time python test/run_test.py --include jit_simple --verbose
+  time python test/run_test.py --include test_jit_simple --verbose
   assert_git_not_dirty
 }
 
 test_python_ge_config_legacy() {
-  time python test/run_test.py --include jit_legacy jit_fuser_legacy --verbose
+  time python test/run_test.py --include test_jit_legacy test_jit_fuser_legacy --verbose
   assert_git_not_dirty
 }
 
 test_python_all_except_nn() {
-  time python test/run_test.py --exclude nn jit_simple jit_legacy jit_fuser_legacy --verbose --bring-to-front quantization quantized quantized_tensor quantized_nn_mods
+  time python test/run_test.py --exclude test_nn test_jit_simple test_jit_legacy test_jit_fuser_legacy --verbose --bring-to-front test_quantization test_quantized test_quantized_tensor test_quantized_nn_mods
   assert_git_not_dirty
 }
 

.jenkins/pytorch/win-test-helpers/test_python_all_except_nn.bat

@@ -1,3 +1,3 @@
 call %SCRIPT_HELPERS_DIR%\setup_pytorch_env.bat
-cd test && python run_test.py --exclude nn jit_simple jit_legacy jit_fuser_legacy --verbose && cd ..
+cd test && python run_test.py --exclude test_nn test_jit_simple test_jit_legacy test_jit_fuser_legacy --verbose && cd ..
 if ERRORLEVEL 1 exit /b 1

.jenkins/pytorch/win-test-helpers/test_python_nn.bat

@@ -8,7 +8,7 @@ python %SCRIPT_HELPERS_DIR%\run_python_nn_smoketests.py
 if ERRORLEVEL 1 exit /b 1
 
 echo Run nn tests
-python run_test.py --include nn --verbose
+python run_test.py --include test_nn --verbose
 if ERRORLEVEL 1 exit /b 1
 
 popd

setup.py

@@ -859,7 +859,7 @@ def print_box(msg):
             ],
             'caffe2': [
                 'python/serialized_test/data/operator_test/*.zip',
-            ]
+            ],
         },
         url='https://pytorch.org/',
         download_url='https://github.com/pytorch/pytorch/tags',

test/test_dist_autograd_spawn.py renamed to test/distributed/rpc/test_dist_autograd_spawn.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-from dist_autograd_test import DistAutogradTest, DistAutogradJitTest
-from common_distributed import MultiProcessTestCase
-from common_utils import TEST_WITH_ASAN, run_tests
+from torch.testing._internal.distributed.rpc.dist_autograd_test import DistAutogradTest, DistAutogradJitTest
+from torch.testing._internal.common_distributed import MultiProcessTestCase
+from torch.testing._internal.common_utils import TEST_WITH_ASAN, run_tests
 
 import unittest
 

test/test_dist_optimizer_spawn.py renamed to test/distributed/rpc/test_dist_optimizer_spawn.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-from dist_optimizer_test import DistOptimizerTest
-from common_distributed import MultiProcessTestCase
-from common_utils import TEST_WITH_ASAN, run_tests
+from torch.testing._internal.distributed.rpc.dist_optimizer_test import DistOptimizerTest
+from torch.testing._internal.common_distributed import MultiProcessTestCase
+from torch.testing._internal.common_utils import TEST_WITH_ASAN, run_tests
 
 import unittest
 

test/test_rpc_spawn.py renamed to test/distributed/rpc/test_rpc_spawn.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-from rpc_test import RpcTest
-from common_distributed import MultiProcessTestCase
-from common_utils import TEST_WITH_ASAN, run_tests
+from torch.testing._internal.distributed.rpc.rpc_test import RpcTest
+from torch.testing._internal.common_distributed import MultiProcessTestCase
+from torch.testing._internal.common_utils import TEST_WITH_ASAN, run_tests
 
 import unittest
 

test/test_c10d.py renamed to test/distributed/test_c10d.py

@@ -19,18 +19,20 @@
 
 import torch
 from torch._six import string_classes
-import common_utils as common
+import torch.testing._internal.common_utils as common
 from torch import nn
 import torch.nn.functional as F
 import torch.distributed as c10d
 import torch.distributed as dist
 from torch.nn.parallel import DistributedDataParallel
 
-from common_distributed import MultiProcessTestCase, \
+from torch.testing._internal.common_distributed import MultiProcessTestCase, \
     requires_gloo, requires_nccl, requires_nccl_version, \
     skip_if_not_multigpu, skip_if_lt_x_gpu, skip_for_known_issues, get_timeout, skip_if_rocm, \
     simple_sparse_reduce_tests
-from common_utils import TestCase, load_tests, run_tests, retry_on_address_already_in_use_error, TEST_WITH_TSAN
+
+from torch.testing._internal.common_utils import TestCase, load_tests, run_tests, \
+    retry_on_address_already_in_use_error, TEST_WITH_TSAN
 
 # load_tests from common_utils is used to automatically filter tests for
 # sharding on sandcastle. This line silences flake warnings

test/test_c10d_spawn.py renamed to test/distributed/test_c10d_spawn.py

@@ -6,9 +6,9 @@
 import torch.distributed as c10d
 import torch.multiprocessing as mp
 
-from common_cuda import TEST_MULTIGPU
-from common_utils import TestCase, load_tests, run_tests
-from common_utils import NO_MULTIPROCESSING_SPAWN
+from torch.testing._internal.common_cuda import TEST_MULTIGPU
+from torch.testing._internal.common_utils import TestCase, load_tests, run_tests
+from torch.testing._internal.common_utils import NO_MULTIPROCESSING_SPAWN
 
 # load_tests from common_utils is used to automatically filter tests for
 # sharding on sandcastle. This line silences flake warnings