Integrate coverage report into CI (microsoft#3854)

.gitignore

@@ -32,6 +32,12 @@ lib-cov
 
 # Coverage directory used by tools like istanbul
 coverage
+junit/
+coverage.xml
+test-*.xml
+.coverage.*
+htmlcov/
+.coverage
 
 # nyc test coverage
 .nyc_output

dependencies/develop.txt

@@ -6,5 +6,7 @@ sphinx-rtd-theme
 sphinxcontrib-websupport
 nbsphinx
 pytest
+pytest-cov
+pytest-azurepipelines
 coverage
 ipython

dependencies/recommended.txt

@@ -2,6 +2,7 @@
 
 -f https://download.pytorch.org/whl/torch_stable.html
 tensorflow
+keras
 torch == 1.6.0+cpu ; sys_platform != "darwin"
 torch == 1.6.0 ; sys_platform == "darwin"
 torchvision == 0.7.0+cpu ; sys_platform != "darwin"

nni/algorithms/compression/pytorch/auto_compress/experiment.py

@@ -5,8 +5,6 @@
 from pathlib import Path, PurePath
 from typing import overload, Union, List
 
-from numpy import tri
-
 from nni.experiment import Experiment, ExperimentConfig
 from nni.algorithms.compression.pytorch.auto_compress.interface import AbstractAutoCompressionModule
 
@@ -62,7 +60,8 @@ def __init__(self, auto_compress_module: AbstractAutoCompressionModule, config=N
 
     def start(self, port: int, debug: bool) -> None:
         trial_code_directory = str(PurePath(Path(self.config.trial_code_directory).absolute())) + '/'
-        assert self.module_file_path.startswith(trial_code_directory), 'The file path of the user-provided module should under trial_code_directory.'
+        assert self.module_file_path.startswith(trial_code_directory), \
+            'The file path of the user-provided module should under trial_code_directory.'
         relative_module_path = self.module_file_path.split(trial_code_directory)[1]
         # only support linux, need refactor?
         command = 'python3 -m nni.algorithms.compression.pytorch.auto_compress.trial_entry --module_file_name {} --module_class_name {}'

nni/algorithms/compression/pytorch/auto_compress/utils.py

@@ -31,10 +31,11 @@ def _add_pruner_config(self, pruner_name: str, config_list: list, **algo_kwargs)
         pruner_name
             Supported pruner name: 'level', 'slim', 'l1', 'l2', 'fpgm', 'taylorfo', 'apoz', 'mean_activation'.
         config_list
-            Except 'op_types' and 'op_names', other config value can be written as `{'_type': ..., '_value': ...}`.
+            Except 'op_types' and 'op_names', other config value can be written as ``{'_type': ..., '_value': ...}``.
         **algo_kwargs
             The additional pruner parameters except 'model', 'config_list', 'optimizer', 'trainer', 'criterion'.
-            i.e., you can set `statistics_batch_num={'_type': 'choice', '_value': [1, 2, 3]}` in TaylorFOWeightFilterPruner or just `statistics_batch_num=1`.
+            i.e., you can set ``statistics_batch_num={'_type': 'choice', '_value': [1, 2, 3]}``
+            in TaylorFOWeightFilterPruner or just ``statistics_batch_num=1``.
         """
         sub_search_space = {'_name': pruner_name}
         for config in config_list:

nni/algorithms/compression/pytorch/pruning/iterative_pruner.py

@@ -84,7 +84,7 @@ def compress(self):
                 self._trainer(self.bound_model, optimizer=self.optimizer, criterion=self._criterion, epoch=epoch)
             # NOTE: workaround for statistics_batch_num bigger than max batch number in one epoch, need refactor
             if hasattr(self.masker, 'statistics_batch_num') and hasattr(self, 'iterations'):
-                if self.iterations < self.masker.statistics_batch_num:
+                if self.iterations < self.masker.statistics_batch_num:  # pylint: disable=access-member-before-definition
                     self.iterations = self.masker.statistics_batch_num
             self.update_mask()
         self.bound_model.train(training)
@@ -118,7 +118,8 @@ class AGPPruner(IterativePruner):
         choose from `['level', 'slim', 'l1', 'l2', 'fpgm', 'taylorfo', 'apoz', 'mean_activation']`, by default `level`
     """
 
-    def __init__(self, model, config_list, optimizer, trainer, criterion, num_iterations=10, epochs_per_iteration=1, pruning_algorithm='level'):
+    def __init__(self, model, config_list, optimizer, trainer, criterion,
+                 num_iterations=10, epochs_per_iteration=1, pruning_algorithm='level'):
         super().__init__(model, config_list, optimizer=optimizer, trainer=trainer, criterion=criterion,
                          num_iterations=num_iterations, epochs_per_iteration=epochs_per_iteration)
         assert isinstance(optimizer, torch.optim.Optimizer), "AGP pruner is an iterative pruner, please pass optimizer of the model to it"

nni/algorithms/compression/pytorch/pruning/sensitivity_pruner.py

@@ -10,9 +10,10 @@
 from schema import And, Optional
 from nni.compression.pytorch.compressor import Pruner
 from nni.compression.pytorch.utils.config_validation import CompressorSchema
-from .constants_pruner import PRUNER_DICT
 from nni.compression.pytorch.utils.sensitivity_analysis import SensitivityAnalysis
 
+from .constants_pruner import PRUNER_DICT
+
 
 MAX_PRUNE_RATIO_PER_ITER = 0.95
 

nni/algorithms/compression/pytorch/quantization/quantizers.py

@@ -245,7 +245,7 @@ def _dequantize(self, op, quantized_val):
     def quantize_weight(self, wrapper, **kwargs):
         config = wrapper.config
         module = wrapper.module
-        input = kwargs['input_tensor']
+        input = kwargs['input_tensor']  # pylint: disable=redefined-builtin
         weight = copy.deepcopy(wrapper.module.old_weight.data)
         weight_bits = get_bits_length(config, 'weight')
         quant_start_step = config.get('quant_start_step', 0)
@@ -304,7 +304,8 @@ def quantize_output(self, output, wrapper, **kwargs):
                                                                        module.ema_decay)
             module.tracked_max_activation = update_ema(module.tracked_max_activation, current_max,
                                                                        module.ema_decay)
-            module.scale, module.zero_point = update_quantization_param(output_bits, module.tracked_min_activation, module.tracked_max_activation)
+            module.scale, module.zero_point = update_quantization_param(
+                output_bits, module.tracked_min_activation, module.tracked_max_activation)
         out = self._quantize(output_bits, module, output)
         out = self._dequantize(module, out)
         return out

nni/algorithms/hpo/dngo_tuner.py

@@ -2,11 +2,12 @@
 
 import numpy as np
 import torch
+from pybnn import DNGO
+from torch.distributions import Normal
+
 import nni.parameter_expressions as parameter_expressions
 from nni import ClassArgsValidator
 from nni.tuner import Tuner
-from pybnn import DNGO
-from torch.distributions import Normal
 
 _logger = logging.getLogger(__name__)
 

nni/algorithms/hpo/hyperband_advisor.py

@@ -265,8 +265,10 @@ def validate_class_args(self, **kwargs):
         }).validate(kwargs)
 
 class Hyperband(MsgDispatcherBase):
-    """Hyperband inherit from MsgDispatcherBase rather than Tuner, because it integrates both tuner's functions and assessor's functions.
-    This is an implementation that could fully leverage available resources or follow the algorithm process, i.e., high parallelism or serial.
+    """
+    Hyperband inherit from MsgDispatcherBase rather than Tuner, because it integrates both tuner's functions and assessor's functions.
+    This is an implementation that could fully leverage available resources or follow the algorithm process,
+    i.e., high parallelism or serial.
     A single execution of Hyperband takes a finite budget of (s_max + 1)B.
 
     Parameters
@@ -346,7 +348,8 @@ def _get_one_trial_job(self):
                     self.curr_hb += 1
                 _logger.debug('create a new bracket, self.curr_hb=%d, self.curr_s=%d', self.curr_hb, self.curr_s)
                 self.curr_bracket_id = '{}-{}'.format(self.curr_hb, self.curr_s)
-                self.brackets[self.curr_bracket_id] = Bracket(self.curr_bracket_id, self.curr_s, self.s_max, self.eta, self.R, self.optimize_mode)
+                self.brackets[self.curr_bracket_id] = Bracket(
+                    self.curr_bracket_id, self.curr_s, self.s_max, self.eta, self.R, self.optimize_mode)
                 next_n, next_r = self.brackets[self.curr_bracket_id].get_n_r()
                 _logger.debug('new bracket, next_n=%d, next_r=%d', next_n, next_r)
                 assert self.searchspace_json is not None and self.random_state is not None

nni/algorithms/hpo/metis_tuner/metis_tuner.py

@@ -15,6 +15,8 @@
 from schema import Schema, Optional
 
 from nni import ClassArgsValidator
+from nni.tuner import Tuner
+from nni.utils import OptimizeMode, extract_scalar_reward
 from . import lib_constraint_summation
 from . import lib_data
 from .Regression_GMM import CreateModel as gmm_create_model
@@ -23,8 +25,6 @@
 from .Regression_GP import OutlierDetection as gp_outlier_detection
 from .Regression_GP import Prediction as gp_prediction
 from .Regression_GP import Selection as gp_selection
-from nni.tuner import Tuner
-from nni.utils import OptimizeMode, extract_scalar_reward
 
 logger = logging.getLogger("Metis_Tuner_AutoML")
 

nni/algorithms/hpo/networkmorphism_tuner/networkmorphism_tuner.py

@@ -8,13 +8,13 @@
 import logging
 import os
 from schema import Optional, Schema
+from nni import ClassArgsValidator
 from nni.tuner import Tuner
 from nni.utils import OptimizeMode, extract_scalar_reward
 from .bayesian import BayesianOptimizer
 from .nn import CnnGenerator, MlpGenerator
 from .utils import Constant
 from .graph import graph_to_json, json_to_graph
-from nni import ClassArgsValidator
 
 logger = logging.getLogger("NetworkMorphism_AutoML")
 

nni/algorithms/nas/pytorch/cream/trainer.py

@@ -1,11 +1,10 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
 
-import os
-import torch
 import logging
-
 from copy import deepcopy
+
+import torch
 from nni.nas.pytorch.trainer import Trainer
 from nni.nas.pytorch.utils import AverageMeterGroup
 
@@ -209,8 +208,8 @@ def _simulate_sgd_update(self, w, g, optimizer):
         return g * optimizer.param_groups[-1]['lr'] + w
 
     # split training images into several slices
-    def _get_minibatch_input(self, input):
-        slice = self.slices
+    def _get_minibatch_input(self, input):  # pylint: disable=redefined-builtin
+        slice = self.slices  # pylint: disable=redefined-builtin
         x = deepcopy(input[:slice].clone().detach())
         return x
 
@@ -259,8 +258,8 @@ def _cross_entropy_loss_with_soft_target(self, pred, soft_target):
         return torch.mean(torch.sum(- soft_target * logsoftmax(pred), 1))
 
     # forward validation data
-    def _forward_validation(self, input, target):
-        slice = self.slices
+    def _forward_validation(self, input, target):  # pylint: disable=redefined-builtin
+        slice = self.slices  # pylint: disable=redefined-builtin
         x = input[slice:slice * 2].clone()
 
         self._replace_mutator_cand(self.current_student_arch)
@@ -281,7 +280,7 @@ def _replace_mutator_cand(self, cand):
         self.mutator._cache = cand
 
     # update meta matching networks
-    def _run_update(self, input, target, batch_idx):
+    def _run_update(self, input, target, batch_idx):  # pylint: disable=redefined-builtin
         if self._isUpdateMeta(batch_idx):
             x = self._get_minibatch_input(input)
 

nni/algorithms/nas/pytorch/fbnet/trainer.py

@@ -128,7 +128,7 @@ def _layer_choice_sample(self):
         layer_id = 0
         for i, stage_name in enumerate(stages):
             ops_names = [op for op in self.lookup_table.lut_ops[stage_name]]
-            for j in range(stage_lnum[i]):
+            for _ in range(stage_lnum[i]):
                 searched_op = ops_names[choice_ids[layer_id]]
                 choice_names.append(searched_op)
                 layer_id += 1

nni/algorithms/nas/pytorch/fbnet/utils.py

@@ -3,7 +3,6 @@
 
 from __future__ import absolute_import, division, print_function
 
-import gc  # noqa: F401
 import os
 import timeit
 import torch
@@ -159,7 +158,7 @@ def supernet_sample(model, state_dict, sampled_arch=[], lookup_table=None):
         layer_id = 0
         for i, stage in enumerate(stages):
             ops_names = [op_name for op_name in lookup_table.lut_ops[stage]]
-            for j in range(stage_lnum[i]):
+            for _ in range(stage_lnum[i]):
                 searched_op = sampled_arch[layer_id]
                 op_i = ops_names.index(searched_op)
                 replace.append(

nni/algorithms/nas/tensorflow/classic_nas/mutator.py

@@ -1,6 +1,8 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
 
+# pylint: skip-file
+
 import json
 import logging
 import os

nni/algorithms/nas/tensorflow/enas/mutator.py

@@ -1,6 +1,8 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
 
+# pylint: skip-file
+
 import tensorflow as tf
 from tensorflow.keras.layers import Dense, Embedding, LSTMCell, RNN
 from tensorflow.keras.losses import SparseCategoricalCrossentropy, Reduction

nni/experiment/launcher.py

@@ -12,7 +12,7 @@
 
 import colorama
 
-import nni_node  # pylint: disable=import-error
+import nni_node  # pylint: disable=wrong-import-order, import-error
 import nni.runtime.protocol
 
 from .config import ExperimentConfig

nni/runtime/log.py

@@ -1,17 +1,15 @@
+import logging
+import sys
 from datetime import datetime
 from io import TextIOBase
-import logging
 from logging import FileHandler, Formatter, Handler, StreamHandler
 from pathlib import Path
-import sys
-import time
 from typing import Optional
 
 import colorama
 
 from .env_vars import dispatcher_env_vars, trial_env_vars
 
-
 handlers = {}
 
 log_format = '[%(asctime)s] %(levelname)s (%(name)s/%(threadName)s) %(message)s'

nni/runtime/msg_dispatcher.py

@@ -6,11 +6,12 @@
 import json_tricks
 
 from nni import NoMoreTrialError
-from .protocol import CommandType, send
-from .msg_dispatcher_base import MsgDispatcherBase
 from nni.assessor import AssessResult
+
 from .common import multi_thread_enabled, multi_phase_enabled
 from .env_vars import dispatcher_env_vars
+from .msg_dispatcher_base import MsgDispatcherBase
+from .protocol import CommandType, send
 from ..utils import MetricType, to_json
 
 _logger = logging.getLogger(__name__)

nni/runtime/platform/local.py

@@ -7,8 +7,8 @@
 import time
 import subprocess
 
-from ..env_vars import trial_env_vars
 from nni.utils import to_json
+from ..env_vars import trial_env_vars
 
 _sysdir = trial_env_vars.NNI_SYS_DIR
 if not os.path.exists(os.path.join(_sysdir, '.nni')):

nni/runtime/platform/standalone.py

@@ -1,9 +1,10 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
 
-import colorama
 import logging
 import warnings
+
+import colorama
 import json_tricks
 
 __all__ = [

nni/tools/nnictl/launcher.py

@@ -14,7 +14,7 @@
 from nni.experiment.config import ExperimentConfig, convert
 from nni.tools.annotation import expand_annotations, generate_search_space
 from nni.tools.package_utils import get_builtin_module_class_name
-import nni_node  # pylint: disable=import-error
+import nni_node  # pylint: disable=import-error, wrong-import-order
 from .launcher_utils import validate_all_content
 from .rest_utils import rest_put, rest_post, check_rest_server, check_response
 from .url_utils import cluster_metadata_url, experiment_url, get_local_urls, set_prefix_url