From e21a6984cc66df1c66b47107e81c0ed1c7ba58d5 Mon Sep 17 00:00:00 2001 From: liuzhe-lz <40699903+liuzhe-lz@users.noreply.github.com> Date: Mon, 26 Oct 2020 23:21:59 +0800 Subject: [PATCH] [v2.0] Refactor code hierarchy (part 2) (#2987) --- .gitignore | 10 +- Makefile | 74 ------ azure-pipelines.yml | 14 +- docs/en_US/nnicli_ref.md | 12 +- nni/__init__.py | 4 +- nni/__main__.py | 6 +- .../compression/pytorch}/pruning/__init__.py | 0 .../pytorch}/pruning/admm_pruner.py | 2 +- .../compression/pytorch}/pruning/agp.py | 4 +- .../pytorch}/pruning/amc/__init__.py | 0 .../pytorch}/pruning/amc/amc_pruner.py | 2 +- .../pruning/amc/channel_pruning_env.py | 2 +- .../pytorch/pruning/amc/lib}/__init__.py | 0 .../pytorch}/pruning/amc/lib/agent.py | 0 .../pytorch}/pruning/amc/lib/memory.py | 0 .../pytorch}/pruning/amc/lib/net_measure.py | 0 .../pytorch}/pruning/amc/lib/utils.py | 0 .../pytorch}/pruning/apply_compression.py | 0 .../pytorch}/pruning/auto_compress_pruner.py | 6 +- .../compression/pytorch}/pruning/constants.py | 2 +- .../pytorch}/pruning/constants_pruner.py | 0 .../pytorch}/pruning/finegrained_pruning.py | 0 .../pytorch}/pruning/lottery_ticket.py | 4 +- .../pytorch}/pruning/net_adapt_pruner.py | 6 +- .../compression/pytorch}/pruning/one_shot.py | 8 +- .../pytorch}/pruning/sensitivity_pruner.py | 6 +- .../pruning/simulated_annealing_pruner.py | 4 +- .../pytorch}/pruning/structured_pruning.py | 0 .../pytorch}/pruning/weight_masker.py | 0 .../pytorch}/quantization/__init__.py | 0 .../pytorch}/quantization/quantizers.py | 4 +- .../tensorflow/pruning/__init__.py | 0 .../tensorflow/pruning/one_shot.py | 2 +- .../gbdt_selector/__init__.py | 0 .../gbdt_selector/gbdt_selector.py | 0 .../gbdt_selector/requirements.txt | 0 .../gradient_selector/__init__.py | 0 .../gradient_selector/constants.py | 0 .../gradient_selector/fginitialize.py | 0 .../gradient_selector/fgtrain.py | 0 .../gradient_selector/gradient_selector.py | 0 .../gradient_selector/learnability.py | 0 .../gradient_selector/requirements.txt | 0 .../gradient_selector/syssettings.py | 0 .../gradient_selector/utils.py | 0 .../hpo/batch_tuner}/__init__.py | 0 .../hpo}/batch_tuner/batch_tuner.py | 0 .../hpo/bohb_advisor}/__init__.py | 0 .../hpo}/bohb_advisor/bohb_advisor.py | 6 +- .../hpo}/bohb_advisor/config_generator.py | 0 .../hpo}/bohb_advisor/requirements.txt | 0 .../hpo}/curvefitting_assessor/__init__.py | 0 .../curvefitting_assessor.py | 0 .../curvefitting_assessor/curvefunctions.py | 0 .../curvefitting_assessor/model_factory.py | 0 .../hpo/evolution_tuner}/__init__.py | 0 .../hpo}/evolution_tuner/evolution_tuner.py | 0 .../hpo/gp_tuner}/__init__.py | 0 nni/{ => algorithms/hpo}/gp_tuner/gp_tuner.py | 0 .../hpo}/gp_tuner/target_space.py | 0 nni/{ => algorithms/hpo}/gp_tuner/util.py | 0 .../hpo}/gridsearch_tuner/__init__.py | 0 .../hpo}/gridsearch_tuner/gridsearch_tuner.py | 0 .../hpo/hyperband_advisor}/__init__.py | 0 .../hyperband_advisor/hyperband_advisor.py | 6 +- .../hpo}/hyperband_advisor/requirements.txt | 0 .../hpo/hyperopt_tuner}/__init__.py | 0 .../hpo}/hyperopt_tuner/hyperopt_tuner.py | 0 .../hpo}/medianstop_assessor/__init__.py | 0 .../medianstop_assessor.py | 0 .../hpo}/medianstop_assessor/test.py | 0 .../metis_tuner/Regression_GMM/CreateModel.py | 0 .../metis_tuner/Regression_GMM/Selection.py | 4 +- .../metis_tuner/Regression_GMM}/__init__.py | 0 .../metis_tuner/Regression_GP/CreateModel.py | 0 .../Regression_GP/OutlierDetection.py | 4 +- .../metis_tuner/Regression_GP/Prediction.py | 0 .../metis_tuner/Regression_GP/Selection.py | 8 +- .../metis_tuner/Regression_GP}/__init__.py | 0 .../hpo/metis_tuner}/__init__.py | 0 .../metis_tuner/lib_acquisition_function.py | 2 +- .../metis_tuner/lib_constraint_summation.py | 0 .../hpo}/metis_tuner/lib_data.py | 0 .../hpo}/metis_tuner/metis_tuner.py | 16 +- .../hpo}/metis_tuner/requirments.txt | 0 .../hpo/networkmorphism_tuner}/__init__.py | 0 .../hpo}/networkmorphism_tuner/bayesian.py | 6 +- .../hpo}/networkmorphism_tuner/graph.py | 6 +- .../graph_transformer.py | 6 +- .../layer_transformer.py | 2 +- .../hpo}/networkmorphism_tuner/layers.py | 2 +- .../networkmorphism_tuner.py | 8 +- .../hpo}/networkmorphism_tuner/nn.py | 6 +- .../hpo}/networkmorphism_tuner/utils.py | 0 .../hpo/pbt_tuner}/__init__.py | 0 .../hpo}/pbt_tuner/pbt_tuner.py | 0 .../hpo}/ppo_tuner/__init__.py | 0 nni/{ => algorithms/hpo}/ppo_tuner/distri.py | 0 nni/{ => algorithms/hpo}/ppo_tuner/model.py | 0 nni/{ => algorithms/hpo}/ppo_tuner/policy.py | 0 .../hpo}/ppo_tuner/ppo_tuner.py | 0 .../hpo}/ppo_tuner/requirements.txt | 0 nni/{ => algorithms/hpo}/ppo_tuner/util.py | 0 .../regularized_evolution_tuner/__init__.py | 0 .../regularized_evolution_tuner.py | 0 .../hpo}/smac_tuner/__init__.py | 0 .../hpo}/smac_tuner/convert_ss_to_scenario.py | 0 .../hpo}/smac_tuner/requirements.txt | 0 .../hpo}/smac_tuner/smac_tuner.py | 0 .../nas/pytorch/cdarts/__init__.py | 0 .../nas/pytorch/cdarts/mutator.py | 0 .../nas/pytorch/cdarts/trainer.py | 2 +- .../nas/pytorch/cdarts/utils.py | 0 .../nas/pytorch/classic_nas/__init__.py | 0 .../nas/pytorch/classic_nas/mutator.py | 2 +- .../nas/pytorch/darts/__init__.py | 0 .../nas/pytorch/darts/mutator.py | 0 .../nas/pytorch/darts/trainer.py | 0 .../nas/pytorch/enas/__init__.py | 0 .../nas/pytorch/enas/mutator.py | 0 .../nas/pytorch/enas/trainer.py | 0 .../nas/pytorch/pdarts/__init__.py | 0 .../nas/pytorch/pdarts/mutator.py | 2 +- .../nas/pytorch/pdarts/trainer.py | 2 +- .../nas/pytorch/proxylessnas/__init__.py | 0 .../nas/pytorch/proxylessnas/mutator.py | 0 .../nas/pytorch/proxylessnas/trainer.py | 0 .../nas/pytorch/proxylessnas/utils.py | 0 .../nas/pytorch/random/__init__.py | 0 .../nas/pytorch/random/mutator.py | 0 .../nas/pytorch/spos/__init__.py | 0 .../nas/pytorch/spos/evolution.py | 2 +- .../nas/pytorch/spos/mutator.py | 0 .../nas/pytorch/spos/trainer.py | 0 .../nas/tensorflow/classic_nas/__init__.py | 0 .../nas/tensorflow/classic_nas/mutator.py | 2 +- .../nas/tensorflow/enas/__init__.py | 0 .../nas/tensorflow/enas/mutator.py | 0 .../nas/tensorflow/enas/trainer.py | 0 nni/assessor.py | 4 +- .../graph_utils.py} | 0 nni/{ => common}/nas_utils.py | 2 +- .../{torch => pytorch}/__init__.py | 2 - .../{torch => pytorch}/compressor.py | 0 .../{torch => pytorch}/default_layers.py | 0 .../{torch => pytorch}/speedup/__init__.py | 0 .../speedup/compress_modules.py | 0 .../{torch => pytorch}/speedup/compressor.py | 6 +- .../{torch => pytorch}/speedup/infer_shape.py | 0 .../pytorch/utils}/__init__.py | 0 .../utils/config_validation.py | 0 .../{torch => pytorch}/utils/counter.py | 4 +- .../{torch => pytorch}/utils/mask_conflict.py | 0 .../utils/num_param_counter.py | 0 .../utils/sensitivity_analysis.py | 4 +- .../utils/shape_dependency.py | 2 +- .../{torch => pytorch}/utils/utils.py | 0 nni/compression/tensorflow/__init__.py | 1 - nni/constants.py | 91 ------- nni/{nnicli => experiment}/__init__.py | 0 nni/{nnicli => experiment}/nni_client.py | 10 +- nni/nas/pytorch/mutator.py | 2 +- nni/{ => runtime}/common.py | 4 + nni/{ => runtime}/env_vars.py | 0 nni/{ => runtime}/msg_dispatcher.py | 4 +- nni/{ => runtime}/msg_dispatcher_base.py | 4 +- nni/{ => runtime}/platform/__init__.py | 0 nni/{ => runtime}/platform/local.py | 2 +- nni/{ => runtime}/platform/standalone.py | 0 nni/{ => runtime}/platform/test.py | 0 nni/{ => runtime}/protocol.py | 6 +- nni/smartparam.py | 4 +- .../annotation}/.gitignore | 0 .../annotation}/__init__.py | 0 .../annotation}/code_generator.py | 0 .../annotation}/search_space_generator.py | 0 .../annotation}/specific_code_generator.py | 2 +- .../annotation}/utils.py | 0 .../gpu_tool}/__init__.py | 0 .../gpu_tool}/gpu_metrics_collector.py | 0 .../nnictl}/__init__.py | 0 .../nnictl}/command_utils.py | 0 nni/{nni_cmd => tools/nnictl}/common_utils.py | 0 .../nnictl}/config_schema.py | 2 +- nni/{nni_cmd => tools/nnictl}/config_utils.py | 0 nni/{nni_cmd => tools/nnictl}/constants.py | 0 nni/{nni_cmd => tools/nnictl}/launcher.py | 4 +- .../nnictl}/launcher_utils.py | 0 nni/{nni_cmd => tools/nnictl}/nnictl.py | 0 nni/{nni_cmd => tools/nnictl}/nnictl_utils.py | 2 +- .../nnictl}/package_management.py | 4 +- nni/{nni_cmd => tools/nnictl}/rest_utils.py | 0 nni/{nni_cmd => tools/nnictl}/ssh_utils.py | 0 .../nnictl}/tensorboard_utils.py | 0 nni/{nni_cmd => tools/nnictl}/updater.py | 0 nni/{nni_cmd => tools/nnictl}/url_utils.py | 0 .../package_utils/__init__.py} | 6 +- nni/tools/package_utils/constants.py | 91 +++++++ .../trial_tool}/__init__.py | 0 .../trial_tool}/aml_channel.py | 0 .../trial_tool}/base_channel.py | 0 .../trial_tool}/commands.py | 0 .../trial_tool}/constants.py | 0 .../trial_tool}/file_channel.py | 0 .../trial_tool}/gpu.py | 0 .../trial_tool}/hdfsClientUtility.py | 0 .../trial_tool}/log_utils.py | 0 .../trial_tool}/rest_utils.py | 0 .../trial_tool}/trial.py | 0 .../trial_tool}/trial_keeper.py | 0 .../trial_tool}/trial_runner.py | 0 .../trial_tool}/url_utils.py | 0 .../trial_tool}/web_channel.py | 0 nni/trial.py | 4 +- nni/tuner.py | 16 +- nni/utils.py | 4 +- setup.py | 222 +++++++++++++---- setup_ts.py | 229 ++++++++++++++++++ test/.coveragerc | 2 +- test/config/integration_tests.yml | 4 +- test/config/integration_tests_tf2.yml | 4 +- test/config/pr_tests.yml | 4 +- test/nni_test/nnitest/validators.py | 4 +- ts/nni_manager/package.json | 2 +- .../training_service/common/gpuData.ts | 2 +- .../training_service/common/util.ts | 2 +- .../training_service/dlts/dltsData.ts | 2 +- .../kubernetes/kubernetesData.ts | 2 +- .../training_service/pai/paiK8S/paiK8SData.ts | 2 +- .../remote_machine/extends/linuxCommands.ts | 4 +- .../remote_machine/extends/windowsCommands.ts | 4 +- .../reusable/trialDispatcher.ts | 3 +- 232 files changed, 671 insertions(+), 374 deletions(-) delete mode 100644 Makefile rename nni/{compression/torch => algorithms/compression/pytorch}/pruning/__init__.py (100%) rename nni/{compression/torch => algorithms/compression/pytorch}/pruning/admm_pruner.py (98%) rename nni/{compression/torch => algorithms/compression/pytorch}/pruning/agp.py (97%) rename nni/{compression/torch => algorithms/compression/pytorch}/pruning/amc/__init__.py (100%) rename nni/{compression/torch => algorithms/compression/pytorch}/pruning/amc/amc_pruner.py (99%) rename nni/{compression/torch => algorithms/compression/pytorch}/pruning/amc/channel_pruning_env.py (99%) rename nni/{batch_tuner => algorithms/compression/pytorch/pruning/amc/lib}/__init__.py (100%) rename nni/{compression/torch => algorithms/compression/pytorch}/pruning/amc/lib/agent.py (100%) rename nni/{compression/torch => algorithms/compression/pytorch}/pruning/amc/lib/memory.py (100%) rename nni/{compression/torch => algorithms/compression/pytorch}/pruning/amc/lib/net_measure.py (100%) rename nni/{compression/torch => algorithms/compression/pytorch}/pruning/amc/lib/utils.py (100%) rename nni/{compression/torch => algorithms/compression/pytorch}/pruning/apply_compression.py (100%) rename nni/{compression/torch => algorithms/compression/pytorch}/pruning/auto_compress_pruner.py (98%) rename nni/{compression/torch => algorithms/compression/pytorch}/pruning/constants.py (87%) rename nni/{compression/torch => algorithms/compression/pytorch}/pruning/constants_pruner.py (100%) rename nni/{compression/torch => algorithms/compression/pytorch}/pruning/finegrained_pruning.py (100%) rename nni/{compression/torch => algorithms/compression/pytorch}/pruning/lottery_ticket.py (97%) rename nni/{compression/torch => algorithms/compression/pytorch}/pruning/net_adapt_pruner.py (98%) rename nni/{compression/torch => algorithms/compression/pytorch}/pruning/one_shot.py (98%) rename nni/{compression/torch => algorithms/compression/pytorch}/pruning/sensitivity_pruner.py (98%) rename nni/{compression/torch => algorithms/compression/pytorch}/pruning/simulated_annealing_pruner.py (99%) rename nni/{compression/torch => algorithms/compression/pytorch}/pruning/structured_pruning.py (100%) rename nni/{compression/torch => algorithms/compression/pytorch}/pruning/weight_masker.py (100%) rename nni/{compression/torch => algorithms/compression/pytorch}/quantization/__init__.py (100%) rename nni/{compression/torch => algorithms/compression/pytorch}/quantization/quantizers.py (98%) rename nni/{ => algorithms}/compression/tensorflow/pruning/__init__.py (100%) rename nni/{ => algorithms}/compression/tensorflow/pruning/one_shot.py (97%) rename nni/{ => algorithms}/feature_engineering/gbdt_selector/__init__.py (100%) rename nni/{ => algorithms}/feature_engineering/gbdt_selector/gbdt_selector.py (100%) rename nni/{ => algorithms}/feature_engineering/gbdt_selector/requirements.txt (100%) rename nni/{ => algorithms}/feature_engineering/gradient_selector/__init__.py (100%) rename nni/{ => algorithms}/feature_engineering/gradient_selector/constants.py (100%) rename nni/{ => algorithms}/feature_engineering/gradient_selector/fginitialize.py (100%) rename nni/{ => algorithms}/feature_engineering/gradient_selector/fgtrain.py (100%) rename nni/{ => algorithms}/feature_engineering/gradient_selector/gradient_selector.py (100%) rename nni/{ => algorithms}/feature_engineering/gradient_selector/learnability.py (100%) rename nni/{ => algorithms}/feature_engineering/gradient_selector/requirements.txt (100%) rename nni/{ => algorithms}/feature_engineering/gradient_selector/syssettings.py (100%) rename nni/{ => algorithms}/feature_engineering/gradient_selector/utils.py (100%) rename nni/{bohb_advisor => algorithms/hpo/batch_tuner}/__init__.py (100%) rename nni/{ => algorithms/hpo}/batch_tuner/batch_tuner.py (100%) rename nni/{compression/torch/pruning/amc/lib => algorithms/hpo/bohb_advisor}/__init__.py (100%) rename nni/{ => algorithms/hpo}/bohb_advisor/bohb_advisor.py (99%) rename nni/{ => algorithms/hpo}/bohb_advisor/config_generator.py (100%) rename nni/{ => algorithms/hpo}/bohb_advisor/requirements.txt (100%) rename nni/{ => algorithms/hpo}/curvefitting_assessor/__init__.py (100%) rename nni/{ => algorithms/hpo}/curvefitting_assessor/curvefitting_assessor.py (100%) rename nni/{ => algorithms/hpo}/curvefitting_assessor/curvefunctions.py (100%) rename nni/{ => algorithms/hpo}/curvefitting_assessor/model_factory.py (100%) rename nni/{compression/torch/utils => algorithms/hpo/evolution_tuner}/__init__.py (100%) rename nni/{ => algorithms/hpo}/evolution_tuner/evolution_tuner.py (100%) rename nni/{evolution_tuner => algorithms/hpo/gp_tuner}/__init__.py (100%) rename nni/{ => algorithms/hpo}/gp_tuner/gp_tuner.py (100%) rename nni/{ => algorithms/hpo}/gp_tuner/target_space.py (100%) rename nni/{ => algorithms/hpo}/gp_tuner/util.py (100%) rename nni/{ => algorithms/hpo}/gridsearch_tuner/__init__.py (100%) rename nni/{ => algorithms/hpo}/gridsearch_tuner/gridsearch_tuner.py (100%) rename nni/{gp_tuner => algorithms/hpo/hyperband_advisor}/__init__.py (100%) rename nni/{ => algorithms/hpo}/hyperband_advisor/hyperband_advisor.py (99%) rename nni/{ => algorithms/hpo}/hyperband_advisor/requirements.txt (100%) rename nni/{hyperband_advisor => algorithms/hpo/hyperopt_tuner}/__init__.py (100%) rename nni/{ => algorithms/hpo}/hyperopt_tuner/hyperopt_tuner.py (100%) rename nni/{ => algorithms/hpo}/medianstop_assessor/__init__.py (100%) rename nni/{ => algorithms/hpo}/medianstop_assessor/medianstop_assessor.py (100%) rename nni/{ => algorithms/hpo}/medianstop_assessor/test.py (100%) rename nni/{ => algorithms/hpo}/metis_tuner/Regression_GMM/CreateModel.py (100%) rename nni/{ => algorithms/hpo}/metis_tuner/Regression_GMM/Selection.py (95%) rename nni/{hyperopt_tuner => algorithms/hpo/metis_tuner/Regression_GMM}/__init__.py (100%) rename nni/{ => algorithms/hpo}/metis_tuner/Regression_GP/CreateModel.py (100%) rename nni/{ => algorithms/hpo}/metis_tuner/Regression_GP/OutlierDetection.py (96%) rename nni/{ => algorithms/hpo}/metis_tuner/Regression_GP/Prediction.py (100%) rename nni/{ => algorithms/hpo}/metis_tuner/Regression_GP/Selection.py (93%) rename nni/{metis_tuner/Regression_GMM => algorithms/hpo/metis_tuner/Regression_GP}/__init__.py (100%) rename nni/{metis_tuner/Regression_GP => algorithms/hpo/metis_tuner}/__init__.py (100%) rename nni/{ => algorithms/hpo}/metis_tuner/lib_acquisition_function.py (99%) rename nni/{ => algorithms/hpo}/metis_tuner/lib_constraint_summation.py (100%) rename nni/{ => algorithms/hpo}/metis_tuner/lib_data.py (100%) rename nni/{ => algorithms/hpo}/metis_tuner/metis_tuner.py (97%) rename nni/{ => algorithms/hpo}/metis_tuner/requirments.txt (100%) rename nni/{metis_tuner => algorithms/hpo/networkmorphism_tuner}/__init__.py (100%) rename nni/{ => algorithms/hpo}/networkmorphism_tuner/bayesian.py (98%) rename nni/{ => algorithms/hpo}/networkmorphism_tuner/graph.py (99%) rename nni/{ => algorithms/hpo}/networkmorphism_tuner/graph_transformer.py (96%) rename nni/{ => algorithms/hpo}/networkmorphism_tuner/layer_transformer.py (99%) rename nni/{ => algorithms/hpo}/networkmorphism_tuner/layers.py (99%) rename nni/{ => algorithms/hpo}/networkmorphism_tuner/networkmorphism_tuner.py (97%) rename nni/{ => algorithms/hpo}/networkmorphism_tuner/nn.py (97%) rename nni/{ => algorithms/hpo}/networkmorphism_tuner/utils.py (100%) rename nni/{networkmorphism_tuner => algorithms/hpo/pbt_tuner}/__init__.py (100%) rename nni/{ => algorithms/hpo}/pbt_tuner/pbt_tuner.py (100%) rename nni/{ => algorithms/hpo}/ppo_tuner/__init__.py (100%) rename nni/{ => algorithms/hpo}/ppo_tuner/distri.py (100%) rename nni/{ => algorithms/hpo}/ppo_tuner/model.py (100%) rename nni/{ => algorithms/hpo}/ppo_tuner/policy.py (100%) rename nni/{ => algorithms/hpo}/ppo_tuner/ppo_tuner.py (100%) rename nni/{ => algorithms/hpo}/ppo_tuner/requirements.txt (100%) rename nni/{ => algorithms/hpo}/ppo_tuner/util.py (100%) rename nni/{ => algorithms/hpo}/regularized_evolution_tuner/__init__.py (100%) rename nni/{ => algorithms/hpo}/regularized_evolution_tuner/regularized_evolution_tuner.py (100%) rename nni/{ => algorithms/hpo}/smac_tuner/__init__.py (100%) rename nni/{ => algorithms/hpo}/smac_tuner/convert_ss_to_scenario.py (100%) rename nni/{ => algorithms/hpo}/smac_tuner/requirements.txt (100%) rename nni/{ => algorithms/hpo}/smac_tuner/smac_tuner.py (100%) rename nni/{ => algorithms}/nas/pytorch/cdarts/__init__.py (100%) rename nni/{ => algorithms}/nas/pytorch/cdarts/mutator.py (100%) rename nni/{ => algorithms}/nas/pytorch/cdarts/trainer.py (98%) rename nni/{ => algorithms}/nas/pytorch/cdarts/utils.py (100%) rename nni/{ => algorithms}/nas/pytorch/classic_nas/__init__.py (100%) rename nni/{ => algorithms}/nas/pytorch/classic_nas/mutator.py (99%) rename nni/{ => algorithms}/nas/pytorch/darts/__init__.py (100%) rename nni/{ => algorithms}/nas/pytorch/darts/mutator.py (100%) rename nni/{ => algorithms}/nas/pytorch/darts/trainer.py (100%) rename nni/{ => algorithms}/nas/pytorch/enas/__init__.py (100%) rename nni/{ => algorithms}/nas/pytorch/enas/mutator.py (100%) rename nni/{ => algorithms}/nas/pytorch/enas/trainer.py (100%) rename nni/{ => algorithms}/nas/pytorch/pdarts/__init__.py (100%) rename nni/{ => algorithms}/nas/pytorch/pdarts/mutator.py (98%) rename nni/{ => algorithms}/nas/pytorch/pdarts/trainer.py (98%) rename nni/{ => algorithms}/nas/pytorch/proxylessnas/__init__.py (100%) rename nni/{ => algorithms}/nas/pytorch/proxylessnas/mutator.py (100%) rename nni/{ => algorithms}/nas/pytorch/proxylessnas/trainer.py (100%) rename nni/{ => algorithms}/nas/pytorch/proxylessnas/utils.py (100%) rename nni/{ => algorithms}/nas/pytorch/random/__init__.py (100%) rename nni/{ => algorithms}/nas/pytorch/random/mutator.py (100%) rename nni/{ => algorithms}/nas/pytorch/spos/__init__.py (100%) rename nni/{ => algorithms}/nas/pytorch/spos/evolution.py (99%) rename nni/{ => algorithms}/nas/pytorch/spos/mutator.py (100%) rename nni/{ => algorithms}/nas/pytorch/spos/trainer.py (100%) rename nni/{ => algorithms}/nas/tensorflow/classic_nas/__init__.py (100%) rename nni/{ => algorithms}/nas/tensorflow/classic_nas/mutator.py (99%) rename nni/{ => algorithms}/nas/tensorflow/enas/__init__.py (100%) rename nni/{ => algorithms}/nas/tensorflow/enas/mutator.py (100%) rename nni/{ => algorithms}/nas/tensorflow/enas/trainer.py (100%) rename nni/{_graph_utils.py => common/graph_utils.py} (100%) rename nni/{ => common}/nas_utils.py (99%) rename nni/compression/{torch => pytorch}/__init__.py (76%) rename nni/compression/{torch => pytorch}/compressor.py (100%) rename nni/compression/{torch => pytorch}/default_layers.py (100%) rename nni/compression/{torch => pytorch}/speedup/__init__.py (100%) rename nni/compression/{torch => pytorch}/speedup/compress_modules.py (100%) rename nni/compression/{torch => pytorch}/speedup/compressor.py (97%) rename nni/compression/{torch => pytorch}/speedup/infer_shape.py (100%) rename nni/{nni_cmd => compression/pytorch/utils}/__init__.py (100%) rename nni/compression/{torch => pytorch}/utils/config_validation.py (100%) rename nni/compression/{torch => pytorch}/utils/counter.py (98%) rename nni/compression/{torch => pytorch}/utils/mask_conflict.py (100%) rename nni/compression/{torch => pytorch}/utils/num_param_counter.py (100%) rename nni/compression/{torch => pytorch}/utils/sensitivity_analysis.py (98%) rename nni/compression/{torch => pytorch}/utils/shape_dependency.py (99%) rename nni/compression/{torch => pytorch}/utils/utils.py (100%) delete mode 100644 nni/constants.py rename nni/{nnicli => experiment}/__init__.py (100%) rename nni/{nnicli => experiment}/nni_client.py (98%) rename nni/{ => runtime}/common.py (95%) rename nni/{ => runtime}/env_vars.py (100%) rename nni/{ => runtime}/msg_dispatcher.py (99%) rename nni/{ => runtime}/msg_dispatcher_base.py (99%) rename nni/{ => runtime}/platform/__init__.py (100%) rename nni/{ => runtime}/platform/local.py (98%) rename nni/{ => runtime}/platform/standalone.py (100%) rename nni/{ => runtime}/platform/test.py (100%) rename nni/{ => runtime}/protocol.py (93%) rename nni/{nni_annotation => tools/annotation}/.gitignore (100%) rename nni/{nni_annotation => tools/annotation}/__init__.py (100%) rename nni/{nni_annotation => tools/annotation}/code_generator.py (100%) rename nni/{nni_annotation => tools/annotation}/search_space_generator.py (100%) rename nni/{nni_annotation => tools/annotation}/specific_code_generator.py (99%) rename nni/{nni_annotation => tools/annotation}/utils.py (100%) rename nni/{nni_gpu_tool => tools/gpu_tool}/__init__.py (100%) rename nni/{nni_gpu_tool => tools/gpu_tool}/gpu_metrics_collector.py (100%) rename nni/{nni_trial_tool => tools/nnictl}/__init__.py (100%) rename nni/{nni_cmd => tools/nnictl}/command_utils.py (100%) rename nni/{nni_cmd => tools/nnictl}/common_utils.py (100%) rename nni/{nni_cmd => tools/nnictl}/config_schema.py (99%) rename nni/{nni_cmd => tools/nnictl}/config_utils.py (100%) rename nni/{nni_cmd => tools/nnictl}/constants.py (100%) rename nni/{nni_cmd => tools/nnictl}/launcher.py (99%) rename nni/{nni_cmd => tools/nnictl}/launcher_utils.py (100%) rename nni/{nni_cmd => tools/nnictl}/nnictl.py (100%) rename nni/{nni_cmd => tools/nnictl}/nnictl_utils.py (99%) rename nni/{nni_cmd => tools/nnictl}/package_management.py (96%) rename nni/{nni_cmd => tools/nnictl}/rest_utils.py (100%) rename nni/{nni_cmd => tools/nnictl}/ssh_utils.py (100%) rename nni/{nni_cmd => tools/nnictl}/tensorboard_utils.py (100%) rename nni/{nni_cmd => tools/nnictl}/updater.py (100%) rename nni/{nni_cmd => tools/nnictl}/url_utils.py (100%) rename nni/{package_utils.py => tools/package_utils/__init__.py} (99%) create mode 100644 nni/tools/package_utils/constants.py rename nni/{pbt_tuner => tools/trial_tool}/__init__.py (100%) rename nni/{nni_trial_tool => tools/trial_tool}/aml_channel.py (100%) rename nni/{nni_trial_tool => tools/trial_tool}/base_channel.py (100%) rename nni/{nni_trial_tool => tools/trial_tool}/commands.py (100%) rename nni/{nni_trial_tool => tools/trial_tool}/constants.py (100%) rename nni/{nni_trial_tool => tools/trial_tool}/file_channel.py (100%) rename nni/{nni_trial_tool => tools/trial_tool}/gpu.py (100%) rename nni/{nni_trial_tool => tools/trial_tool}/hdfsClientUtility.py (100%) rename nni/{nni_trial_tool => tools/trial_tool}/log_utils.py (100%) rename nni/{nni_trial_tool => tools/trial_tool}/rest_utils.py (100%) rename nni/{nni_trial_tool => tools/trial_tool}/trial.py (100%) rename nni/{nni_trial_tool => tools/trial_tool}/trial_keeper.py (100%) rename nni/{nni_trial_tool => tools/trial_tool}/trial_runner.py (100%) rename nni/{nni_trial_tool => tools/trial_tool}/url_utils.py (100%) rename nni/{nni_trial_tool => tools/trial_tool}/web_channel.py (100%) create mode 100644 setup_ts.py diff --git a/.gitignore b/.gitignore index f59fcf6041..3d0e2bee92 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,10 @@ -/nni_node/node -/nni_node/node.exe -/toolchain +/nni_node/ +/toolchain/ + +# unit test generated files +/test/model_path/ +/test/temp.json +/test/ut/sdk/*.pth # Logs diff --git a/Makefile b/Makefile deleted file mode 100644 index d826aa5d54..0000000000 --- a/Makefile +++ /dev/null @@ -1,74 +0,0 @@ -SHELL := /bin/bash - -_INFO := $(shell echo -e '\033[1;36m') -_END := $(shell echo -e '\033[0m') - -## Detect OS -UNAME_S := $(shell uname -s) -ifeq ($(UNAME_S), Linux) - OS_SPEC := linux -else ifeq ($(UNAME_S), Darwin) - OS_SPEC := darwin -else - $(error platform $(UNAME_S) not supported) -endif - -_PWD := $(PWD) -YARN ?= PATH=$(_PWD)/nni_node:$${PATH} $(PWD)/toolchain/yarn/bin/yarn - - -.PHONY: build -build: nni_node/node create-symlinks - #$(_INFO) Building NNI Manager $(_END) - cd ts/nni_manager && $(YARN) && $(YARN) build - cp -rf ts/nni_manager/config ts/nni_manager/dist/ - #$(_INFO) Building WebUI $(_END) - cd ts/webui && $(YARN) && $(YARN) build - #$(_INFO) Building NAS UI $(_END) - cd ts/nasui && $(YARN) && $(YARN) build - - -nni_node/node: - mkdir -p toolchain - wget https://nodejs.org/dist/v10.22.1/node-v10.22.1-$(OS_SPEC)-x64.tar.xz -O toolchain/node.tar.xz - wget https://github.com/yarnpkg/yarn/releases/download/v1.22.10/yarn-v1.22.10.tar.gz -O toolchain/yarn.tar.gz - - mkdir -p toolchain/node toolchain/yarn - tar -xf toolchain/node.tar.xz -C toolchain/node --strip-components 1 - tar -xf toolchain/yarn.tar.gz -C toolchain/yarn --strip-components 1 - - cp toolchain/node/bin/node nni_node/ - - -.PHONY: create-symlinks -create-symlinks: - ln -sfT ../ts/nni_manager/dist/common nni_node/common - ln -sfT ../ts/nni_manager/dist/config nni_node/config - ln -sfT ../ts/nni_manager/dist/core nni_node/core - ln -sfT ../ts/nni_manager/dist/rest_server nni_node/rest_server - ln -sfT ../ts/nni_manager/dist/training_service nni_node/training_service - ln -sfT ../ts/nni_manager/dist/main.js nni_node/main.js - ln -sfT ../ts/nni_manager/package.json nni_node/package.json - ln -sfT ../ts/nni_manager/node_modules nni_node/node_modules - - ln -sfT ../ts/webui/build nni_node/static - - mkdir -p nni_node/nasui - ln -sfT ../../ts/nasui/build nni_node/nasui/build - ln -sfT ../../ts/nasui/server.js nni_node/nasui/server.js - - -.PHONY: clean -clean: - -rm nni_node/node - -rm -rf toolchain - -rm -rf nni_node/common - -rm -rf nni_node/config - -rm -rf nni_node/core - -rm -rf nni_node/rest_server - -rm -rf nni_node/training_service - -rm -rf nni_node/main.js - -rm -rf nni_node/package.json - -rm -rf nni_node/node_modules - -rm -rf nni_node/nasui - -rm -rf nni_node/static diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2745b46d50..147d0cd997 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -14,8 +14,7 @@ jobs: echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}" displayName: 'Install python tools' - script: | - make - python3 -m pip install -U -e . + python3 setup.py develop displayName: 'Install nni toolkit via source code' - script: | set -e @@ -46,10 +45,10 @@ jobs: cd test python3 nni_test/nnitest/run_tests.py --config config/pr_tests.yml displayName: 'Simple test' - - script: | - cd docs/en_US/ - sphinx-build -M html . _build -W - displayName: 'Sphinx Documentation Build check' + #- script: | + # cd docs/en_US/ + # sphinx-build -M html . _build -W + # displayName: 'Sphinx Documentation Build check' - job: 'ubuntu_1804_python36_legacy_torch_tf' pool: @@ -65,8 +64,7 @@ jobs: echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}" displayName: 'Install python tools' - script: | - make - python3 -m pip install -U -e . + python3 setup.py develop displayName: 'Install nni toolkit via source code' - script: | set -e diff --git a/docs/en_US/nnicli_ref.md b/docs/en_US/nnicli_ref.md index a7f6672dbf..d308ebca77 100644 --- a/docs/en_US/nnicli_ref.md +++ b/docs/en_US/nnicli_ref.md @@ -3,7 +3,7 @@ NNI client is a python API of `nnictl`, which implements the most commonly used commands. Users can use this API to control their experiments, collect experiment results and conduct advanced analyses based on experiment results in python code directly instead of using command line. Here is an example: ``` -from nni.nnicli import Experiment +from nni.experiment import Experiment # create an experiment instance exp = Experiment() @@ -28,14 +28,14 @@ exp.stop_experiment() ## References ```eval_rst -.. autoclass:: nni.nnicli.Experiment +.. autoclass:: nni.experiment.Experiment :members: -.. autoclass:: nni.nnicli.TrialJob +.. autoclass:: nni.experiment.TrialJob :members: -.. autoclass:: nni.nnicli.TrialHyperParameters +.. autoclass:: nni.experiment.TrialHyperParameters :members: -.. autoclass:: nni.nnicli.TrialMetricData +.. autoclass:: nni.experiment.TrialMetricData :members: -.. autoclass:: nni.nnicli.TrialResult +.. autoclass:: nni.experiment.TrialResult :members: ``` diff --git a/nni/__init__.py b/nni/__init__.py index f83650eee1..0630571ae6 100644 --- a/nni/__init__.py +++ b/nni/__init__.py @@ -3,13 +3,13 @@ __version__ = '999.0.0-developing' -from .env_vars import dispatcher_env_vars +from .runtime.env_vars import dispatcher_env_vars from .utils import ClassArgsValidator if dispatcher_env_vars.SDK_PROCESS != 'dispatcher': from .trial import * from .smartparam import * - from .nas_utils import training_update + from .common.nas_utils import training_update class NoMoreTrialError(Exception): def __init__(self, ErrorInfo): diff --git a/nni/__main__.py b/nni/__main__.py index 003a2bfe41..e3f982f42f 100644 --- a/nni/__main__.py +++ b/nni/__main__.py @@ -7,9 +7,9 @@ import json import base64 -from .common import enable_multi_thread, enable_multi_phase -from .msg_dispatcher import MsgDispatcher -from .package_utils import create_builtin_class_instance, create_customized_class_instance +from .runtime.common import enable_multi_thread, enable_multi_phase +from .runtime.msg_dispatcher import MsgDispatcher +from .tools.package_utils import create_builtin_class_instance, create_customized_class_instance logger = logging.getLogger('nni.main') logger.debug('START') diff --git a/nni/compression/torch/pruning/__init__.py b/nni/algorithms/compression/pytorch/pruning/__init__.py similarity index 100% rename from nni/compression/torch/pruning/__init__.py rename to nni/algorithms/compression/pytorch/pruning/__init__.py diff --git a/nni/compression/torch/pruning/admm_pruner.py b/nni/algorithms/compression/pytorch/pruning/admm_pruner.py similarity index 98% rename from nni/compression/torch/pruning/admm_pruner.py rename to nni/algorithms/compression/pytorch/pruning/admm_pruner.py index ae1a9ec9dc..f65f1405e1 100644 --- a/nni/compression/torch/pruning/admm_pruner.py +++ b/nni/algorithms/compression/pytorch/pruning/admm_pruner.py @@ -5,7 +5,7 @@ import torch from schema import And, Optional -from ..utils.config_validation import CompressorSchema +from nni.compression.pytorch.utils.config_validation import CompressorSchema from .constants import MASKER_DICT from .one_shot import OneshotPruner diff --git a/nni/compression/torch/pruning/agp.py b/nni/algorithms/compression/pytorch/pruning/agp.py similarity index 97% rename from nni/compression/torch/pruning/agp.py rename to nni/algorithms/compression/pytorch/pruning/agp.py index 3e34ba5aef..ef9ca71635 100644 --- a/nni/compression/torch/pruning/agp.py +++ b/nni/algorithms/compression/pytorch/pruning/agp.py @@ -13,8 +13,8 @@ import torch from schema import And, Optional from .constants import MASKER_DICT -from ..utils.config_validation import CompressorSchema -from ..compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from nni.compression.pytorch.compressor import Pruner __all__ = ['AGPPruner'] diff --git a/nni/compression/torch/pruning/amc/__init__.py b/nni/algorithms/compression/pytorch/pruning/amc/__init__.py similarity index 100% rename from nni/compression/torch/pruning/amc/__init__.py rename to nni/algorithms/compression/pytorch/pruning/amc/__init__.py diff --git a/nni/compression/torch/pruning/amc/amc_pruner.py b/nni/algorithms/compression/pytorch/pruning/amc/amc_pruner.py similarity index 99% rename from nni/compression/torch/pruning/amc/amc_pruner.py rename to nni/algorithms/compression/pytorch/pruning/amc/amc_pruner.py index 540ec3fbbc..c2d12429d5 100644 --- a/nni/compression/torch/pruning/amc/amc_pruner.py +++ b/nni/algorithms/compression/pytorch/pruning/amc/amc_pruner.py @@ -9,7 +9,7 @@ import torch from torch.utils.tensorboard import SummaryWriter -from nni.compression.torch.compressor import Pruner +from nni.compression.pytorch.compressor import Pruner from .channel_pruning_env import ChannelPruningEnv from .lib.agent import DDPG from .lib.utils import get_output_folder diff --git a/nni/compression/torch/pruning/amc/channel_pruning_env.py b/nni/algorithms/compression/pytorch/pruning/amc/channel_pruning_env.py similarity index 99% rename from nni/compression/torch/pruning/amc/channel_pruning_env.py rename to nni/algorithms/compression/pytorch/pruning/amc/channel_pruning_env.py index eb5e86b5d2..443daf7efb 100644 --- a/nni/compression/torch/pruning/amc/channel_pruning_env.py +++ b/nni/algorithms/compression/pytorch/pruning/amc/channel_pruning_env.py @@ -10,7 +10,7 @@ import torch import torch.nn as nn -from nni.compression.torch.compressor import PrunerModuleWrapper +from nni.compression.pytorch.compressor import PrunerModuleWrapper from .. import AMCWeightMasker _logger = logging.getLogger(__name__) diff --git a/nni/batch_tuner/__init__.py b/nni/algorithms/compression/pytorch/pruning/amc/lib/__init__.py similarity index 100% rename from nni/batch_tuner/__init__.py rename to nni/algorithms/compression/pytorch/pruning/amc/lib/__init__.py diff --git a/nni/compression/torch/pruning/amc/lib/agent.py b/nni/algorithms/compression/pytorch/pruning/amc/lib/agent.py similarity index 100% rename from nni/compression/torch/pruning/amc/lib/agent.py rename to nni/algorithms/compression/pytorch/pruning/amc/lib/agent.py diff --git a/nni/compression/torch/pruning/amc/lib/memory.py b/nni/algorithms/compression/pytorch/pruning/amc/lib/memory.py similarity index 100% rename from nni/compression/torch/pruning/amc/lib/memory.py rename to nni/algorithms/compression/pytorch/pruning/amc/lib/memory.py diff --git a/nni/compression/torch/pruning/amc/lib/net_measure.py b/nni/algorithms/compression/pytorch/pruning/amc/lib/net_measure.py similarity index 100% rename from nni/compression/torch/pruning/amc/lib/net_measure.py rename to nni/algorithms/compression/pytorch/pruning/amc/lib/net_measure.py diff --git a/nni/compression/torch/pruning/amc/lib/utils.py b/nni/algorithms/compression/pytorch/pruning/amc/lib/utils.py similarity index 100% rename from nni/compression/torch/pruning/amc/lib/utils.py rename to nni/algorithms/compression/pytorch/pruning/amc/lib/utils.py diff --git a/nni/compression/torch/pruning/apply_compression.py b/nni/algorithms/compression/pytorch/pruning/apply_compression.py similarity index 100% rename from nni/compression/torch/pruning/apply_compression.py rename to nni/algorithms/compression/pytorch/pruning/apply_compression.py diff --git a/nni/compression/torch/pruning/auto_compress_pruner.py b/nni/algorithms/compression/pytorch/pruning/auto_compress_pruner.py similarity index 98% rename from nni/compression/torch/pruning/auto_compress_pruner.py rename to nni/algorithms/compression/pytorch/pruning/auto_compress_pruner.py index 24db9f2f88..d52c6ec42d 100644 --- a/nni/compression/torch/pruning/auto_compress_pruner.py +++ b/nni/algorithms/compression/pytorch/pruning/auto_compress_pruner.py @@ -8,10 +8,10 @@ from schema import And, Optional from nni.utils import OptimizeMode -from nni.compression.torch import ModelSpeedup +from nni.compression.pytorch import ModelSpeedup -from ..compressor import Pruner -from ..utils.config_validation import CompressorSchema +from nni.compression.pytorch.compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema from .simulated_annealing_pruner import SimulatedAnnealingPruner from .admm_pruner import ADMMPruner diff --git a/nni/compression/torch/pruning/constants.py b/nni/algorithms/compression/pytorch/pruning/constants.py similarity index 87% rename from nni/compression/torch/pruning/constants.py rename to nni/algorithms/compression/pytorch/pruning/constants.py index 2d0545d09d..24b84340cc 100644 --- a/nni/compression/torch/pruning/constants.py +++ b/nni/algorithms/compression/pytorch/pruning/constants.py @@ -2,7 +2,7 @@ # Licensed under the MIT license. -from ..pruning import LevelPrunerMasker, SlimPrunerMasker, L1FilterPrunerMasker, \ +from . import LevelPrunerMasker, SlimPrunerMasker, L1FilterPrunerMasker, \ L2FilterPrunerMasker, FPGMPrunerMasker, TaylorFOWeightFilterPrunerMasker, \ ActivationAPoZRankFilterPrunerMasker, ActivationMeanRankFilterPrunerMasker diff --git a/nni/compression/torch/pruning/constants_pruner.py b/nni/algorithms/compression/pytorch/pruning/constants_pruner.py similarity index 100% rename from nni/compression/torch/pruning/constants_pruner.py rename to nni/algorithms/compression/pytorch/pruning/constants_pruner.py diff --git a/nni/compression/torch/pruning/finegrained_pruning.py b/nni/algorithms/compression/pytorch/pruning/finegrained_pruning.py similarity index 100% rename from nni/compression/torch/pruning/finegrained_pruning.py rename to nni/algorithms/compression/pytorch/pruning/finegrained_pruning.py diff --git a/nni/compression/torch/pruning/lottery_ticket.py b/nni/algorithms/compression/pytorch/pruning/lottery_ticket.py similarity index 97% rename from nni/compression/torch/pruning/lottery_ticket.py rename to nni/algorithms/compression/pytorch/pruning/lottery_ticket.py index 3a05217753..b0d041dd02 100644 --- a/nni/compression/torch/pruning/lottery_ticket.py +++ b/nni/algorithms/compression/pytorch/pruning/lottery_ticket.py @@ -5,8 +5,8 @@ import logging import torch from schema import And, Optional -from ..utils.config_validation import CompressorSchema -from ..compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from nni.compression.pytorch.compressor import Pruner from .finegrained_pruning import LevelPrunerMasker logger = logging.getLogger('torch pruner') diff --git a/nni/compression/torch/pruning/net_adapt_pruner.py b/nni/algorithms/compression/pytorch/pruning/net_adapt_pruner.py similarity index 98% rename from nni/compression/torch/pruning/net_adapt_pruner.py rename to nni/algorithms/compression/pytorch/pruning/net_adapt_pruner.py index 81d0c47d14..6f55234d5b 100644 --- a/nni/compression/torch/pruning/net_adapt_pruner.py +++ b/nni/algorithms/compression/pytorch/pruning/net_adapt_pruner.py @@ -10,9 +10,9 @@ from nni.utils import OptimizeMode -from ..compressor import Pruner -from ..utils.config_validation import CompressorSchema -from ..utils.num_param_counter import get_total_num_weights +from nni.compression.pytorch.compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from nni.compression.pytorch.utils.num_param_counter import get_total_num_weights from .constants_pruner import PRUNER_DICT diff --git a/nni/compression/torch/pruning/one_shot.py b/nni/algorithms/compression/pytorch/pruning/one_shot.py similarity index 98% rename from nni/compression/torch/pruning/one_shot.py rename to nni/algorithms/compression/pytorch/pruning/one_shot.py index 1958af91e2..75e2a7c307 100644 --- a/nni/compression/torch/pruning/one_shot.py +++ b/nni/algorithms/compression/pytorch/pruning/one_shot.py @@ -3,11 +3,11 @@ import logging from schema import And, Optional, SchemaError -from nni._graph_utils import TorchModuleGraph -from nni.compression.torch.utils.shape_dependency import ChannelDependency, GroupDependency +from nni.common.graph_utils import TorchModuleGraph +from nni.compression.pytorch.utils.shape_dependency import ChannelDependency, GroupDependency from .constants import MASKER_DICT -from ..utils.config_validation import CompressorSchema -from ..compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from nni.compression.pytorch.compressor import Pruner __all__ = ['LevelPruner', 'SlimPruner', 'L1FilterPruner', 'L2FilterPruner', 'FPGMPruner', diff --git a/nni/compression/torch/pruning/sensitivity_pruner.py b/nni/algorithms/compression/pytorch/pruning/sensitivity_pruner.py similarity index 98% rename from nni/compression/torch/pruning/sensitivity_pruner.py rename to nni/algorithms/compression/pytorch/pruning/sensitivity_pruner.py index ed72b1f542..037e7efc5e 100644 --- a/nni/compression/torch/pruning/sensitivity_pruner.py +++ b/nni/algorithms/compression/pytorch/pruning/sensitivity_pruner.py @@ -8,10 +8,10 @@ import torch from schema import And, Optional -from ..compressor import Pruner -from ..utils.config_validation import CompressorSchema +from nni.compression.pytorch.compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema from .constants_pruner import PRUNER_DICT -from ..utils.sensitivity_analysis import SensitivityAnalysis +from nni.compression.pytorch.utils.sensitivity_analysis import SensitivityAnalysis MAX_PRUNE_RATIO_PER_ITER = 0.95 diff --git a/nni/compression/torch/pruning/simulated_annealing_pruner.py b/nni/algorithms/compression/pytorch/pruning/simulated_annealing_pruner.py similarity index 99% rename from nni/compression/torch/pruning/simulated_annealing_pruner.py rename to nni/algorithms/compression/pytorch/pruning/simulated_annealing_pruner.py index accd325d06..91cf160ff3 100644 --- a/nni/compression/torch/pruning/simulated_annealing_pruner.py +++ b/nni/algorithms/compression/pytorch/pruning/simulated_annealing_pruner.py @@ -12,8 +12,8 @@ from nni.utils import OptimizeMode -from ..compressor import Pruner -from ..utils.config_validation import CompressorSchema +from nni.compression.pytorch.compressor import Pruner +from nni.compression.pytorch.utils.config_validation import CompressorSchema from .constants_pruner import PRUNER_DICT diff --git a/nni/compression/torch/pruning/structured_pruning.py b/nni/algorithms/compression/pytorch/pruning/structured_pruning.py similarity index 100% rename from nni/compression/torch/pruning/structured_pruning.py rename to nni/algorithms/compression/pytorch/pruning/structured_pruning.py diff --git a/nni/compression/torch/pruning/weight_masker.py b/nni/algorithms/compression/pytorch/pruning/weight_masker.py similarity index 100% rename from nni/compression/torch/pruning/weight_masker.py rename to nni/algorithms/compression/pytorch/pruning/weight_masker.py diff --git a/nni/compression/torch/quantization/__init__.py b/nni/algorithms/compression/pytorch/quantization/__init__.py similarity index 100% rename from nni/compression/torch/quantization/__init__.py rename to nni/algorithms/compression/pytorch/quantization/__init__.py diff --git a/nni/compression/torch/quantization/quantizers.py b/nni/algorithms/compression/pytorch/quantization/quantizers.py similarity index 98% rename from nni/compression/torch/quantization/quantizers.py rename to nni/algorithms/compression/pytorch/quantization/quantizers.py index 5f4ad36d5e..689b3c56b2 100644 --- a/nni/compression/torch/quantization/quantizers.py +++ b/nni/algorithms/compression/pytorch/quantization/quantizers.py @@ -5,8 +5,8 @@ import copy import torch from schema import Schema, And, Or, Optional -from ..utils.config_validation import CompressorSchema -from ..compressor import Quantizer, QuantGrad, QuantType +from nni.compression.pytorch.utils.config_validation import CompressorSchema +from nni.compression.pytorch.compressor import Quantizer, QuantGrad, QuantType __all__ = ['NaiveQuantizer', 'QAT_Quantizer', 'DoReFaQuantizer', 'BNNQuantizer'] diff --git a/nni/compression/tensorflow/pruning/__init__.py b/nni/algorithms/compression/tensorflow/pruning/__init__.py similarity index 100% rename from nni/compression/tensorflow/pruning/__init__.py rename to nni/algorithms/compression/tensorflow/pruning/__init__.py diff --git a/nni/compression/tensorflow/pruning/one_shot.py b/nni/algorithms/compression/tensorflow/pruning/one_shot.py similarity index 97% rename from nni/compression/tensorflow/pruning/one_shot.py rename to nni/algorithms/compression/tensorflow/pruning/one_shot.py index 13e45d8592..76dc1915ce 100644 --- a/nni/compression/tensorflow/pruning/one_shot.py +++ b/nni/algorithms/compression/tensorflow/pruning/one_shot.py @@ -1,6 +1,6 @@ import tensorflow as tf -from ..compressor import Pruner +from nni.compression.tensorflow import Pruner __all__ = [ 'OneshotPruner', diff --git a/nni/feature_engineering/gbdt_selector/__init__.py b/nni/algorithms/feature_engineering/gbdt_selector/__init__.py similarity index 100% rename from nni/feature_engineering/gbdt_selector/__init__.py rename to nni/algorithms/feature_engineering/gbdt_selector/__init__.py diff --git a/nni/feature_engineering/gbdt_selector/gbdt_selector.py b/nni/algorithms/feature_engineering/gbdt_selector/gbdt_selector.py similarity index 100% rename from nni/feature_engineering/gbdt_selector/gbdt_selector.py rename to nni/algorithms/feature_engineering/gbdt_selector/gbdt_selector.py diff --git a/nni/feature_engineering/gbdt_selector/requirements.txt b/nni/algorithms/feature_engineering/gbdt_selector/requirements.txt similarity index 100% rename from nni/feature_engineering/gbdt_selector/requirements.txt rename to nni/algorithms/feature_engineering/gbdt_selector/requirements.txt diff --git a/nni/feature_engineering/gradient_selector/__init__.py b/nni/algorithms/feature_engineering/gradient_selector/__init__.py similarity index 100% rename from nni/feature_engineering/gradient_selector/__init__.py rename to nni/algorithms/feature_engineering/gradient_selector/__init__.py diff --git a/nni/feature_engineering/gradient_selector/constants.py b/nni/algorithms/feature_engineering/gradient_selector/constants.py similarity index 100% rename from nni/feature_engineering/gradient_selector/constants.py rename to nni/algorithms/feature_engineering/gradient_selector/constants.py diff --git a/nni/feature_engineering/gradient_selector/fginitialize.py b/nni/algorithms/feature_engineering/gradient_selector/fginitialize.py similarity index 100% rename from nni/feature_engineering/gradient_selector/fginitialize.py rename to nni/algorithms/feature_engineering/gradient_selector/fginitialize.py diff --git a/nni/feature_engineering/gradient_selector/fgtrain.py b/nni/algorithms/feature_engineering/gradient_selector/fgtrain.py similarity index 100% rename from nni/feature_engineering/gradient_selector/fgtrain.py rename to nni/algorithms/feature_engineering/gradient_selector/fgtrain.py diff --git a/nni/feature_engineering/gradient_selector/gradient_selector.py b/nni/algorithms/feature_engineering/gradient_selector/gradient_selector.py similarity index 100% rename from nni/feature_engineering/gradient_selector/gradient_selector.py rename to nni/algorithms/feature_engineering/gradient_selector/gradient_selector.py diff --git a/nni/feature_engineering/gradient_selector/learnability.py b/nni/algorithms/feature_engineering/gradient_selector/learnability.py similarity index 100% rename from nni/feature_engineering/gradient_selector/learnability.py rename to nni/algorithms/feature_engineering/gradient_selector/learnability.py diff --git a/nni/feature_engineering/gradient_selector/requirements.txt b/nni/algorithms/feature_engineering/gradient_selector/requirements.txt similarity index 100% rename from nni/feature_engineering/gradient_selector/requirements.txt rename to nni/algorithms/feature_engineering/gradient_selector/requirements.txt diff --git a/nni/feature_engineering/gradient_selector/syssettings.py b/nni/algorithms/feature_engineering/gradient_selector/syssettings.py similarity index 100% rename from nni/feature_engineering/gradient_selector/syssettings.py rename to nni/algorithms/feature_engineering/gradient_selector/syssettings.py diff --git a/nni/feature_engineering/gradient_selector/utils.py b/nni/algorithms/feature_engineering/gradient_selector/utils.py similarity index 100% rename from nni/feature_engineering/gradient_selector/utils.py rename to nni/algorithms/feature_engineering/gradient_selector/utils.py diff --git a/nni/bohb_advisor/__init__.py b/nni/algorithms/hpo/batch_tuner/__init__.py similarity index 100% rename from nni/bohb_advisor/__init__.py rename to nni/algorithms/hpo/batch_tuner/__init__.py diff --git a/nni/batch_tuner/batch_tuner.py b/nni/algorithms/hpo/batch_tuner/batch_tuner.py similarity index 100% rename from nni/batch_tuner/batch_tuner.py rename to nni/algorithms/hpo/batch_tuner/batch_tuner.py diff --git a/nni/compression/torch/pruning/amc/lib/__init__.py b/nni/algorithms/hpo/bohb_advisor/__init__.py similarity index 100% rename from nni/compression/torch/pruning/amc/lib/__init__.py rename to nni/algorithms/hpo/bohb_advisor/__init__.py diff --git a/nni/bohb_advisor/bohb_advisor.py b/nni/algorithms/hpo/bohb_advisor/bohb_advisor.py similarity index 99% rename from nni/bohb_advisor/bohb_advisor.py rename to nni/algorithms/hpo/bohb_advisor/bohb_advisor.py index 40a78ada69..73687abc5c 100644 --- a/nni/bohb_advisor/bohb_advisor.py +++ b/nni/algorithms/hpo/bohb_advisor/bohb_advisor.py @@ -14,10 +14,10 @@ import ConfigSpace.hyperparameters as CSH from nni import ClassArgsValidator -from nni.protocol import CommandType, send -from nni.msg_dispatcher_base import MsgDispatcherBase +from nni.runtime.protocol import CommandType, send +from nni.runtime.msg_dispatcher_base import MsgDispatcherBase from nni.utils import OptimizeMode, MetricType, extract_scalar_reward -from nni.common import multi_phase_enabled +from nni.runtime.common import multi_phase_enabled from .config_generator import CG_BOHB diff --git a/nni/bohb_advisor/config_generator.py b/nni/algorithms/hpo/bohb_advisor/config_generator.py similarity index 100% rename from nni/bohb_advisor/config_generator.py rename to nni/algorithms/hpo/bohb_advisor/config_generator.py diff --git a/nni/bohb_advisor/requirements.txt b/nni/algorithms/hpo/bohb_advisor/requirements.txt similarity index 100% rename from nni/bohb_advisor/requirements.txt rename to nni/algorithms/hpo/bohb_advisor/requirements.txt diff --git a/nni/curvefitting_assessor/__init__.py b/nni/algorithms/hpo/curvefitting_assessor/__init__.py similarity index 100% rename from nni/curvefitting_assessor/__init__.py rename to nni/algorithms/hpo/curvefitting_assessor/__init__.py diff --git a/nni/curvefitting_assessor/curvefitting_assessor.py b/nni/algorithms/hpo/curvefitting_assessor/curvefitting_assessor.py similarity index 100% rename from nni/curvefitting_assessor/curvefitting_assessor.py rename to nni/algorithms/hpo/curvefitting_assessor/curvefitting_assessor.py diff --git a/nni/curvefitting_assessor/curvefunctions.py b/nni/algorithms/hpo/curvefitting_assessor/curvefunctions.py similarity index 100% rename from nni/curvefitting_assessor/curvefunctions.py rename to nni/algorithms/hpo/curvefitting_assessor/curvefunctions.py diff --git a/nni/curvefitting_assessor/model_factory.py b/nni/algorithms/hpo/curvefitting_assessor/model_factory.py similarity index 100% rename from nni/curvefitting_assessor/model_factory.py rename to nni/algorithms/hpo/curvefitting_assessor/model_factory.py diff --git a/nni/compression/torch/utils/__init__.py b/nni/algorithms/hpo/evolution_tuner/__init__.py similarity index 100% rename from nni/compression/torch/utils/__init__.py rename to nni/algorithms/hpo/evolution_tuner/__init__.py diff --git a/nni/evolution_tuner/evolution_tuner.py b/nni/algorithms/hpo/evolution_tuner/evolution_tuner.py similarity index 100% rename from nni/evolution_tuner/evolution_tuner.py rename to nni/algorithms/hpo/evolution_tuner/evolution_tuner.py diff --git a/nni/evolution_tuner/__init__.py b/nni/algorithms/hpo/gp_tuner/__init__.py similarity index 100% rename from nni/evolution_tuner/__init__.py rename to nni/algorithms/hpo/gp_tuner/__init__.py diff --git a/nni/gp_tuner/gp_tuner.py b/nni/algorithms/hpo/gp_tuner/gp_tuner.py similarity index 100% rename from nni/gp_tuner/gp_tuner.py rename to nni/algorithms/hpo/gp_tuner/gp_tuner.py diff --git a/nni/gp_tuner/target_space.py b/nni/algorithms/hpo/gp_tuner/target_space.py similarity index 100% rename from nni/gp_tuner/target_space.py rename to nni/algorithms/hpo/gp_tuner/target_space.py diff --git a/nni/gp_tuner/util.py b/nni/algorithms/hpo/gp_tuner/util.py similarity index 100% rename from nni/gp_tuner/util.py rename to nni/algorithms/hpo/gp_tuner/util.py diff --git a/nni/gridsearch_tuner/__init__.py b/nni/algorithms/hpo/gridsearch_tuner/__init__.py similarity index 100% rename from nni/gridsearch_tuner/__init__.py rename to nni/algorithms/hpo/gridsearch_tuner/__init__.py diff --git a/nni/gridsearch_tuner/gridsearch_tuner.py b/nni/algorithms/hpo/gridsearch_tuner/gridsearch_tuner.py similarity index 100% rename from nni/gridsearch_tuner/gridsearch_tuner.py rename to nni/algorithms/hpo/gridsearch_tuner/gridsearch_tuner.py diff --git a/nni/gp_tuner/__init__.py b/nni/algorithms/hpo/hyperband_advisor/__init__.py similarity index 100% rename from nni/gp_tuner/__init__.py rename to nni/algorithms/hpo/hyperband_advisor/__init__.py diff --git a/nni/hyperband_advisor/hyperband_advisor.py b/nni/algorithms/hpo/hyperband_advisor/hyperband_advisor.py similarity index 99% rename from nni/hyperband_advisor/hyperband_advisor.py rename to nni/algorithms/hpo/hyperband_advisor/hyperband_advisor.py index 2b13d8f3c3..72e2c28e1a 100644 --- a/nni/hyperband_advisor/hyperband_advisor.py +++ b/nni/algorithms/hpo/hyperband_advisor/hyperband_advisor.py @@ -15,9 +15,9 @@ from schema import Schema, Optional from nni import ClassArgsValidator -from nni.common import multi_phase_enabled -from nni.msg_dispatcher_base import MsgDispatcherBase -from nni.protocol import CommandType, send +from nni.runtime.common import multi_phase_enabled +from nni.runtime.msg_dispatcher_base import MsgDispatcherBase +from nni.runtime.protocol import CommandType, send from nni.utils import NodeType, OptimizeMode, MetricType, extract_scalar_reward from nni import parameter_expressions diff --git a/nni/hyperband_advisor/requirements.txt b/nni/algorithms/hpo/hyperband_advisor/requirements.txt similarity index 100% rename from nni/hyperband_advisor/requirements.txt rename to nni/algorithms/hpo/hyperband_advisor/requirements.txt diff --git a/nni/hyperband_advisor/__init__.py b/nni/algorithms/hpo/hyperopt_tuner/__init__.py similarity index 100% rename from nni/hyperband_advisor/__init__.py rename to nni/algorithms/hpo/hyperopt_tuner/__init__.py diff --git a/nni/hyperopt_tuner/hyperopt_tuner.py b/nni/algorithms/hpo/hyperopt_tuner/hyperopt_tuner.py similarity index 100% rename from nni/hyperopt_tuner/hyperopt_tuner.py rename to nni/algorithms/hpo/hyperopt_tuner/hyperopt_tuner.py diff --git a/nni/medianstop_assessor/__init__.py b/nni/algorithms/hpo/medianstop_assessor/__init__.py similarity index 100% rename from nni/medianstop_assessor/__init__.py rename to nni/algorithms/hpo/medianstop_assessor/__init__.py diff --git a/nni/medianstop_assessor/medianstop_assessor.py b/nni/algorithms/hpo/medianstop_assessor/medianstop_assessor.py similarity index 100% rename from nni/medianstop_assessor/medianstop_assessor.py rename to nni/algorithms/hpo/medianstop_assessor/medianstop_assessor.py diff --git a/nni/medianstop_assessor/test.py b/nni/algorithms/hpo/medianstop_assessor/test.py similarity index 100% rename from nni/medianstop_assessor/test.py rename to nni/algorithms/hpo/medianstop_assessor/test.py diff --git a/nni/metis_tuner/Regression_GMM/CreateModel.py b/nni/algorithms/hpo/metis_tuner/Regression_GMM/CreateModel.py similarity index 100% rename from nni/metis_tuner/Regression_GMM/CreateModel.py rename to nni/algorithms/hpo/metis_tuner/Regression_GMM/CreateModel.py diff --git a/nni/metis_tuner/Regression_GMM/Selection.py b/nni/algorithms/hpo/metis_tuner/Regression_GMM/Selection.py similarity index 95% rename from nni/metis_tuner/Regression_GMM/Selection.py rename to nni/algorithms/hpo/metis_tuner/Regression_GMM/Selection.py index 9b0c78860a..0bca96647d 100644 --- a/nni/metis_tuner/Regression_GMM/Selection.py +++ b/nni/algorithms/hpo/metis_tuner/Regression_GMM/Selection.py @@ -5,8 +5,8 @@ import random import sys -import nni.metis_tuner.lib_acquisition_function as lib_acquisition_function -import nni.metis_tuner.lib_constraint_summation as lib_constraint_summation +from .. import lib_acquisition_function +from .. import lib_constraint_summation sys.path.insert(1, os.path.join(sys.path[0], '..')) diff --git a/nni/hyperopt_tuner/__init__.py b/nni/algorithms/hpo/metis_tuner/Regression_GMM/__init__.py similarity index 100% rename from nni/hyperopt_tuner/__init__.py rename to nni/algorithms/hpo/metis_tuner/Regression_GMM/__init__.py diff --git a/nni/metis_tuner/Regression_GP/CreateModel.py b/nni/algorithms/hpo/metis_tuner/Regression_GP/CreateModel.py similarity index 100% rename from nni/metis_tuner/Regression_GP/CreateModel.py rename to nni/algorithms/hpo/metis_tuner/Regression_GP/CreateModel.py diff --git a/nni/metis_tuner/Regression_GP/OutlierDetection.py b/nni/algorithms/hpo/metis_tuner/Regression_GP/OutlierDetection.py similarity index 96% rename from nni/metis_tuner/Regression_GP/OutlierDetection.py rename to nni/algorithms/hpo/metis_tuner/Regression_GP/OutlierDetection.py index 43234ca3e4..f07a93dd3e 100644 --- a/nni/metis_tuner/Regression_GP/OutlierDetection.py +++ b/nni/algorithms/hpo/metis_tuner/Regression_GP/OutlierDetection.py @@ -9,8 +9,8 @@ import sys from multiprocessing.dummy import Pool as ThreadPool -import nni.metis_tuner.Regression_GP.CreateModel as gp_create_model -import nni.metis_tuner.Regression_GP.Prediction as gp_prediction +from . import CreateModel as gp_create_model +from . import Prediction as gp_prediction sys.path.insert(1, os.path.join(sys.path[0], '..')) diff --git a/nni/metis_tuner/Regression_GP/Prediction.py b/nni/algorithms/hpo/metis_tuner/Regression_GP/Prediction.py similarity index 100% rename from nni/metis_tuner/Regression_GP/Prediction.py rename to nni/algorithms/hpo/metis_tuner/Regression_GP/Prediction.py diff --git a/nni/metis_tuner/Regression_GP/Selection.py b/nni/algorithms/hpo/metis_tuner/Regression_GP/Selection.py similarity index 93% rename from nni/metis_tuner/Regression_GP/Selection.py rename to nni/algorithms/hpo/metis_tuner/Regression_GP/Selection.py index 0cf19f88f6..68383ed0f2 100644 --- a/nni/metis_tuner/Regression_GP/Selection.py +++ b/nni/algorithms/hpo/metis_tuner/Regression_GP/Selection.py @@ -5,10 +5,10 @@ import random import sys -import nni.metis_tuner.lib_acquisition_function as lib_acquisition_function -import nni.metis_tuner.lib_constraint_summation as lib_constraint_summation -import nni.metis_tuner.lib_data as lib_data -import nni.metis_tuner.Regression_GP.Prediction as gp_prediction +from .. import lib_acquisition_function +from .. import lib_constraint_summation +from .. import lib_data +from . import Prediction as gp_prediction sys.path.insert(1, os.path.join(sys.path[0], '..')) diff --git a/nni/metis_tuner/Regression_GMM/__init__.py b/nni/algorithms/hpo/metis_tuner/Regression_GP/__init__.py similarity index 100% rename from nni/metis_tuner/Regression_GMM/__init__.py rename to nni/algorithms/hpo/metis_tuner/Regression_GP/__init__.py diff --git a/nni/metis_tuner/Regression_GP/__init__.py b/nni/algorithms/hpo/metis_tuner/__init__.py similarity index 100% rename from nni/metis_tuner/Regression_GP/__init__.py rename to nni/algorithms/hpo/metis_tuner/__init__.py diff --git a/nni/metis_tuner/lib_acquisition_function.py b/nni/algorithms/hpo/metis_tuner/lib_acquisition_function.py similarity index 99% rename from nni/metis_tuner/lib_acquisition_function.py rename to nni/algorithms/hpo/metis_tuner/lib_acquisition_function.py index a59de4891b..f1b1edfe01 100644 --- a/nni/metis_tuner/lib_acquisition_function.py +++ b/nni/algorithms/hpo/metis_tuner/lib_acquisition_function.py @@ -11,7 +11,7 @@ from scipy.stats import norm from scipy.optimize import minimize -import nni.metis_tuner.lib_data as lib_data +from . import lib_data def next_hyperparameter_expected_improvement(fun_prediction, diff --git a/nni/metis_tuner/lib_constraint_summation.py b/nni/algorithms/hpo/metis_tuner/lib_constraint_summation.py similarity index 100% rename from nni/metis_tuner/lib_constraint_summation.py rename to nni/algorithms/hpo/metis_tuner/lib_constraint_summation.py diff --git a/nni/metis_tuner/lib_data.py b/nni/algorithms/hpo/metis_tuner/lib_data.py similarity index 100% rename from nni/metis_tuner/lib_data.py rename to nni/algorithms/hpo/metis_tuner/lib_data.py diff --git a/nni/metis_tuner/metis_tuner.py b/nni/algorithms/hpo/metis_tuner/metis_tuner.py similarity index 97% rename from nni/metis_tuner/metis_tuner.py rename to nni/algorithms/hpo/metis_tuner/metis_tuner.py index cdd50dbcf3..1a0670f5e5 100644 --- a/nni/metis_tuner/metis_tuner.py +++ b/nni/algorithms/hpo/metis_tuner/metis_tuner.py @@ -15,14 +15,14 @@ from schema import Schema, Optional from nni import ClassArgsValidator -import nni.metis_tuner.lib_constraint_summation as lib_constraint_summation -import nni.metis_tuner.lib_data as lib_data -import nni.metis_tuner.Regression_GMM.CreateModel as gmm_create_model -import nni.metis_tuner.Regression_GMM.Selection as gmm_selection -import nni.metis_tuner.Regression_GP.CreateModel as gp_create_model -import nni.metis_tuner.Regression_GP.OutlierDetection as gp_outlier_detection -import nni.metis_tuner.Regression_GP.Prediction as gp_prediction -import nni.metis_tuner.Regression_GP.Selection as gp_selection +from . import lib_constraint_summation +from . import lib_data +from .Regression_GMM import CreateModel as gmm_create_model +from .Regression_GMM import Selection as gmm_selection +from .Regression_GP import CreateModel as gp_create_model +from .Regression_GP import OutlierDetection as gp_outlier_detection +from .Regression_GP import Prediction as gp_prediction +from .Regression_GP import Selection as gp_selection from nni.tuner import Tuner from nni.utils import OptimizeMode, extract_scalar_reward diff --git a/nni/metis_tuner/requirments.txt b/nni/algorithms/hpo/metis_tuner/requirments.txt similarity index 100% rename from nni/metis_tuner/requirments.txt rename to nni/algorithms/hpo/metis_tuner/requirments.txt diff --git a/nni/metis_tuner/__init__.py b/nni/algorithms/hpo/networkmorphism_tuner/__init__.py similarity index 100% rename from nni/metis_tuner/__init__.py rename to nni/algorithms/hpo/networkmorphism_tuner/__init__.py diff --git a/nni/networkmorphism_tuner/bayesian.py b/nni/algorithms/hpo/networkmorphism_tuner/bayesian.py similarity index 98% rename from nni/networkmorphism_tuner/bayesian.py rename to nni/algorithms/hpo/networkmorphism_tuner/bayesian.py index 8d838337fd..54c1996dc7 100644 --- a/nni/networkmorphism_tuner/bayesian.py +++ b/nni/algorithms/hpo/networkmorphism_tuner/bayesian.py @@ -13,9 +13,9 @@ from sklearn.metrics.pairwise import rbf_kernel from nni.utils import OptimizeMode -from nni.networkmorphism_tuner.graph_transformer import transform -from nni.networkmorphism_tuner.utils import Constant -from nni.networkmorphism_tuner.layers import is_layer +from .graph_transformer import transform +from .utils import Constant +from .layers import is_layer def layer_distance(a, b): diff --git a/nni/networkmorphism_tuner/graph.py b/nni/algorithms/hpo/networkmorphism_tuner/graph.py similarity index 99% rename from nni/networkmorphism_tuner/graph.py rename to nni/algorithms/hpo/networkmorphism_tuner/graph.py index 30adc4aab5..9c96b6c2f0 100644 --- a/nni/networkmorphism_tuner/graph.py +++ b/nni/algorithms/hpo/networkmorphism_tuner/graph.py @@ -9,7 +9,7 @@ import numpy as np import torch -from nni.networkmorphism_tuner.layer_transformer import ( +from .layer_transformer import ( add_noise, wider_bn, wider_next_conv, @@ -20,7 +20,7 @@ init_conv_weight, init_bn_weight, ) -from nni.networkmorphism_tuner.layers import ( +from .layers import ( StubAdd, StubConcatenate, StubReLU, @@ -36,7 +36,7 @@ layer_description_extractor, layer_description_builder, ) -from nni.networkmorphism_tuner.utils import Constant +from .utils import Constant class NetworkDescriptor: diff --git a/nni/networkmorphism_tuner/graph_transformer.py b/nni/algorithms/hpo/networkmorphism_tuner/graph_transformer.py similarity index 96% rename from nni/networkmorphism_tuner/graph_transformer.py rename to nni/algorithms/hpo/networkmorphism_tuner/graph_transformer.py index d29668e94d..b03112d63b 100644 --- a/nni/networkmorphism_tuner/graph_transformer.py +++ b/nni/algorithms/hpo/networkmorphism_tuner/graph_transformer.py @@ -5,8 +5,8 @@ from random import randrange, sample -from nni.networkmorphism_tuner.graph import NetworkDescriptor -from nni.networkmorphism_tuner.layers import ( +from .graph import NetworkDescriptor +from .layers import ( StubDense, StubReLU, get_batch_norm_class, @@ -15,7 +15,7 @@ get_pooling_class, is_layer, ) -from nni.networkmorphism_tuner.utils import Constant +from .utils import Constant def to_wider_graph(graph): diff --git a/nni/networkmorphism_tuner/layer_transformer.py b/nni/algorithms/hpo/networkmorphism_tuner/layer_transformer.py similarity index 99% rename from nni/networkmorphism_tuner/layer_transformer.py rename to nni/algorithms/hpo/networkmorphism_tuner/layer_transformer.py index c3642dadc7..6ffd1b20fb 100644 --- a/nni/networkmorphism_tuner/layer_transformer.py +++ b/nni/algorithms/hpo/networkmorphism_tuner/layer_transformer.py @@ -3,7 +3,7 @@ import numpy as np -from nni.networkmorphism_tuner.layers import ( +from .layers import ( StubDense, StubReLU, get_batch_norm_class, diff --git a/nni/networkmorphism_tuner/layers.py b/nni/algorithms/hpo/networkmorphism_tuner/layers.py similarity index 99% rename from nni/networkmorphism_tuner/layers.py rename to nni/algorithms/hpo/networkmorphism_tuner/layers.py index 365561299c..a96c87b780 100644 --- a/nni/networkmorphism_tuner/layers.py +++ b/nni/algorithms/hpo/networkmorphism_tuner/layers.py @@ -7,7 +7,7 @@ import torch from torch import nn from torch.nn import functional -from nni.networkmorphism_tuner.utils import Constant +from .utils import Constant class AvgPool(nn.Module): diff --git a/nni/networkmorphism_tuner/networkmorphism_tuner.py b/nni/algorithms/hpo/networkmorphism_tuner/networkmorphism_tuner.py similarity index 97% rename from nni/networkmorphism_tuner/networkmorphism_tuner.py rename to nni/algorithms/hpo/networkmorphism_tuner/networkmorphism_tuner.py index 50706756dd..6a73cad3c6 100644 --- a/nni/networkmorphism_tuner/networkmorphism_tuner.py +++ b/nni/algorithms/hpo/networkmorphism_tuner/networkmorphism_tuner.py @@ -10,10 +10,10 @@ from schema import Optional, Schema from nni.tuner import Tuner from nni.utils import OptimizeMode, extract_scalar_reward -from nni.networkmorphism_tuner.bayesian import BayesianOptimizer -from nni.networkmorphism_tuner.nn import CnnGenerator, MlpGenerator -from nni.networkmorphism_tuner.utils import Constant -from nni.networkmorphism_tuner.graph import graph_to_json, json_to_graph +from .bayesian import BayesianOptimizer +from .nn import CnnGenerator, MlpGenerator +from .utils import Constant +from .graph import graph_to_json, json_to_graph from nni import ClassArgsValidator logger = logging.getLogger("NetworkMorphism_AutoML") diff --git a/nni/networkmorphism_tuner/nn.py b/nni/algorithms/hpo/networkmorphism_tuner/nn.py similarity index 97% rename from nni/networkmorphism_tuner/nn.py rename to nni/algorithms/hpo/networkmorphism_tuner/nn.py index 14061682cc..9e0072f9b3 100644 --- a/nni/networkmorphism_tuner/nn.py +++ b/nni/algorithms/hpo/networkmorphism_tuner/nn.py @@ -3,14 +3,14 @@ from abc import abstractmethod -from nni.networkmorphism_tuner.graph import Graph -from nni.networkmorphism_tuner.layers import (StubDense, StubDropout1d, +from .graph import Graph +from .layers import (StubDense, StubDropout1d, StubReLU, get_batch_norm_class, get_conv_class, get_dropout_class, get_global_avg_pooling_class, get_pooling_class) -from nni.networkmorphism_tuner.utils import Constant +from .utils import Constant class NetworkGenerator: diff --git a/nni/networkmorphism_tuner/utils.py b/nni/algorithms/hpo/networkmorphism_tuner/utils.py similarity index 100% rename from nni/networkmorphism_tuner/utils.py rename to nni/algorithms/hpo/networkmorphism_tuner/utils.py diff --git a/nni/networkmorphism_tuner/__init__.py b/nni/algorithms/hpo/pbt_tuner/__init__.py similarity index 100% rename from nni/networkmorphism_tuner/__init__.py rename to nni/algorithms/hpo/pbt_tuner/__init__.py diff --git a/nni/pbt_tuner/pbt_tuner.py b/nni/algorithms/hpo/pbt_tuner/pbt_tuner.py similarity index 100% rename from nni/pbt_tuner/pbt_tuner.py rename to nni/algorithms/hpo/pbt_tuner/pbt_tuner.py diff --git a/nni/ppo_tuner/__init__.py b/nni/algorithms/hpo/ppo_tuner/__init__.py similarity index 100% rename from nni/ppo_tuner/__init__.py rename to nni/algorithms/hpo/ppo_tuner/__init__.py diff --git a/nni/ppo_tuner/distri.py b/nni/algorithms/hpo/ppo_tuner/distri.py similarity index 100% rename from nni/ppo_tuner/distri.py rename to nni/algorithms/hpo/ppo_tuner/distri.py diff --git a/nni/ppo_tuner/model.py b/nni/algorithms/hpo/ppo_tuner/model.py similarity index 100% rename from nni/ppo_tuner/model.py rename to nni/algorithms/hpo/ppo_tuner/model.py diff --git a/nni/ppo_tuner/policy.py b/nni/algorithms/hpo/ppo_tuner/policy.py similarity index 100% rename from nni/ppo_tuner/policy.py rename to nni/algorithms/hpo/ppo_tuner/policy.py diff --git a/nni/ppo_tuner/ppo_tuner.py b/nni/algorithms/hpo/ppo_tuner/ppo_tuner.py similarity index 100% rename from nni/ppo_tuner/ppo_tuner.py rename to nni/algorithms/hpo/ppo_tuner/ppo_tuner.py diff --git a/nni/ppo_tuner/requirements.txt b/nni/algorithms/hpo/ppo_tuner/requirements.txt similarity index 100% rename from nni/ppo_tuner/requirements.txt rename to nni/algorithms/hpo/ppo_tuner/requirements.txt diff --git a/nni/ppo_tuner/util.py b/nni/algorithms/hpo/ppo_tuner/util.py similarity index 100% rename from nni/ppo_tuner/util.py rename to nni/algorithms/hpo/ppo_tuner/util.py diff --git a/nni/regularized_evolution_tuner/__init__.py b/nni/algorithms/hpo/regularized_evolution_tuner/__init__.py similarity index 100% rename from nni/regularized_evolution_tuner/__init__.py rename to nni/algorithms/hpo/regularized_evolution_tuner/__init__.py diff --git a/nni/regularized_evolution_tuner/regularized_evolution_tuner.py b/nni/algorithms/hpo/regularized_evolution_tuner/regularized_evolution_tuner.py similarity index 100% rename from nni/regularized_evolution_tuner/regularized_evolution_tuner.py rename to nni/algorithms/hpo/regularized_evolution_tuner/regularized_evolution_tuner.py diff --git a/nni/smac_tuner/__init__.py b/nni/algorithms/hpo/smac_tuner/__init__.py similarity index 100% rename from nni/smac_tuner/__init__.py rename to nni/algorithms/hpo/smac_tuner/__init__.py diff --git a/nni/smac_tuner/convert_ss_to_scenario.py b/nni/algorithms/hpo/smac_tuner/convert_ss_to_scenario.py similarity index 100% rename from nni/smac_tuner/convert_ss_to_scenario.py rename to nni/algorithms/hpo/smac_tuner/convert_ss_to_scenario.py diff --git a/nni/smac_tuner/requirements.txt b/nni/algorithms/hpo/smac_tuner/requirements.txt similarity index 100% rename from nni/smac_tuner/requirements.txt rename to nni/algorithms/hpo/smac_tuner/requirements.txt diff --git a/nni/smac_tuner/smac_tuner.py b/nni/algorithms/hpo/smac_tuner/smac_tuner.py similarity index 100% rename from nni/smac_tuner/smac_tuner.py rename to nni/algorithms/hpo/smac_tuner/smac_tuner.py diff --git a/nni/nas/pytorch/cdarts/__init__.py b/nni/algorithms/nas/pytorch/cdarts/__init__.py similarity index 100% rename from nni/nas/pytorch/cdarts/__init__.py rename to nni/algorithms/nas/pytorch/cdarts/__init__.py diff --git a/nni/nas/pytorch/cdarts/mutator.py b/nni/algorithms/nas/pytorch/cdarts/mutator.py similarity index 100% rename from nni/nas/pytorch/cdarts/mutator.py rename to nni/algorithms/nas/pytorch/cdarts/mutator.py diff --git a/nni/nas/pytorch/cdarts/trainer.py b/nni/algorithms/nas/pytorch/cdarts/trainer.py similarity index 98% rename from nni/nas/pytorch/cdarts/trainer.py rename to nni/algorithms/nas/pytorch/cdarts/trainer.py index fa6d09e3e7..1a5174216f 100644 --- a/nni/nas/pytorch/cdarts/trainer.py +++ b/nni/algorithms/nas/pytorch/cdarts/trainer.py @@ -10,7 +10,7 @@ import torch.nn.functional as F import apex # pylint: disable=import-error from apex.parallel import DistributedDataParallel # pylint: disable=import-error -from nni.nas.pytorch.cdarts import RegularizedDartsMutator, RegularizedMutatorParallel, DartsDiscreteMutator # pylint: disable=wrong-import-order +from .mutator import RegularizedDartsMutator, RegularizedMutatorParallel, DartsDiscreteMutator # pylint: disable=wrong-import-order from nni.nas.pytorch.utils import AverageMeterGroup # pylint: disable=wrong-import-order from .utils import CyclicIterator, TorchTensorEncoder, accuracy, reduce_metrics diff --git a/nni/nas/pytorch/cdarts/utils.py b/nni/algorithms/nas/pytorch/cdarts/utils.py similarity index 100% rename from nni/nas/pytorch/cdarts/utils.py rename to nni/algorithms/nas/pytorch/cdarts/utils.py diff --git a/nni/nas/pytorch/classic_nas/__init__.py b/nni/algorithms/nas/pytorch/classic_nas/__init__.py similarity index 100% rename from nni/nas/pytorch/classic_nas/__init__.py rename to nni/algorithms/nas/pytorch/classic_nas/__init__.py diff --git a/nni/nas/pytorch/classic_nas/mutator.py b/nni/algorithms/nas/pytorch/classic_nas/mutator.py similarity index 99% rename from nni/nas/pytorch/classic_nas/mutator.py rename to nni/algorithms/nas/pytorch/classic_nas/mutator.py index e018e758d1..7254a8b0b4 100644 --- a/nni/nas/pytorch/classic_nas/mutator.py +++ b/nni/algorithms/nas/pytorch/classic_nas/mutator.py @@ -9,7 +9,7 @@ import torch import nni -from nni.env_vars import trial_env_vars +from nni.runtime.env_vars import trial_env_vars from nni.nas.pytorch.mutables import LayerChoice, InputChoice, MutableScope from nni.nas.pytorch.mutator import Mutator diff --git a/nni/nas/pytorch/darts/__init__.py b/nni/algorithms/nas/pytorch/darts/__init__.py similarity index 100% rename from nni/nas/pytorch/darts/__init__.py rename to nni/algorithms/nas/pytorch/darts/__init__.py diff --git a/nni/nas/pytorch/darts/mutator.py b/nni/algorithms/nas/pytorch/darts/mutator.py similarity index 100% rename from nni/nas/pytorch/darts/mutator.py rename to nni/algorithms/nas/pytorch/darts/mutator.py diff --git a/nni/nas/pytorch/darts/trainer.py b/nni/algorithms/nas/pytorch/darts/trainer.py similarity index 100% rename from nni/nas/pytorch/darts/trainer.py rename to nni/algorithms/nas/pytorch/darts/trainer.py diff --git a/nni/nas/pytorch/enas/__init__.py b/nni/algorithms/nas/pytorch/enas/__init__.py similarity index 100% rename from nni/nas/pytorch/enas/__init__.py rename to nni/algorithms/nas/pytorch/enas/__init__.py diff --git a/nni/nas/pytorch/enas/mutator.py b/nni/algorithms/nas/pytorch/enas/mutator.py similarity index 100% rename from nni/nas/pytorch/enas/mutator.py rename to nni/algorithms/nas/pytorch/enas/mutator.py diff --git a/nni/nas/pytorch/enas/trainer.py b/nni/algorithms/nas/pytorch/enas/trainer.py similarity index 100% rename from nni/nas/pytorch/enas/trainer.py rename to nni/algorithms/nas/pytorch/enas/trainer.py diff --git a/nni/nas/pytorch/pdarts/__init__.py b/nni/algorithms/nas/pytorch/pdarts/__init__.py similarity index 100% rename from nni/nas/pytorch/pdarts/__init__.py rename to nni/algorithms/nas/pytorch/pdarts/__init__.py diff --git a/nni/nas/pytorch/pdarts/mutator.py b/nni/algorithms/nas/pytorch/pdarts/mutator.py similarity index 98% rename from nni/nas/pytorch/pdarts/mutator.py rename to nni/algorithms/nas/pytorch/pdarts/mutator.py index 5bafba4042..09ad51c5e4 100644 --- a/nni/nas/pytorch/pdarts/mutator.py +++ b/nni/algorithms/nas/pytorch/pdarts/mutator.py @@ -7,7 +7,7 @@ import torch from torch import nn -from nni.nas.pytorch.darts import DartsMutator +from nni.algorithms.nas.pytorch.darts import DartsMutator from nni.nas.pytorch.mutables import LayerChoice diff --git a/nni/nas/pytorch/pdarts/trainer.py b/nni/algorithms/nas/pytorch/pdarts/trainer.py similarity index 98% rename from nni/nas/pytorch/pdarts/trainer.py rename to nni/algorithms/nas/pytorch/pdarts/trainer.py index 8d3ad64dda..7f23a6e222 100644 --- a/nni/nas/pytorch/pdarts/trainer.py +++ b/nni/algorithms/nas/pytorch/pdarts/trainer.py @@ -5,7 +5,7 @@ import logging from nni.nas.pytorch.callbacks import LRSchedulerCallback -from nni.nas.pytorch.darts import DartsTrainer +from nni.algorithms.nas.pytorch.darts import DartsTrainer from nni.nas.pytorch.trainer import BaseTrainer, TorchTensorEncoder from .mutator import PdartsMutator diff --git a/nni/nas/pytorch/proxylessnas/__init__.py b/nni/algorithms/nas/pytorch/proxylessnas/__init__.py similarity index 100% rename from nni/nas/pytorch/proxylessnas/__init__.py rename to nni/algorithms/nas/pytorch/proxylessnas/__init__.py diff --git a/nni/nas/pytorch/proxylessnas/mutator.py b/nni/algorithms/nas/pytorch/proxylessnas/mutator.py similarity index 100% rename from nni/nas/pytorch/proxylessnas/mutator.py rename to nni/algorithms/nas/pytorch/proxylessnas/mutator.py diff --git a/nni/nas/pytorch/proxylessnas/trainer.py b/nni/algorithms/nas/pytorch/proxylessnas/trainer.py similarity index 100% rename from nni/nas/pytorch/proxylessnas/trainer.py rename to nni/algorithms/nas/pytorch/proxylessnas/trainer.py diff --git a/nni/nas/pytorch/proxylessnas/utils.py b/nni/algorithms/nas/pytorch/proxylessnas/utils.py similarity index 100% rename from nni/nas/pytorch/proxylessnas/utils.py rename to nni/algorithms/nas/pytorch/proxylessnas/utils.py diff --git a/nni/nas/pytorch/random/__init__.py b/nni/algorithms/nas/pytorch/random/__init__.py similarity index 100% rename from nni/nas/pytorch/random/__init__.py rename to nni/algorithms/nas/pytorch/random/__init__.py diff --git a/nni/nas/pytorch/random/mutator.py b/nni/algorithms/nas/pytorch/random/mutator.py similarity index 100% rename from nni/nas/pytorch/random/mutator.py rename to nni/algorithms/nas/pytorch/random/mutator.py diff --git a/nni/nas/pytorch/spos/__init__.py b/nni/algorithms/nas/pytorch/spos/__init__.py similarity index 100% rename from nni/nas/pytorch/spos/__init__.py rename to nni/algorithms/nas/pytorch/spos/__init__.py diff --git a/nni/nas/pytorch/spos/evolution.py b/nni/algorithms/nas/pytorch/spos/evolution.py similarity index 99% rename from nni/nas/pytorch/spos/evolution.py rename to nni/algorithms/nas/pytorch/spos/evolution.py index 9042d30474..bd099e276e 100644 --- a/nni/nas/pytorch/spos/evolution.py +++ b/nni/algorithms/nas/pytorch/spos/evolution.py @@ -9,7 +9,7 @@ import numpy as np from nni.tuner import Tuner -from nni.nas.pytorch.classic_nas.mutator import LAYER_CHOICE, INPUT_CHOICE +from nni.algorithms.nas.pytorch.classic_nas.mutator import LAYER_CHOICE, INPUT_CHOICE _logger = logging.getLogger(__name__) diff --git a/nni/nas/pytorch/spos/mutator.py b/nni/algorithms/nas/pytorch/spos/mutator.py similarity index 100% rename from nni/nas/pytorch/spos/mutator.py rename to nni/algorithms/nas/pytorch/spos/mutator.py diff --git a/nni/nas/pytorch/spos/trainer.py b/nni/algorithms/nas/pytorch/spos/trainer.py similarity index 100% rename from nni/nas/pytorch/spos/trainer.py rename to nni/algorithms/nas/pytorch/spos/trainer.py diff --git a/nni/nas/tensorflow/classic_nas/__init__.py b/nni/algorithms/nas/tensorflow/classic_nas/__init__.py similarity index 100% rename from nni/nas/tensorflow/classic_nas/__init__.py rename to nni/algorithms/nas/tensorflow/classic_nas/__init__.py diff --git a/nni/nas/tensorflow/classic_nas/mutator.py b/nni/algorithms/nas/tensorflow/classic_nas/mutator.py similarity index 99% rename from nni/nas/tensorflow/classic_nas/mutator.py rename to nni/algorithms/nas/tensorflow/classic_nas/mutator.py index 4b9212f342..fad4987fed 100644 --- a/nni/nas/tensorflow/classic_nas/mutator.py +++ b/nni/algorithms/nas/tensorflow/classic_nas/mutator.py @@ -9,7 +9,7 @@ import tensorflow as tf import nni -from nni.env_vars import trial_env_vars +from nni.runtime.env_vars import trial_env_vars from nni.nas.tensorflow.mutables import LayerChoice, InputChoice, MutableScope from nni.nas.tensorflow.mutator import Mutator diff --git a/nni/nas/tensorflow/enas/__init__.py b/nni/algorithms/nas/tensorflow/enas/__init__.py similarity index 100% rename from nni/nas/tensorflow/enas/__init__.py rename to nni/algorithms/nas/tensorflow/enas/__init__.py diff --git a/nni/nas/tensorflow/enas/mutator.py b/nni/algorithms/nas/tensorflow/enas/mutator.py similarity index 100% rename from nni/nas/tensorflow/enas/mutator.py rename to nni/algorithms/nas/tensorflow/enas/mutator.py diff --git a/nni/nas/tensorflow/enas/trainer.py b/nni/algorithms/nas/tensorflow/enas/trainer.py similarity index 100% rename from nni/nas/tensorflow/enas/trainer.py rename to nni/algorithms/nas/tensorflow/enas/trainer.py diff --git a/nni/assessor.py b/nni/assessor.py index 9094de21e7..b70995cbad 100644 --- a/nni/assessor.py +++ b/nni/assessor.py @@ -50,8 +50,8 @@ class Assessor(Recoverable): See Also -------- Builtin assessors: - :class:`~nni.medianstop_assessor.MedianstopAssessor` - :class:`~nni.curvefitting_assessor.CurvefittingAssessor` + :class:`~nni.algorithms.hpo.medianstop_assessor.MedianstopAssessor` + :class:`~nni.algorithms.hpo.curvefitting_assessor.CurvefittingAssessor` """ def assess_trial(self, trial_job_id, trial_history): diff --git a/nni/_graph_utils.py b/nni/common/graph_utils.py similarity index 100% rename from nni/_graph_utils.py rename to nni/common/graph_utils.py diff --git a/nni/nas_utils.py b/nni/common/nas_utils.py similarity index 99% rename from nni/nas_utils.py rename to nni/common/nas_utils.py index 147faf1ec1..d7f050ec9b 100644 --- a/nni/nas_utils.py +++ b/nni/common/nas_utils.py @@ -4,7 +4,7 @@ import functools import logging -from . import trial +from .. import trial _logger = logging.getLogger(__name__) diff --git a/nni/compression/torch/__init__.py b/nni/compression/pytorch/__init__.py similarity index 76% rename from nni/compression/torch/__init__.py rename to nni/compression/pytorch/__init__.py index 4b55c0f5de..e20f284fef 100644 --- a/nni/compression/torch/__init__.py +++ b/nni/compression/pytorch/__init__.py @@ -2,6 +2,4 @@ # Licensed under the MIT license. from .speedup import ModelSpeedup -from .pruning import * -from .quantization import * from .compressor import Compressor, Pruner, Quantizer diff --git a/nni/compression/torch/compressor.py b/nni/compression/pytorch/compressor.py similarity index 100% rename from nni/compression/torch/compressor.py rename to nni/compression/pytorch/compressor.py diff --git a/nni/compression/torch/default_layers.py b/nni/compression/pytorch/default_layers.py similarity index 100% rename from nni/compression/torch/default_layers.py rename to nni/compression/pytorch/default_layers.py diff --git a/nni/compression/torch/speedup/__init__.py b/nni/compression/pytorch/speedup/__init__.py similarity index 100% rename from nni/compression/torch/speedup/__init__.py rename to nni/compression/pytorch/speedup/__init__.py diff --git a/nni/compression/torch/speedup/compress_modules.py b/nni/compression/pytorch/speedup/compress_modules.py similarity index 100% rename from nni/compression/torch/speedup/compress_modules.py rename to nni/compression/pytorch/speedup/compress_modules.py diff --git a/nni/compression/torch/speedup/compressor.py b/nni/compression/pytorch/speedup/compressor.py similarity index 97% rename from nni/compression/torch/speedup/compressor.py rename to nni/compression/pytorch/speedup/compressor.py index a9ae7aef54..0f308bbf18 100644 --- a/nni/compression/torch/speedup/compressor.py +++ b/nni/compression/pytorch/speedup/compressor.py @@ -3,8 +3,8 @@ import logging import torch -from nni.compression.torch.utils.mask_conflict import fix_mask_conflict -from nni.compression.torch.utils.utils import get_module_by_name +from nni.compression.pytorch.utils.mask_conflict import fix_mask_conflict +from nni.compression.pytorch.utils.utils import get_module_by_name from .compress_modules import replace_module from .infer_shape import ModuleMasks, infer_from_mask, infer_from_inshape, infer_from_outshape, set_conv_prune_dim @@ -29,7 +29,7 @@ def __init__(self, model, dummy_input, masks_file, map_location=None): map_location : str the device on which masks are placed, same to map_location in ```torch.load``` """ - from nni._graph_utils import build_module_graph + from nni.common.graph_utils import build_module_graph self.bound_model = model self.masks = torch.load(masks_file, map_location) diff --git a/nni/compression/torch/speedup/infer_shape.py b/nni/compression/pytorch/speedup/infer_shape.py similarity index 100% rename from nni/compression/torch/speedup/infer_shape.py rename to nni/compression/pytorch/speedup/infer_shape.py diff --git a/nni/nni_cmd/__init__.py b/nni/compression/pytorch/utils/__init__.py similarity index 100% rename from nni/nni_cmd/__init__.py rename to nni/compression/pytorch/utils/__init__.py diff --git a/nni/compression/torch/utils/config_validation.py b/nni/compression/pytorch/utils/config_validation.py similarity index 100% rename from nni/compression/torch/utils/config_validation.py rename to nni/compression/pytorch/utils/config_validation.py diff --git a/nni/compression/torch/utils/counter.py b/nni/compression/pytorch/utils/counter.py similarity index 98% rename from nni/compression/torch/utils/counter.py rename to nni/compression/pytorch/utils/counter.py index f4a3db7aa7..6061e8a8d0 100644 --- a/nni/compression/torch/utils/counter.py +++ b/nni/compression/pytorch/utils/counter.py @@ -3,7 +3,7 @@ import torch import torch.nn as nn -from nni.compression.torch.compressor import PrunerModuleWrapper +from nni.compression.pytorch.compressor import PrunerModuleWrapper try: from thop import profile @@ -132,4 +132,4 @@ def count_linear_mask(m, x, y): nn.Conv2d: count_convNd_mask, nn.Conv3d: count_convNd_mask, nn.Linear: count_linear_mask, -} \ No newline at end of file +} diff --git a/nni/compression/torch/utils/mask_conflict.py b/nni/compression/pytorch/utils/mask_conflict.py similarity index 100% rename from nni/compression/torch/utils/mask_conflict.py rename to nni/compression/pytorch/utils/mask_conflict.py diff --git a/nni/compression/torch/utils/num_param_counter.py b/nni/compression/pytorch/utils/num_param_counter.py similarity index 100% rename from nni/compression/torch/utils/num_param_counter.py rename to nni/compression/pytorch/utils/num_param_counter.py diff --git a/nni/compression/torch/utils/sensitivity_analysis.py b/nni/compression/pytorch/utils/sensitivity_analysis.py similarity index 98% rename from nni/compression/torch/utils/sensitivity_analysis.py rename to nni/compression/pytorch/utils/sensitivity_analysis.py index ef58237006..a36a523feb 100644 --- a/nni/compression/torch/utils/sensitivity_analysis.py +++ b/nni/compression/pytorch/utils/sensitivity_analysis.py @@ -9,7 +9,7 @@ import numpy as np import torch.nn as nn -from ..pruning.constants_pruner import PRUNER_DICT +# FIXME: I don't know where "utils" should be SUPPORTED_OP_NAME = ['Conv2d', 'Conv1d'] SUPPORTED_OP_TYPE = [getattr(nn, name) for name in SUPPORTED_OP_NAME] @@ -63,6 +63,8 @@ def __init__(self, model, val_func, sparsities=None, prune_type='l1', early_stop This value is effective only when the early_stop_mode is set. """ + from nni.algorithms.compression.pytorch.pruning.constants_pruner import PRUNER_DICT + self.model = model self.val_func = val_func self.target_layer = OrderedDict() diff --git a/nni/compression/torch/utils/shape_dependency.py b/nni/compression/pytorch/utils/shape_dependency.py similarity index 99% rename from nni/compression/torch/utils/shape_dependency.py rename to nni/compression/pytorch/utils/shape_dependency.py index a238848d86..3c2b63cd0b 100644 --- a/nni/compression/torch/utils/shape_dependency.py +++ b/nni/compression/pytorch/utils/shape_dependency.py @@ -17,7 +17,7 @@ def __init__(self, model=None, dummy_input=None, traced_model=None): """ Build the graph for the model. """ - from nni._graph_utils import TorchModuleGraph + from nni.common.graph_utils import TorchModuleGraph # check if the input is legal if traced_model is None: diff --git a/nni/compression/torch/utils/utils.py b/nni/compression/pytorch/utils/utils.py similarity index 100% rename from nni/compression/torch/utils/utils.py rename to nni/compression/pytorch/utils/utils.py diff --git a/nni/compression/tensorflow/__init__.py b/nni/compression/tensorflow/__init__.py index 00d41ee55b..d05fade2f1 100644 --- a/nni/compression/tensorflow/__init__.py +++ b/nni/compression/tensorflow/__init__.py @@ -2,4 +2,3 @@ # Licensed under the MIT license. from .compressor import Compressor, Pruner -from .pruning import * diff --git a/nni/constants.py b/nni/constants.py deleted file mode 100644 index 3e98f6eb6f..0000000000 --- a/nni/constants.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -BuiltinAlgorithms = { - 'tuners': [ - { - 'name': 'TPE', - 'class_name': 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', - 'class_args': { - 'algorithm_name': 'tpe' - }, - 'class_args_validator': 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' - }, - { - 'name': 'Random', - 'class_name': 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', - 'class_args': { - 'algorithm_name': 'random_search' - }, - 'accept_class_args': False, - 'class_args_validator': 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' - }, - { - 'name': 'Anneal', - 'class_name': 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', - 'class_args': { - 'algorithm_name': 'anneal' - }, - 'class_args_validator': 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' - }, - { - 'name': 'Evolution', - 'class_name': 'nni.evolution_tuner.evolution_tuner.EvolutionTuner', - 'class_args_validator': 'nni.evolution_tuner.evolution_tuner.EvolutionClassArgsValidator' - }, - { - 'name': 'BatchTuner', - 'class_name': 'nni.batch_tuner.batch_tuner.BatchTuner', - 'accept_class_args': False, - }, - { - 'name': 'GridSearch', - 'class_name': 'nni.gridsearch_tuner.gridsearch_tuner.GridSearchTuner', - 'accept_class_args': False, - }, - { - 'name': 'NetworkMorphism', - 'class_name': 'nni.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismTuner', - 'class_args_validator': 'nni.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismClassArgsValidator' - }, - { - 'name': 'MetisTuner', - 'class_name': 'nni.metis_tuner.metis_tuner.MetisTuner', - 'class_args_validator': 'nni.metis_tuner.metis_tuner.MetisClassArgsValidator' - }, - { - 'name': 'GPTuner', - 'class_name': 'nni.gp_tuner.gp_tuner.GPTuner', - 'class_args_validator': 'nni.gp_tuner.gp_tuner.GPClassArgsValidator' - }, - { - 'name': 'PBTTuner', - 'class_name': 'nni.pbt_tuner.pbt_tuner.PBTTuner', - 'class_args_validator': 'nni.pbt_tuner.pbt_tuner.PBTClassArgsValidator' - }, - { - 'name': 'RegularizedEvolutionTuner', - 'class_name': 'nni.regularized_evolution_tuner.regularized_evolution_tuner.RegularizedEvolutionTuner', - 'class_args_validator': 'nni.regularized_evolution_tuner.regularized_evolution_tuner.EvolutionClassArgsValidator' - } - ], - 'assessors': [ - { - 'name': 'Medianstop', - 'class_name': 'nni.medianstop_assessor.medianstop_assessor.MedianstopAssessor', - 'class_args_validator': 'nni.medianstop_assessor.medianstop_assessor.MedianstopClassArgsValidator' - }, - { - 'name': 'Curvefitting', - 'class_name': 'nni.curvefitting_assessor.curvefitting_assessor.CurvefittingAssessor', - 'class_args_validator': 'nni.curvefitting_assessor.curvefitting_assessor.CurvefittingClassArgsValidator' - }, - ], - 'advisors': [ - { - 'name': 'Hyperband', - 'class_name': 'nni.hyperband_advisor.hyperband_advisor.Hyperband', - 'class_args_validator': 'nni.hyperband_advisor.hyperband_advisor.HyperbandClassArgsValidator' - } - ] -} diff --git a/nni/nnicli/__init__.py b/nni/experiment/__init__.py similarity index 100% rename from nni/nnicli/__init__.py rename to nni/experiment/__init__.py diff --git a/nni/nnicli/nni_client.py b/nni/experiment/nni_client.py similarity index 98% rename from nni/nnicli/nni_client.py rename to nni/experiment/nni_client.py index 4a60390821..1dddafd219 100644 --- a/nni/nnicli/nni_client.py +++ b/nni/experiment/nni_client.py @@ -5,7 +5,7 @@ Example: -from nni.nnicli import Experiment +from nni.experiment import Experiment exp = Experiment() exp.start_experiment('../../../../examples/trials/mnist-pytorch/config.yml') @@ -196,16 +196,16 @@ class TrialJob: Trial job id. status: str Job status. - hyperParameters: list of `nnicli.TrialHyperParameters` - See `nnicli.TrialHyperParameters`. + hyperParameters: list of `nni.experiment.TrialHyperParameters` + See `nni.experiment.TrialHyperParameters`. logPath: str Log path. startTime: int Job start time (timestamp). endTime: int Job end time (timestamp). - finalMetricData: list of `nnicli.TrialMetricData` - See `nnicli.TrialMetricData`. + finalMetricData: list of `nni.experiment.TrialMetricData` + See `nni.experiment.TrialMetricData`. parameter_index: int Parameter index. """ diff --git a/nni/nas/pytorch/mutator.py b/nni/nas/pytorch/mutator.py index 1e97ab2123..1845ba73b2 100644 --- a/nni/nas/pytorch/mutator.py +++ b/nni/nas/pytorch/mutator.py @@ -107,7 +107,7 @@ def graph(self, inputs): """ if not torch.__version__.startswith("1.4"): logger.warning("Graph is only tested with PyTorch 1.4. Other versions might not work.") - from nni._graph_utils import build_graph + from nni.common.graph_utils import build_graph from google.protobuf import json_format # protobuf should be installed as long as tensorboard is installed try: diff --git a/nni/common.py b/nni/runtime/common.py similarity index 95% rename from nni/common.py rename to nni/runtime/common.py index 97532dae5d..5ec56fbb62 100644 --- a/nni/common.py +++ b/nni/runtime/common.py @@ -4,6 +4,7 @@ from datetime import datetime from io import TextIOBase import logging +import os import sys import time @@ -33,6 +34,9 @@ def init_logger(logger_file_path, log_level_name='info'): This will redirect anything from logging.getLogger() as well as stdout to specified file. logger_file_path: path of logger file (path-like object). """ + if os.environ.get('NNI_PLATFORM') == 'unittest': + return # fixme: launching logic needs refactor + log_level = log_level_map.get(log_level_name, logging.INFO) logger_file = open(logger_file_path, 'w') fmt = '[%(asctime)s] %(levelname)s (%(name)s/%(threadName)s) %(message)s' diff --git a/nni/env_vars.py b/nni/runtime/env_vars.py similarity index 100% rename from nni/env_vars.py rename to nni/runtime/env_vars.py diff --git a/nni/msg_dispatcher.py b/nni/runtime/msg_dispatcher.py similarity index 99% rename from nni/msg_dispatcher.py rename to nni/runtime/msg_dispatcher.py index 7e3232e4f9..4f24294fc7 100644 --- a/nni/msg_dispatcher.py +++ b/nni/runtime/msg_dispatcher.py @@ -8,10 +8,10 @@ from nni import NoMoreTrialError from .protocol import CommandType, send from .msg_dispatcher_base import MsgDispatcherBase -from .assessor import AssessResult +from nni.assessor import AssessResult from .common import multi_thread_enabled, multi_phase_enabled from .env_vars import dispatcher_env_vars -from .utils import MetricType, to_json +from ..utils import MetricType, to_json _logger = logging.getLogger(__name__) diff --git a/nni/msg_dispatcher_base.py b/nni/runtime/msg_dispatcher_base.py similarity index 99% rename from nni/msg_dispatcher_base.py rename to nni/runtime/msg_dispatcher_base.py index e323257d7f..66af52df28 100644 --- a/nni/msg_dispatcher_base.py +++ b/nni/runtime/msg_dispatcher_base.py @@ -9,8 +9,8 @@ from .common import multi_thread_enabled from .env_vars import dispatcher_env_vars -from .utils import init_dispatcher_logger -from .recoverable import Recoverable +from ..utils import init_dispatcher_logger +from ..recoverable import Recoverable from .protocol import CommandType, receive init_dispatcher_logger() diff --git a/nni/platform/__init__.py b/nni/runtime/platform/__init__.py similarity index 100% rename from nni/platform/__init__.py rename to nni/runtime/platform/__init__.py diff --git a/nni/platform/local.py b/nni/runtime/platform/local.py similarity index 98% rename from nni/platform/local.py rename to nni/runtime/platform/local.py index a25f27546f..5d8124d3ff 100644 --- a/nni/platform/local.py +++ b/nni/runtime/platform/local.py @@ -9,7 +9,7 @@ from ..common import init_logger from ..env_vars import trial_env_vars -from ..utils import to_json +from nni.utils import to_json _sysdir = trial_env_vars.NNI_SYS_DIR if not os.path.exists(os.path.join(_sysdir, '.nni')): diff --git a/nni/platform/standalone.py b/nni/runtime/platform/standalone.py similarity index 100% rename from nni/platform/standalone.py rename to nni/runtime/platform/standalone.py diff --git a/nni/platform/test.py b/nni/runtime/platform/test.py similarity index 100% rename from nni/platform/test.py rename to nni/runtime/platform/test.py diff --git a/nni/protocol.py b/nni/runtime/protocol.py similarity index 93% rename from nni/protocol.py rename to nni/runtime/protocol.py index ca2d7069d2..57ea7fbc0b 100644 --- a/nni/protocol.py +++ b/nni/runtime/protocol.py @@ -2,6 +2,7 @@ # Licensed under the MIT license. import logging +import os import threading from enum import Enum @@ -27,8 +28,9 @@ class CommandType(Enum): _lock = threading.Lock() try: - _in_file = open(3, 'rb') - _out_file = open(4, 'wb') + if os.environ.get('NNI_PLATFORM') != 'unittest': + _in_file = open(3, 'rb') + _out_file = open(4, 'wb') except OSError: _msg = 'IPC pipeline not exists, maybe you are importing tuner/assessor from trial code?' logging.getLogger(__name__).warning(_msg) diff --git a/nni/smartparam.py b/nni/smartparam.py index 5841714e11..dde0ac2bd6 100644 --- a/nni/smartparam.py +++ b/nni/smartparam.py @@ -3,10 +3,10 @@ import numpy as np -from .env_vars import trial_env_vars +from .runtime.env_vars import trial_env_vars from . import trial from . import parameter_expressions as param_exp -from .nas_utils import classic_mode, enas_mode, oneshot_mode, darts_mode +from .common.nas_utils import classic_mode, enas_mode, oneshot_mode, darts_mode __all__ = [ diff --git a/nni/nni_annotation/.gitignore b/nni/tools/annotation/.gitignore similarity index 100% rename from nni/nni_annotation/.gitignore rename to nni/tools/annotation/.gitignore diff --git a/nni/nni_annotation/__init__.py b/nni/tools/annotation/__init__.py similarity index 100% rename from nni/nni_annotation/__init__.py rename to nni/tools/annotation/__init__.py diff --git a/nni/nni_annotation/code_generator.py b/nni/tools/annotation/code_generator.py similarity index 100% rename from nni/nni_annotation/code_generator.py rename to nni/tools/annotation/code_generator.py diff --git a/nni/nni_annotation/search_space_generator.py b/nni/tools/annotation/search_space_generator.py similarity index 100% rename from nni/nni_annotation/search_space_generator.py rename to nni/tools/annotation/search_space_generator.py diff --git a/nni/nni_annotation/specific_code_generator.py b/nni/tools/annotation/specific_code_generator.py similarity index 99% rename from nni/nni_annotation/specific_code_generator.py rename to nni/tools/annotation/specific_code_generator.py index 37a3966a81..08659594ea 100644 --- a/nni/nni_annotation/specific_code_generator.py +++ b/nni/tools/annotation/specific_code_generator.py @@ -3,7 +3,7 @@ import ast import astor -from nni.nni_cmd.common_utils import print_warning +from nni.tools.nnictl.common_utils import print_warning from .utils import ast_Num, ast_Str diff --git a/nni/nni_annotation/utils.py b/nni/tools/annotation/utils.py similarity index 100% rename from nni/nni_annotation/utils.py rename to nni/tools/annotation/utils.py diff --git a/nni/nni_gpu_tool/__init__.py b/nni/tools/gpu_tool/__init__.py similarity index 100% rename from nni/nni_gpu_tool/__init__.py rename to nni/tools/gpu_tool/__init__.py diff --git a/nni/nni_gpu_tool/gpu_metrics_collector.py b/nni/tools/gpu_tool/gpu_metrics_collector.py similarity index 100% rename from nni/nni_gpu_tool/gpu_metrics_collector.py rename to nni/tools/gpu_tool/gpu_metrics_collector.py diff --git a/nni/nni_trial_tool/__init__.py b/nni/tools/nnictl/__init__.py similarity index 100% rename from nni/nni_trial_tool/__init__.py rename to nni/tools/nnictl/__init__.py diff --git a/nni/nni_cmd/command_utils.py b/nni/tools/nnictl/command_utils.py similarity index 100% rename from nni/nni_cmd/command_utils.py rename to nni/tools/nnictl/command_utils.py diff --git a/nni/nni_cmd/common_utils.py b/nni/tools/nnictl/common_utils.py similarity index 100% rename from nni/nni_cmd/common_utils.py rename to nni/tools/nnictl/common_utils.py diff --git a/nni/nni_cmd/config_schema.py b/nni/tools/nnictl/config_schema.py similarity index 99% rename from nni/nni_cmd/config_schema.py rename to nni/tools/nnictl/config_schema.py index 3c8115a5e8..d320163595 100644 --- a/nni/nni_cmd/config_schema.py +++ b/nni/tools/nnictl/config_schema.py @@ -6,7 +6,7 @@ import os import netifaces from schema import Schema, And, Optional, Regex, Or, SchemaError -from nni.package_utils import create_validator_instance, get_all_builtin_names, get_builtin_algo_meta +from nni.tools.package_utils import create_validator_instance, get_all_builtin_names, get_builtin_algo_meta from .constants import SCHEMA_TYPE_ERROR, SCHEMA_RANGE_ERROR, SCHEMA_PATH_ERROR from .common_utils import get_yml_content, print_warning diff --git a/nni/nni_cmd/config_utils.py b/nni/tools/nnictl/config_utils.py similarity index 100% rename from nni/nni_cmd/config_utils.py rename to nni/tools/nnictl/config_utils.py diff --git a/nni/nni_cmd/constants.py b/nni/tools/nnictl/constants.py similarity index 100% rename from nni/nni_cmd/constants.py rename to nni/tools/nnictl/constants.py diff --git a/nni/nni_cmd/launcher.py b/nni/tools/nnictl/launcher.py similarity index 99% rename from nni/nni_cmd/launcher.py rename to nni/tools/nnictl/launcher.py index 068d7a3ae4..2c15608454 100644 --- a/nni/nni_cmd/launcher.py +++ b/nni/tools/nnictl/launcher.py @@ -9,8 +9,8 @@ import time import tempfile from subprocess import Popen, check_call, CalledProcessError, PIPE, STDOUT -from nni.nni_annotation import expand_annotations, generate_search_space -from nni.package_utils import get_builtin_module_class_name +from nni.tools.annotation import expand_annotations, generate_search_space +from nni.tools.package_utils import get_builtin_module_class_name import nni_node from .launcher_utils import validate_all_content from .rest_utils import rest_put, rest_post, check_rest_server, check_response diff --git a/nni/nni_cmd/launcher_utils.py b/nni/tools/nnictl/launcher_utils.py similarity index 100% rename from nni/nni_cmd/launcher_utils.py rename to nni/tools/nnictl/launcher_utils.py diff --git a/nni/nni_cmd/nnictl.py b/nni/tools/nnictl/nnictl.py similarity index 100% rename from nni/nni_cmd/nnictl.py rename to nni/tools/nnictl/nnictl.py diff --git a/nni/nni_cmd/nnictl_utils.py b/nni/tools/nnictl/nnictl_utils.py similarity index 99% rename from nni/nni_cmd/nnictl_utils.py rename to nni/tools/nnictl/nnictl_utils.py index 8d70175990..3427d698b0 100644 --- a/nni/nni_cmd/nnictl_utils.py +++ b/nni/tools/nnictl/nnictl_utils.py @@ -13,7 +13,7 @@ from datetime import datetime, timezone from subprocess import Popen from pyhdfs import HdfsClient -from nni.nni_annotation import expand_annotations +from nni.tools.annotation import expand_annotations import nni_node from .rest_utils import rest_get, rest_delete, check_rest_server_quick, check_response from .url_utils import trial_jobs_url, experiment_url, trial_job_id_url, export_data_url, metric_data_url diff --git a/nni/nni_cmd/package_management.py b/nni/tools/nnictl/package_management.py similarity index 96% rename from nni/nni_cmd/package_management.py rename to nni/tools/nnictl/package_management.py index 942665b97a..5eef340752 100644 --- a/nni/nni_cmd/package_management.py +++ b/nni/tools/nnictl/package_management.py @@ -6,7 +6,7 @@ import json import pkginfo import nni -from nni.package_utils import read_installed_package_meta, get_installed_package_meta, \ +from nni.tools.package_utils import read_installed_package_meta, get_installed_package_meta, \ write_package_meta, get_builtin_algo_meta, get_not_installable_builtin_names, ALGO_TYPES from .constants import INSTALLABLE_PACKAGE_META @@ -19,7 +19,7 @@ def install_by_name(package_name): if package_name not in INSTALLABLE_PACKAGE_META: raise RuntimeError('{} is not found in installable packages!'.format(package_name)) - requirements_path = os.path.join(nni.__path__[0], INSTALLABLE_PACKAGE_META[package_name]['code_sub_dir'], 'requirements.txt') + requirements_path = os.path.join(nni.__path__[0], 'algorithms/hpo', INSTALLABLE_PACKAGE_META[package_name]['code_sub_dir'], 'requirements.txt') assert os.path.exists(requirements_path) return install_requirements_command(requirements_path) diff --git a/nni/nni_cmd/rest_utils.py b/nni/tools/nnictl/rest_utils.py similarity index 100% rename from nni/nni_cmd/rest_utils.py rename to nni/tools/nnictl/rest_utils.py diff --git a/nni/nni_cmd/ssh_utils.py b/nni/tools/nnictl/ssh_utils.py similarity index 100% rename from nni/nni_cmd/ssh_utils.py rename to nni/tools/nnictl/ssh_utils.py diff --git a/nni/nni_cmd/tensorboard_utils.py b/nni/tools/nnictl/tensorboard_utils.py similarity index 100% rename from nni/nni_cmd/tensorboard_utils.py rename to nni/tools/nnictl/tensorboard_utils.py diff --git a/nni/nni_cmd/updater.py b/nni/tools/nnictl/updater.py similarity index 100% rename from nni/nni_cmd/updater.py rename to nni/tools/nnictl/updater.py diff --git a/nni/nni_cmd/url_utils.py b/nni/tools/nnictl/url_utils.py similarity index 100% rename from nni/nni_cmd/url_utils.py rename to nni/tools/nnictl/url_utils.py diff --git a/nni/package_utils.py b/nni/tools/package_utils/__init__.py similarity index 99% rename from nni/package_utils.py rename to nni/tools/package_utils/__init__.py index b55073d709..9a1c004053 100644 --- a/nni/package_utils.py +++ b/nni/tools/package_utils/__init__.py @@ -1,12 +1,12 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +from collections import defaultdict +import importlib import os from pathlib import Path import sys -from collections import defaultdict -from pathlib import Path -import importlib + import ruamel.yaml as yaml import nni diff --git a/nni/tools/package_utils/constants.py b/nni/tools/package_utils/constants.py new file mode 100644 index 0000000000..952310b5f9 --- /dev/null +++ b/nni/tools/package_utils/constants.py @@ -0,0 +1,91 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +BuiltinAlgorithms = { + 'tuners': [ + { + 'name': 'TPE', + 'class_name': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', + 'class_args': { + 'algorithm_name': 'tpe' + }, + 'class_args_validator': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' + }, + { + 'name': 'Random', + 'class_name': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', + 'class_args': { + 'algorithm_name': 'random_search' + }, + 'accept_class_args': False, + 'class_args_validator': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' + }, + { + 'name': 'Anneal', + 'class_name': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', + 'class_args': { + 'algorithm_name': 'anneal' + }, + 'class_args_validator': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' + }, + { + 'name': 'Evolution', + 'class_name': 'nni.algorithms.hpo.evolution_tuner.evolution_tuner.EvolutionTuner', + 'class_args_validator': 'nni.algorithms.hpo.evolution_tuner.evolution_tuner.EvolutionClassArgsValidator' + }, + { + 'name': 'BatchTuner', + 'class_name': 'nni.algorithms.hpo.batch_tuner.batch_tuner.BatchTuner', + 'accept_class_args': False, + }, + { + 'name': 'GridSearch', + 'class_name': 'nni.algorithms.hpo.gridsearch_tuner.gridsearch_tuner.GridSearchTuner', + 'accept_class_args': False, + }, + { + 'name': 'NetworkMorphism', + 'class_name': 'nni.algorithms.hpo.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismTuner', + 'class_args_validator': 'nni.algorithms.hpo.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismClassArgsValidator' + }, + { + 'name': 'MetisTuner', + 'class_name': 'nni.algorithms.hpo.metis_tuner.metis_tuner.MetisTuner', + 'class_args_validator': 'nni.algorithms.hpo.metis_tuner.metis_tuner.MetisClassArgsValidator' + }, + { + 'name': 'GPTuner', + 'class_name': 'nni.algorithms.hpo.gp_tuner.gp_tuner.GPTuner', + 'class_args_validator': 'nni.algorithms.hpo.gp_tuner.gp_tuner.GPClassArgsValidator' + }, + { + 'name': 'PBTTuner', + 'class_name': 'nni.algorithms.hpo.pbt_tuner.pbt_tuner.PBTTuner', + 'class_args_validator': 'nni.algorithms.hpo.pbt_tuner.pbt_tuner.PBTClassArgsValidator' + }, + { + 'name': 'RegularizedEvolutionTuner', + 'class_name': 'nni.algorithms.hpo.regularized_evolution_tuner.regularized_evolution_tuner.RegularizedEvolutionTuner', + 'class_args_validator': 'nni.algorithms.hpo.regularized_evolution_tuner.regularized_evolution_tuner.EvolutionClassArgsValidator' + } + ], + 'assessors': [ + { + 'name': 'Medianstop', + 'class_name': 'nni.algorithms.hpo.medianstop_assessor.medianstop_assessor.MedianstopAssessor', + 'class_args_validator': 'nni.algorithms.hpo.medianstop_assessor.medianstop_assessor.MedianstopClassArgsValidator' + }, + { + 'name': 'Curvefitting', + 'class_name': 'nni.algorithms.hpo.curvefitting_assessor.curvefitting_assessor.CurvefittingAssessor', + 'class_args_validator': 'nni.algorithms.hpo.curvefitting_assessor.curvefitting_assessor.CurvefittingClassArgsValidator' + }, + ], + 'advisors': [ + { + 'name': 'Hyperband', + 'class_name': 'nni.algorithms.hpo.hyperband_advisor.hyperband_advisor.Hyperband', + 'class_args_validator': 'nni.algorithms.hpo.hyperband_advisor.hyperband_advisor.HyperbandClassArgsValidator' + } + ] +} diff --git a/nni/pbt_tuner/__init__.py b/nni/tools/trial_tool/__init__.py similarity index 100% rename from nni/pbt_tuner/__init__.py rename to nni/tools/trial_tool/__init__.py diff --git a/nni/nni_trial_tool/aml_channel.py b/nni/tools/trial_tool/aml_channel.py similarity index 100% rename from nni/nni_trial_tool/aml_channel.py rename to nni/tools/trial_tool/aml_channel.py diff --git a/nni/nni_trial_tool/base_channel.py b/nni/tools/trial_tool/base_channel.py similarity index 100% rename from nni/nni_trial_tool/base_channel.py rename to nni/tools/trial_tool/base_channel.py diff --git a/nni/nni_trial_tool/commands.py b/nni/tools/trial_tool/commands.py similarity index 100% rename from nni/nni_trial_tool/commands.py rename to nni/tools/trial_tool/commands.py diff --git a/nni/nni_trial_tool/constants.py b/nni/tools/trial_tool/constants.py similarity index 100% rename from nni/nni_trial_tool/constants.py rename to nni/tools/trial_tool/constants.py diff --git a/nni/nni_trial_tool/file_channel.py b/nni/tools/trial_tool/file_channel.py similarity index 100% rename from nni/nni_trial_tool/file_channel.py rename to nni/tools/trial_tool/file_channel.py diff --git a/nni/nni_trial_tool/gpu.py b/nni/tools/trial_tool/gpu.py similarity index 100% rename from nni/nni_trial_tool/gpu.py rename to nni/tools/trial_tool/gpu.py diff --git a/nni/nni_trial_tool/hdfsClientUtility.py b/nni/tools/trial_tool/hdfsClientUtility.py similarity index 100% rename from nni/nni_trial_tool/hdfsClientUtility.py rename to nni/tools/trial_tool/hdfsClientUtility.py diff --git a/nni/nni_trial_tool/log_utils.py b/nni/tools/trial_tool/log_utils.py similarity index 100% rename from nni/nni_trial_tool/log_utils.py rename to nni/tools/trial_tool/log_utils.py diff --git a/nni/nni_trial_tool/rest_utils.py b/nni/tools/trial_tool/rest_utils.py similarity index 100% rename from nni/nni_trial_tool/rest_utils.py rename to nni/tools/trial_tool/rest_utils.py diff --git a/nni/nni_trial_tool/trial.py b/nni/tools/trial_tool/trial.py similarity index 100% rename from nni/nni_trial_tool/trial.py rename to nni/tools/trial_tool/trial.py diff --git a/nni/nni_trial_tool/trial_keeper.py b/nni/tools/trial_tool/trial_keeper.py similarity index 100% rename from nni/nni_trial_tool/trial_keeper.py rename to nni/tools/trial_tool/trial_keeper.py diff --git a/nni/nni_trial_tool/trial_runner.py b/nni/tools/trial_tool/trial_runner.py similarity index 100% rename from nni/nni_trial_tool/trial_runner.py rename to nni/tools/trial_tool/trial_runner.py diff --git a/nni/nni_trial_tool/url_utils.py b/nni/tools/trial_tool/url_utils.py similarity index 100% rename from nni/nni_trial_tool/url_utils.py rename to nni/tools/trial_tool/url_utils.py diff --git a/nni/nni_trial_tool/web_channel.py b/nni/tools/trial_tool/web_channel.py similarity index 100% rename from nni/nni_trial_tool/web_channel.py rename to nni/tools/trial_tool/web_channel.py diff --git a/nni/trial.py b/nni/trial.py index d4851316f5..cdb2b1e683 100644 --- a/nni/trial.py +++ b/nni/trial.py @@ -2,8 +2,8 @@ # Licensed under the MIT license. from .utils import to_json -from .env_vars import trial_env_vars -from . import platform +from .runtime.env_vars import trial_env_vars +from .runtime import platform __all__ = [ diff --git a/nni/tuner.py b/nni/tuner.py index 0f8bb1e352..4fbcc011d0 100644 --- a/nni/tuner.py +++ b/nni/tuner.py @@ -57,14 +57,14 @@ class Tuner(Recoverable): See Also -------- Builtin tuners: - :class:`~nni.hyperopt_tuner.hyperopt_tuner.HyperoptTuner` - :class:`~nni.evolution_tuner.evolution_tuner.EvolutionTuner` - :class:`~nni.smac_tuner.SMACTuner` - :class:`~nni.gridsearch_tuner.GridSearchTuner` - :class:`~nni.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismTuner` - :class:`~nni.metis_tuner.mets_tuner.MetisTuner` - :class:`~nni.ppo_tuner.PPOTuner` - :class:`~nni.gp_tuner.gp_tuner.GPTuner` + :class:`~nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner` + :class:`~nni.algorithms.hpo.evolution_tuner.evolution_tuner.EvolutionTuner` + :class:`~nni.algorithms.hpo.smac_tuner.SMACTuner` + :class:`~nni.algorithms.hpo.gridsearch_tuner.GridSearchTuner` + :class:`~nni.algorithms.hpo.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismTuner` + :class:`~nni.algorithms.hpo.metis_tuner.mets_tuner.MetisTuner` + :class:`~nni.algorithms.hpo.ppo_tuner.PPOTuner` + :class:`~nni.algorithms.hpo.gp_tuner.gp_tuner.GPTuner` """ def generate_parameters(self, parameter_id, **kwargs): diff --git a/nni/utils.py b/nni/utils.py index 0346aec9fc..fe782dae2c 100644 --- a/nni/utils.py +++ b/nni/utils.py @@ -9,8 +9,8 @@ from schema import And from . import parameter_expressions -from .common import init_logger -from .env_vars import dispatcher_env_vars +from .runtime.common import init_logger +from .runtime.env_vars import dispatcher_env_vars to_json = functools.partial(json_tricks.dumps, allow_nan=True) diff --git a/setup.py b/setup.py index e5b84f1019..0011d0b20e 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,125 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +""" +Script for installation and distribution. + +You can use environment variable `NNI_RELEASE` to set release version. + +If release version is not set, default to a development build whose version string will be `999.dev0`. + + +## Development ## + +Build and install for development: + + $ python setup.py develop + +Uninstall: + + $ pip uninstall nni + +Remove generated files: (use "--all" to remove toolchain and built wheel) + + $ python setup.py clean [--all] + +Build TypeScript modules without install: + + $ python setup.py build_ts + + +## Release ## + +Build wheel package: + + $ NNI_RELEASE=2.0 python setup.py build_ts + $ NNI_RELEASE=2.0 python setup.py bdist_wheel -p manylinux1_x86_64 + +Where "2.0" is version string and "manylinux1_x86_64" is platform. +The platform may also be "macosx_10_9_x86_64" or "win_amd64". + +`build_ts` must be manually invoked before `bdist_wheel`, +or setuptools cannot locate JS files which should be packed into wheel. +""" + +from distutils.cmd import Command +from distutils.command.build import build +from distutils.command.clean import clean +import glob import os -from setuptools import setup +import shutil + +import setuptools +from setuptools.command.develop import develop + +import setup_ts -version = '999.0.0-developing' +dependencies = [ + 'astor', + 'hyperopt==0.1.2', + 'json_tricks', + 'netifaces', + 'numpy', + 'psutil', + 'ruamel.yaml', + 'requests', + 'responses', + 'scipy', + 'schema', + 'PythonWebHDFS', + 'colorama', + 'scikit-learn>=0.23.2', + 'pkginfo', + 'websockets' +] + + +release = os.environ.get('NNI_RELEASE') + +def _setup(): + setuptools.setup( + name = 'nni', + version = release or '999.dev0', + description = 'Neural Network Intelligence project', + long_description = open('README.md', encoding='utf-8').read(), + long_description_content_type = 'text/markdown', + url = 'https://github.com/Microsoft/nni', + author = 'Microsoft NNI Team', + author_email = 'nni@microsoft.com', + license = 'MIT', + classifiers = [ + 'License :: OSI Approved :: MIT License', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: Microsoft :: Windows :: Windows 10', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python :: 3 :: Only', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + ], + + packages = _find_python_packages(), + package_data = { + 'nni': ['**/requirements.txt'], + 'nni_node': _find_node_files() # note: this does not work before building + }, + + python_requires = '>=3.6', + install_requires = dependencies, + setup_requires = ['requests'], + + entry_points = { + 'console_scripts' : [ + 'nnictl = nni.tools.nnictl.nnictl:parse_args' + ] + }, + + cmdclass = { + 'build': Build, + 'build_ts': BuildTs, + 'clean': Clean, + 'develop': Develop, + } + ) def _find_python_packages(): @@ -24,45 +138,65 @@ def _find_node_files(): return sorted(files) -setup( - name = 'nni', - version = version, - author = 'Microsoft NNI Team', - author_email = 'nni@microsoft.com', - description = 'Neural Network Intelligence project', - long_description = open('README.md', encoding='utf-8').read(), - license = 'MIT', - url = 'https://github.com/Microsoft/nni', - - packages = _find_python_packages(), - package_data = { - 'nni': ['**/requirements.txt'], - 'nni_node': _find_node_files() - }, - - python_requires = '>=3.6', - install_requires = [ - 'astor', - 'hyperopt==0.1.2', - 'json_tricks', - 'netifaces', - 'numpy', - 'psutil', - 'ruamel.yaml', - 'requests', - 'responses', - 'scipy', - 'schema', - 'PythonWebHDFS', - 'colorama', - 'scikit-learn>=0.23.2', - 'pkginfo', - 'websockets' - ], - - entry_points = { - 'console_scripts' : [ - 'nnictl = nni.nni_cmd.nnictl:parse_args' - ] - } -) +class BuildTs(Command): + description = 'build TypeScript modules' + + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + setup_ts.build(release) + +class Build(build): + def run(self): + assert release, 'Please set environment variable "NNI_RELEASE="' + assert os.path.isfile('nni_node/main.js'), 'Please run "build_ts" before "build"' + assert not os.path.islink('nni_node/main.js'), 'This is a development build' + super().run() + +class Develop(develop): + def finalize_options(self): + self.user = True # always use `develop --user` + super().finalize_options() + + def run(self): + setup_ts.build(release=None) + super().run() + +class Clean(clean): + def finalize_options(self): + self._all = self.all + self.all = True # always use `clean --all` + super().finalize_options() + + def run(self): + super().run() + setup_ts.clean(self._all) + _clean_temp_files() + shutil.rmtree('nni.egg-info', ignore_errors=True) + if self._all: + shutil.rmtree('dist', ignore_errors=True) + + +def _clean_temp_files(): + for pattern in _temp_files: + for path in glob.glob(pattern): + if os.path.islink(path) or os.path.isfile(path): + os.remove(path) + else: + shutil.rmtree(path) + +_temp_files = [ + # unit test + 'test/model_path/', + 'test/temp.json', + 'test/ut/sdk/*.pth' +] + + +_setup() diff --git a/setup_ts.py b/setup_ts.py new file mode 100644 index 0000000000..5872ce7bbd --- /dev/null +++ b/setup_ts.py @@ -0,0 +1,229 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Script for building TypeScript modules. +This script is called by `setup.py` and common users should avoid using this directly. + +It compiles TypeScript source files in `ts` directory, +and copies (or links) JavaScript output as well as dependencies to `nni_node`. + +You can set environment `GLOBAL_TOOLCHAIN=1` to use global node and yarn, if you know what you are doing. +""" + +from io import BytesIO +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tarfile +from zipfile import ZipFile + + +node_version = 'v10.22.1' +yarn_version = 'v1.22.10' + + +def build(release): + """ + Compile TypeScript modules and copy or symlink to nni_node directory. + + `release` is the version number without leading letter "v". + + If `release` is None or empty, this is a development build and uses symlinks; + otherwise this is a release build and copies files instead. + """ + if release or not os.environ.get('GLOBAL_TOOLCHAIN'): + download_toolchain() + compile_ts() + if release: + copy_nni_node(release) + else: + symlink_nni_node() + +def clean(clean_all=False): + """ + Remove TypeScript-related intermediate files. + Python intermediate files are not touched here. + """ + clear_nni_node() + for path in generated_directories: + shutil.rmtree(path, ignore_errors=True) + if clean_all: + shutil.rmtree('toolchain', ignore_errors=True) + Path('nni_node', node_executable).unlink() + + +if sys.platform == 'linux' or sys.platform == 'darwin': + node_executable = 'node' + node_spec = f'node-{node_version}-{sys.platform}-x64' + node_download_url = f'https://nodejs.org/dist/latest-v10.x/{node_spec}.tar.xz' + node_extractor = lambda data: tarfile.open(fileobj=BytesIO(data), mode='r:xz') + node_executable_in_tarball = 'bin/node' + +elif sys.platform == 'win32': + node_executable = 'node.exe' + node_spec = f'node-{node_version}-win-x64' + node_download_url = f'https://nodejs.org/dist/latest-v10.x/{node_spec}.zip' + node_extractor = lambda data: ZipFile(BytesIO(data)) + node_executable_in_tarball = 'node.exe' + +else: + raise RuntimeError('Unsupported system') + +yarn_executable = 'yarn' if sys.platform != 'win32' else 'yarn.cmd' +yarn_download_url = f'https://github.com/yarnpkg/yarn/releases/download/{yarn_version}/yarn-{yarn_version}.tar.gz' + + +def download_toolchain(): + """ + Download and extract node and yarn, + then copy node executable to nni_node directory. + """ + if Path('nni_node', node_executable).is_file(): + return + Path('toolchain').mkdir(exist_ok=True) + import requests # place it here so setup.py can install it before importing + + _print(f'Downloading node.js from {node_download_url}') + resp = requests.get(node_download_url) + resp.raise_for_status() + _print('Extracting node.js') + tarball = node_extractor(resp.content) + tarball.extractall('toolchain') + shutil.rmtree('toolchain/node', ignore_errors=True) + Path('toolchain', node_spec).rename('toolchain/node') + + _print(f'Downloading yarn from {yarn_download_url}') + resp = requests.get(yarn_download_url) + resp.raise_for_status() + _print('Extracting yarn') + tarball = tarfile.open(fileobj=BytesIO(resp.content), mode='r:gz') + tarball.extractall('toolchain') + shutil.rmtree('toolchain/yarn', ignore_errors=True) + Path(f'toolchain/yarn-{yarn_version}').rename('toolchain/yarn') + + src = Path('toolchain/node', node_executable_in_tarball) + dst = Path('nni_node', node_executable) + shutil.copyfile(src, dst) + + +def compile_ts(): + """ + Use yarn to download dependencies and compile TypeScript code. + """ + _print('Building NNI manager') + _yarn('ts/nni_manager') + _yarn('ts/nni_manager', 'build') + # todo: I don't think these should be here + shutil.rmtree('ts/nni_manager/dist/config', ignore_errors=True) + shutil.copytree('ts/nni_manager/config', 'ts/nni_manager/dist/config') + + _print('Building web UI') + _yarn('ts/webui') + _yarn('ts/webui', 'build') + + _print('Building NAS UI') + _yarn('ts/nasui') + _yarn('ts/nasui', 'build') + + +def symlink_nni_node(): + """ + Create symlinks to compiled JS files. + If you manually modify and compile TS source files you don't need to install again. + """ + _print('Creating symlinks') + clear_nni_node() + + for path in Path('ts/nni_manager/dist').iterdir(): + _symlink(path, Path('nni_node', path.name)) + _symlink('ts/nni_manager/package.json', 'nni_node/package.json') + _symlink('ts/nni_manager/node_modules', 'nni_node/node_modules') + + _symlink('ts/webui/build', 'nni_node/static') + + Path('nni_node/nasui').mkdir(exist_ok=True) + _symlink('ts/nasui/build', 'nni_node/nasui/build') + _symlink('ts/nasui/server.js', 'nni_node/nasui/server.js') + + +def copy_nni_node(version): + """ + Copy compiled JS files to nni_node. + This is meant for building release package, so you need to provide version string. + The version will written to `package.json` in nni_node directory, + while `package.json` in ts directory will be left unchanged. + """ + _print('Copying files') + clear_nni_node() + + # copytree(..., dirs_exist_ok=True) is not supported by Python 3.6 + for path in Path('ts/nni_manager/dist').iterdir(): + if path.is_file(): + shutil.copyfile(path, Path('nni_node', path.name)) + else: + shutil.copytree(path, Path('nni_node', path.name)) + + package_json = json.load(open('ts/nni_manager/package.json')) + if version.count('.') == 1: # node.js semver requires at least three parts + version = version + '.0' + package_json['version'] = version + json.dump(package_json, open('nni_node/package.json', 'w'), indent=2) + + _yarn('ts/nni_manager', '--prod', '--cwd', str(Path('nni_node').resolve())) + + shutil.copytree('ts/webui/build', 'nni_node/static') + + Path('nni_node/nasui').mkdir(exist_ok=True) + shutil.copytree('ts/nasui/build', 'nni_node/nasui/build') + shutil.copyfile('ts/nasui/server.js', 'nni_node/nasui/server.js') + + +def clear_nni_node(): + """ + Remove compiled files in nni_node. + Use `clean()` if you what to remove files in ts as well. + """ + for path in Path('nni_node').iterdir(): + if path.name not in ('__init__.py', 'node', 'node.exe'): + if path.is_symlink() or path.is_file(): + path.unlink() + else: + shutil.rmtree(path) + + +_yarn_env = dict(os.environ) +_yarn_env['PATH'] = str(Path('nni_node').resolve()) + ':' + os.environ['PATH'] +_yarn_path = Path('toolchain/yarn/bin', yarn_executable).resolve() + +def _yarn(path, *args): + if os.environ.get('GLOBAL_TOOLCHAIN'): + subprocess.run(['yarn', *args], cwd=path, check=True) + else: + subprocess.run([_yarn_path, *args], cwd=path, check=True, env=_yarn_env) + + +def _symlink(target_file, link_location): + target = Path(target_file) + link = Path(link_location) + relative = os.path.relpath(target, link.parent) + link.symlink_to(relative, target.is_dir()) + + +def _print(*args): + print('\033[1;36m# ', end='') + print(*args, end='') + print('\033[0m') + + +generated_directories = [ + 'ts/nni_manager/dist', + 'ts/nni_manager/node_modules', + 'ts/webui/build', + 'ts/webui/node_modules', + 'ts/nasui/build', + 'ts/nasui/node_modules', +] diff --git a/test/.coveragerc b/test/.coveragerc index fdcb4e097d..1565dbf27b 100644 --- a/test/.coveragerc +++ b/test/.coveragerc @@ -3,7 +3,7 @@ branch = True parallel = True data_file = ${COVERAGE_DATA_FILE} -source = nni, nni_cmd, nni_trial_tool +source = nni, nni.tools.cmd, nni.tools.trial_tool concurrency = multiprocessing diff --git a/test/config/integration_tests.yml b/test/config/integration_tests.yml index d826a79785..40f6a39165 100644 --- a/test/config/integration_tests.yml +++ b/test/config/integration_tests.yml @@ -147,8 +147,8 @@ testCases: config: maxTrialNum: 4 trialConcurrency: 4 - launchCommand: python3 -c 'from nni.nnicli import Experiment; exp = Experiment(); exp.start_experiment("$configFile")' - stopCommand: python3 -c 'from nni.nnicli import Experiment; exp = Experiment(); exp.connect_experiment("http://localhost:8080/"); exp.stop_experiment()' + launchCommand: python3 -c 'from nni.experiment import Experiment; exp = Experiment(); exp.start_experiment("$configFile")' + stopCommand: python3 -c 'from nni.experiment import Experiment; exp = Experiment(); exp.connect_experiment("http://localhost:8080/"); exp.stop_experiment()' validator: class: NnicliValidator platform: linux darwin diff --git a/test/config/integration_tests_tf2.yml b/test/config/integration_tests_tf2.yml index e4be58f7d9..8bc74f9b7a 100644 --- a/test/config/integration_tests_tf2.yml +++ b/test/config/integration_tests_tf2.yml @@ -110,8 +110,8 @@ testCases: config: maxTrialNum: 4 trialConcurrency: 4 - launchCommand: python3 -c 'from nni.nnicli import Experiment; exp = Experiment(); exp.start_experiment("$configFile")' - stopCommand: python3 -c 'from nni.nnicli import Experiment; exp = Experiment(); exp.connect_experiment("http://localhost:8080/"); exp.stop_experiment()' + launchCommand: python3 -c 'from nni.experiment import Experiment; exp = Experiment(); exp.start_experiment("$configFile")' + stopCommand: python3 -c 'from nni.experiment import Experiment; exp = Experiment(); exp.connect_experiment("http://localhost:8080/"); exp.stop_experiment()' validator: class: NnicliValidator platform: linux darwin diff --git a/test/config/pr_tests.yml b/test/config/pr_tests.yml index 7cf9f38224..1e8ee8862a 100644 --- a/test/config/pr_tests.yml +++ b/test/config/pr_tests.yml @@ -47,8 +47,8 @@ testCases: config: maxTrialNum: 4 trialConcurrency: 4 - launchCommand: python3 -c 'from nni.nnicli import Experiment; exp = Experiment(); exp.start_experiment("$configFile")' - stopCommand: python3 -c 'from nni.nnicli import Experiment; exp = Experiment(); exp.connect_experiment("http://localhost:8080/"); exp.stop_experiment()' + launchCommand: python3 -c 'from nni.experiment import Experiment; exp = Experiment(); exp.start_experiment("$configFile")' + stopCommand: python3 -c 'from nni.experiment import Experiment; exp = Experiment(); exp.connect_experiment("http://localhost:8080/"); exp.stop_experiment()' validator: class: NnicliValidator platform: linux darwin diff --git a/test/nni_test/nnitest/validators.py b/test/nni_test/nnitest/validators.py index 4b277f2ec1..2df7342eb6 100644 --- a/test/nni_test/nnitest/validators.py +++ b/test/nni_test/nnitest/validators.py @@ -6,8 +6,8 @@ import subprocess import json import requests -from nni.nnicli import Experiment -from nni.nni_cmd.updater import load_search_space +from nni.experiment import Experiment +from nni.tools.nnictl.updater import load_search_space from utils import METRICS_URL, GET_IMPORTED_DATA_URL diff --git a/ts/nni_manager/package.json b/ts/nni_manager/package.json index e93172d81e..aabb58e6d9 100644 --- a/ts/nni_manager/package.json +++ b/ts/nni_manager/package.json @@ -12,7 +12,6 @@ "license": "MIT", "dependencies": { "azure-storage": "^2.10.2", - "chai-as-promised": "^7.1.1", "child-process-promise": "^2.2.1", "express": "^4.16.3", "express-joi-validator": "^2.0.0", @@ -53,6 +52,7 @@ "@typescript-eslint/eslint-plugin": "^2.10.0", "@typescript-eslint/parser": "^2.10.0", "chai": "^4.1.2", + "chai-as-promised": "^7.1.1", "eslint": "^6.7.2", "glob": "^7.1.3", "mocha": "^8.1.3", diff --git a/ts/nni_manager/training_service/common/gpuData.ts b/ts/nni_manager/training_service/common/gpuData.ts index 835569255e..8767baf6e4 100644 --- a/ts/nni_manager/training_service/common/gpuData.ts +++ b/ts/nni_manager/training_service/common/gpuData.ts @@ -79,6 +79,6 @@ export function parseGpuIndices(gpuIndices?: string): Set | undefined { export const GPU_INFO_COLLECTOR_FORMAT_WINDOWS: string = ` $env:METRIC_OUTPUT_DIR="{0}" -$app = Start-Process "python" -ArgumentList "-m nni_gpu_tool.gpu_metrics_collector" -passthru -NoNewWindow +$app = Start-Process "python" -ArgumentList "-m nni.tools.gpu_tool.gpu_metrics_collector" -passthru -NoNewWindow Write $app.ID | Out-File {1} -NoNewline -encoding utf8 `; diff --git a/ts/nni_manager/training_service/common/util.ts b/ts/nni_manager/training_service/common/util.ts index 61bd3cb622..791d7dcebb 100644 --- a/ts/nni_manager/training_service/common/util.ts +++ b/ts/nni_manager/training_service/common/util.ts @@ -236,7 +236,7 @@ export function getScriptName(fileNamePrefix: string): string { } export function getGpuMetricsCollectorBashScriptContent(scriptFolder: string): string { - return `echo $$ > ${scriptFolder}/pid ; METRIC_OUTPUT_DIR=${scriptFolder} python3 -m nni_gpu_tool.gpu_metrics_collector`; + return `echo $$ > ${scriptFolder}/pid ; METRIC_OUTPUT_DIR=${scriptFolder} python3 -m nni.tools.gpu_tool.gpu_metrics_collector`; } export function runGpuMetricsCollector(scriptFolder: string): void { diff --git a/ts/nni_manager/training_service/dlts/dltsData.ts b/ts/nni_manager/training_service/dlts/dltsData.ts index f658414535..3a59c3d8d2 100644 --- a/ts/nni_manager/training_service/dlts/dltsData.ts +++ b/ts/nni_manager/training_service/dlts/dltsData.ts @@ -4,5 +4,5 @@ export const DLTS_TRIAL_COMMAND_FORMAT: string = `export NNI_PLATFORM=dlts NNI_SYS_DIR={0} NNI_OUTPUT_DIR={1} NNI_TRIAL_JOB_ID={2} NNI_EXP_ID={3} NNI_TRIAL_SEQ_ID={4} MULTI_PHASE={5} \ && cd $NNI_SYS_DIR && sh install_nni.sh \ -&& cd '{6}' && python3 -m nni_trial_tool.trial_keeper --trial_command '{7}' \ +&& cd '{6}' && python3 -m nni.tools.trial_tool.trial_keeper --trial_command '{7}' \ --nnimanager_ip '{8}' --nnimanager_port '{9}' --nni_manager_version '{10}' --log_collection '{11}'`; diff --git a/ts/nni_manager/training_service/kubernetes/kubernetesData.ts b/ts/nni_manager/training_service/kubernetes/kubernetesData.ts index 6f0f3f4342..eb96202aed 100644 --- a/ts/nni_manager/training_service/kubernetes/kubernetesData.ts +++ b/ts/nni_manager/training_service/kubernetes/kubernetesData.ts @@ -52,5 +52,5 @@ mkdir -p $NNI_OUTPUT_DIR cp -r $NNI_CODE_DIR/. $NNI_SYS_DIR/code sh $NNI_SYS_DIR/install_nni.sh cd $NNI_SYS_DIR/code -python3 -m nni_trial_tool.trial_keeper --trial_command '{8}' --nnimanager_ip {9} --nnimanager_port {10} \ +python3 -m nni.tools.trial_tool.trial_keeper --trial_command '{8}' --nnimanager_ip {9} --nnimanager_port {10} \ --nni_manager_version '{11}' --log_collection '{12}' 1>$NNI_OUTPUT_DIR/trialkeeper_stdout 2>$NNI_OUTPUT_DIR/trialkeeper_stderr`; diff --git a/ts/nni_manager/training_service/pai/paiK8S/paiK8SData.ts b/ts/nni_manager/training_service/pai/paiK8S/paiK8SData.ts index 2c6b9f3d66..1d4aef9423 100644 --- a/ts/nni_manager/training_service/pai/paiK8S/paiK8SData.ts +++ b/ts/nni_manager/training_service/pai/paiK8S/paiK8SData.ts @@ -16,5 +16,5 @@ fi`; export const PAI_K8S_TRIAL_COMMAND_FORMAT: string = `export NNI_PLATFORM=pai NNI_SYS_DIR={0} NNI_OUTPUT_DIR={1} NNI_TRIAL_JOB_ID={2} NNI_EXP_ID={3} NNI_TRIAL_SEQ_ID={4} MULTI_PHASE={5} \ && NNI_CODE_DIR={6} && mkdir -p $NNI_SYS_DIR/code && cp -r $NNI_CODE_DIR/. $NNI_SYS_DIR/code && sh $NNI_SYS_DIR/install_nni.sh \ -&& cd $NNI_SYS_DIR/code && python3 -m nni_trial_tool.trial_keeper --trial_command '{7}' --nnimanager_ip '{8}' --nnimanager_port '{9}' \ +&& cd $NNI_SYS_DIR/code && python3 -m nni.tools.trial_tool.trial_keeper --trial_command '{7}' --nnimanager_ip '{8}' --nnimanager_port '{9}' \ --nni_manager_version '{10}' --log_collection '{11}' | tee $NNI_OUTPUT_DIR/trial.log`; diff --git a/ts/nni_manager/training_service/remote_machine/extends/linuxCommands.ts b/ts/nni_manager/training_service/remote_machine/extends/linuxCommands.ts index 0dab8a63ea..441cb97987 100644 --- a/ts/nni_manager/training_service/remote_machine/extends/linuxCommands.ts +++ b/ts/nni_manager/training_service/remote_machine/extends/linuxCommands.ts @@ -26,7 +26,7 @@ class LinuxCommands extends OsCommands { cp -r $NNI_CODE_DIR/. $NNI_SYS_DIR/code sh $NNI_SYS_DIR/install_nni.sh cd $NNI_SYS_DIR/code - python3 -m nni_trial_tool.trial_keeper --trial_command '${cudaVisibleSetting} ${command}' --nnimanager_ip '${nniManagerAddress}' \ + python3 -m nni.tools.trial_tool.trial_keeper --trial_command '${cudaVisibleSetting} ${command}' --nnimanager_ip '${nniManagerAddress}' \ --nnimanager_port '${nniManagerPort}' --nni_manager_version '${nniManagerVersion}' \ --job_id_file ${jobIdFileName} \ --log_collection '${logCollection}' 1>$NNI_OUTPUT_DIR/trialkeeper_stdout 2>$NNI_OUTPUT_DIR/trialkeeper_stderr @@ -34,7 +34,7 @@ class LinuxCommands extends OsCommands { } public generateGpuStatsScript(scriptFolder: string): string { - return `echo $$ > ${scriptFolder}/pid ; METRIC_OUTPUT_DIR=${scriptFolder} python3 -m nni_gpu_tool.gpu_metrics_collector`; + return `echo $$ > ${scriptFolder}/pid ; METRIC_OUTPUT_DIR=${scriptFolder} python3 -m nni.tools.gpu_tool.gpu_metrics_collector`; } public createFolder(folderName: string, sharedFolder: boolean = false): string { diff --git a/ts/nni_manager/training_service/remote_machine/extends/windowsCommands.ts b/ts/nni_manager/training_service/remote_machine/extends/windowsCommands.ts index 2a81157e29..28395ca45f 100644 --- a/ts/nni_manager/training_service/remote_machine/extends/windowsCommands.ts +++ b/ts/nni_manager/training_service/remote_machine/extends/windowsCommands.ts @@ -38,7 +38,7 @@ class WindowsCommands extends OsCommands { ) echo starting script - python -m nni_trial_tool.trial_keeper --trial_command "${command}" --nnimanager_ip "${nniManagerAddress}" --nnimanager_port "${nniManagerPort}" --nni_manager_version "${nniManagerVersion}" --log_collection "${logCollection}" --job_id_file ${jobIdFileName} 1>%NNI_OUTPUT_DIR%/trialkeeper_stdout 2>%NNI_OUTPUT_DIR%/trialkeeper_stderr + python -m nni.tools.trial_tool.trial_keeper --trial_command "${command}" --nnimanager_ip "${nniManagerAddress}" --nnimanager_port "${nniManagerPort}" --nni_manager_version "${nniManagerVersion}" --log_collection "${logCollection}" --job_id_file ${jobIdFileName} 1>%NNI_OUTPUT_DIR%/trialkeeper_stdout 2>%NNI_OUTPUT_DIR%/trialkeeper_stderr echo save exit code(%ERRORLEVEL%) and time echo|set /p="%ERRORLEVEL% " > ${exitCodeFile} @@ -46,7 +46,7 @@ class WindowsCommands extends OsCommands { } public generateGpuStatsScript(scriptFolder: string): string { - return `powershell -command $env:Path=If($env:prePath){$env:prePath}Else{$env:Path};$env:METRIC_OUTPUT_DIR='${scriptFolder}';$app = Start-Process -FilePath python -NoNewWindow -passthru -ArgumentList '-m nni_gpu_tool.gpu_metrics_collector' -RedirectStandardOutput ${scriptFolder}\\scriptstdout -RedirectStandardError ${scriptFolder}\\scriptstderr;Write $PID ^| Out-File ${scriptFolder}\\pid -NoNewline -encoding utf8;wait-process $app.ID`; + return `powershell -command $env:Path=If($env:prePath){$env:prePath}Else{$env:Path};$env:METRIC_OUTPUT_DIR='${scriptFolder}';$app = Start-Process -FilePath python -NoNewWindow -passthru -ArgumentList '-m nni.tools.gpu_tool.gpu_metrics_collector' -RedirectStandardOutput ${scriptFolder}\\scriptstdout -RedirectStandardError ${scriptFolder}\\scriptstderr;Write $PID ^| Out-File ${scriptFolder}\\pid -NoNewline -encoding utf8;wait-process $app.ID`; } public createFolder(folderName: string, sharedFolder: boolean = false): string { diff --git a/ts/nni_manager/training_service/reusable/trialDispatcher.ts b/ts/nni_manager/training_service/reusable/trialDispatcher.ts index a56f6d8330..7ae58c6e81 100644 --- a/ts/nni_manager/training_service/reusable/trialDispatcher.ts +++ b/ts/nni_manager/training_service/reusable/trialDispatcher.ts @@ -228,6 +228,7 @@ class TrialDispatcher implements TrainingService { const runnerSettings = storageService.joinPath(envDir, "settings.json"); await storageService.save(JSON.stringify(this.runnerSettings), runnerSettings); + // FIXME: what the hell is this? if (this.isDeveloping) { let trialToolsPath = path.join(__dirname, "../../../../../tools/nni_trial_tool"); if (false === fs.existsSync(trialToolsPath)) { @@ -598,7 +599,7 @@ class TrialDispatcher implements TrainingService { const envName = `nni_exp_${this.experimentId}_env_${envId}`; const environment = environmentService.createEnvironmentInformation(envId, envName); - environment.command = `sh ../install_nni.sh && python3 -m nni_trial_tool.trial_runner`; + environment.command = `sh ../install_nni.sh && python3 -m nni.tools.trial_tool.trial_runner`; if (this.isDeveloping) { environment.command = "[ -d \"nni_trial_tool\" ] && echo \"nni_trial_tool exists already\" || (mkdir ./nni_trial_tool && tar -xof ../nni_trial_tool.tar.gz -C ./nni_trial_tool) && pip3 install websockets && " + environment.command;