From ad228f4b919d4b284b6dddd52800959fd5f17e97 Mon Sep 17 00:00:00 2001 From: Ivan Trusov Date: Thu, 27 Jan 2022 22:21:22 +0100 Subject: [PATCH] [FEATURE][#141] add fallback to conf/deployment.yml [FEATURE][#141] add fallback to conf/deployment.yml --- CHANGELOG.md | 5 +++++ dbx/commands/deploy.py | 13 +++++++++++-- .../components/.github/workflows/onpush.yml | 4 ++-- .../components/.github/workflows/onrelease.yml | 2 +- .../projects/python_basic/components/.gitlab-ci.yml | 4 ++-- .../python_basic/components/azure-pipelines.yml | 4 ++-- tests/unit/test_deploy.py | 4 +--- 7 files changed, 24 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a2d0ba9..c33fbf6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ---- ## [Unreleased] - YYYY-MM-DD + +### Added +- Recognition of `conf/deployment.yml` file from conf directory as a default parameter +- Remove unnecessary references of `conf/deployment.yml` in CI pipelines + ### Changed - Upgraded minimal `mlflow` version to 1.23 diff --git a/dbx/commands/deploy.py b/dbx/commands/deploy.py index 76c6d968..0eeaef6c 100644 --- a/dbx/commands/deploy.py +++ b/dbx/commands/deploy.py @@ -131,7 +131,7 @@ def deploy( if not branch_name: branch_name = get_current_branch_name() - _verify_deployment_file(deployment_file) + deployment_file = _finalize_deployment_file_path(deployment_file) deployment_file_config = get_deployment_config(deployment_file) deployment = deployment_file_config.get_environment(environment) @@ -258,14 +258,23 @@ def _log_dbx_file(content: Dict[Any, Any], name: str): shutil.rmtree(temp_dir) -def _verify_deployment_file(deployment_file: str): +def _finalize_deployment_file_path(deployment_file: str) -> str: file_extension = deployment_file.split(".").pop() + if file_extension not in ["json", "yaml", "yml"]: raise Exception('Deployment file should have one of these extensions: [".json", ".yaml", ".yml"]') + if deployment_file == DEFAULT_DEPLOYMENT_FILE_PATH and not pathlib.Path(deployment_file).exists(): + fallback_file_path = "conf/deployment.yml" + if pathlib.Path(fallback_file_path).exists(): + dbx_echo("Found conf/deployment.yml file, using it as a deployment file") + return fallback_file_path + if not pathlib.Path(deployment_file).exists(): raise Exception(f"Deployment file ({deployment_file}) does not exist") + return deployment_file + def _preprocess_deployment(deployment: Dict[str, Any], requested_jobs: Union[List[str], None]): if "jobs" not in deployment: diff --git a/dbx/templates/projects/python_basic/components/.github/workflows/onpush.yml b/dbx/templates/projects/python_basic/components/.github/workflows/onpush.yml index 133d3adc..3a5316ae 100644 --- a/dbx/templates/projects/python_basic/components/.github/workflows/onpush.yml +++ b/dbx/templates/projects/python_basic/components/.github/workflows/onpush.yml @@ -42,11 +42,11 @@ jobs: - name: Deploy integration test run: | - dbx deploy --deployment-file conf/deployment.yml --jobs={{project_name}}-sample-integration-test --files-only + dbx deploy --jobs={{project_name}}-sample-integration-test --files-only - name: Run integration test run: | - dbx launch --deployment-file conf/deployment.yml --job={{project_name}}-sample-integration-test --as-run-submit --trace + dbx launch --job={{project_name}}-sample-integration-test --as-run-submit --trace diff --git a/dbx/templates/projects/python_basic/components/.github/workflows/onrelease.yml b/dbx/templates/projects/python_basic/components/.github/workflows/onrelease.yml index 9e0c5d81..88ba5437 100644 --- a/dbx/templates/projects/python_basic/components/.github/workflows/onrelease.yml +++ b/dbx/templates/projects/python_basic/components/.github/workflows/onrelease.yml @@ -37,7 +37,7 @@ jobs: - name: Deploy the job run: | - dbx deploy --deployment-file conf/deployment.yml --jobs={{project_name}}-sample + dbx deploy --jobs={{project_name}}-sample - name: Create Release id: create_release diff --git a/dbx/templates/projects/python_basic/components/.gitlab-ci.yml b/dbx/templates/projects/python_basic/components/.gitlab-ci.yml index 9a28b0dd..612a6c1a 100644 --- a/dbx/templates/projects/python_basic/components/.gitlab-ci.yml +++ b/dbx/templates/projects/python_basic/components/.gitlab-ci.yml @@ -23,7 +23,7 @@ integration-testing: - pip install -r unit-requirements.txt - pip install -e . - echo "Deploying tests" - - dbx deploy --deployment-file conf/deployment.yml --jobs={{project_name}}-integration-test --files-only + - dbx deploy --jobs={{project_name}}-integration-test --files-only - echo "Running tests" - dbx launch --job={{project_name}}-integration-test --as-run-submit --trace @@ -38,4 +38,4 @@ release: - pip install -r unit-requirements.txt - pip install -e . - echo "Deploying Job" - - dbx deploy --deployment-file conf/deployment.yml --jobs={{project_name}}-sample + - dbx deploy --jobs={{project_name}}-sample diff --git a/dbx/templates/projects/python_basic/components/azure-pipelines.yml b/dbx/templates/projects/python_basic/components/azure-pipelines.yml index 5e29f593..ae8709ac 100644 --- a/dbx/templates/projects/python_basic/components/azure-pipelines.yml +++ b/dbx/templates/projects/python_basic/components/azure-pipelines.yml @@ -44,7 +44,7 @@ stages: displayName: 'Run Unit tests' - script: | - dbx deploy --deployment-file conf/deployment.yml --jobs={{project_name}}-sample-integration-test --files-only + dbx deploy --jobs={{project_name}}-sample-integration-test --files-only displayName: 'Deploy integration test' - script: | @@ -94,7 +94,7 @@ stages: displayName: 'Run Unit tests' - script: | - dbx deploy --deployment-file conf/deployment.yml --jobs={{project_name}}-sample + dbx deploy --jobs={{project_name}}-sample displayName: 'Deploy the job' - task: PublishTestResults@2 diff --git a/tests/unit/test_deploy.py b/tests/unit/test_deploy.py index 976ed18c..d9beb03e 100644 --- a/tests/unit/test_deploy.py +++ b/tests/unit/test_deploy.py @@ -282,8 +282,6 @@ def test_deploy_path_adjustment_yaml(self, *_): [ "--environment", "default", - "--deployment-file", - "conf/deployment.yml", "--write-specs-to-file", ".dbx/deployment-result.json", ], @@ -304,7 +302,7 @@ def test_deploy_path_adjustment_yaml(self, *_): "/dbfs/Shared/dbx-testing" ) ) - # self.assertEqual(deploy_result.exit_code, 0) + self.assertEqual(deploy_result.exit_code, 0) @patch("databricks_cli.sdk.service.DbfsService.get_status", return_value=None) @patch(