[FEATURE][databrickslabs#141] add fallback to conf/deployment.yml

scholer · Jan 27, 2022 · ad228f4 · ad228f4
1 parent bcab412
commit ad228f4
Show file tree

Hide file tree

Showing 7 changed files with 24 additions and 12 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -11,6 +11,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ----
 
 ## [Unreleased] - YYYY-MM-DD
+
+### Added
+- Recognition of `conf/deployment.yml` file from conf directory as a default parameter
+- Remove unnecessary references of `conf/deployment.yml` in CI pipelines
+
 ### Changed
 
 - Upgraded minimal `mlflow` version to 1.23

diff --git a/dbx/commands/deploy.py b/dbx/commands/deploy.py
@@ -131,7 +131,7 @@ def deploy(
     if not branch_name:
         branch_name = get_current_branch_name()
 
-    _verify_deployment_file(deployment_file)
+    deployment_file = _finalize_deployment_file_path(deployment_file)
 
     deployment_file_config = get_deployment_config(deployment_file)
     deployment = deployment_file_config.get_environment(environment)
@@ -258,14 +258,23 @@ def _log_dbx_file(content: Dict[Any, Any], name: str):
     shutil.rmtree(temp_dir)
 
 
-def _verify_deployment_file(deployment_file: str):
+def _finalize_deployment_file_path(deployment_file: str) -> str:
     file_extension = deployment_file.split(".").pop()
+
     if file_extension not in ["json", "yaml", "yml"]:
         raise Exception('Deployment file should have one of these extensions: [".json", ".yaml", ".yml"]')
 
+    if deployment_file == DEFAULT_DEPLOYMENT_FILE_PATH and not pathlib.Path(deployment_file).exists():
+        fallback_file_path = "conf/deployment.yml"
+        if pathlib.Path(fallback_file_path).exists():
+            dbx_echo("Found conf/deployment.yml file, using it as a deployment file")
+            return fallback_file_path
+
     if not pathlib.Path(deployment_file).exists():
         raise Exception(f"Deployment file ({deployment_file}) does not exist")
 
+    return deployment_file
+
 
 def _preprocess_deployment(deployment: Dict[str, Any], requested_jobs: Union[List[str], None]):
     if "jobs" not in deployment:

diff --git a/dbx/templates/projects/python_basic/components/.github/workflows/onpush.yml b/dbx/templates/projects/python_basic/components/.github/workflows/onpush.yml
@@ -42,11 +42,11 @@ jobs:
 
       - name: Deploy integration test
         run: |
-          dbx deploy --deployment-file conf/deployment.yml --jobs={{project_name}}-sample-integration-test --files-only
+          dbx deploy --jobs={{project_name}}-sample-integration-test --files-only
 
       - name: Run integration test
         run: |
-          dbx launch --deployment-file conf/deployment.yml --job={{project_name}}-sample-integration-test --as-run-submit --trace
+          dbx launch --job={{project_name}}-sample-integration-test --as-run-submit --trace
 
 
 
diff --git a/dbx/templates/projects/python_basic/components/.github/workflows/onrelease.yml b/dbx/templates/projects/python_basic/components/.github/workflows/onrelease.yml
@@ -37,7 +37,7 @@ jobs:
 
       - name: Deploy the job
         run: |
-          dbx deploy --deployment-file conf/deployment.yml --jobs={{project_name}}-sample
+          dbx deploy --jobs={{project_name}}-sample
 
       - name: Create Release
         id: create_release

diff --git a/dbx/templates/projects/python_basic/components/.gitlab-ci.yml b/dbx/templates/projects/python_basic/components/.gitlab-ci.yml
@@ -23,7 +23,7 @@ integration-testing:
     - pip install -r unit-requirements.txt
     - pip install -e .
     - echo "Deploying tests"
-    - dbx deploy --deployment-file conf/deployment.yml --jobs={{project_name}}-integration-test --files-only
+    - dbx deploy --jobs={{project_name}}-integration-test --files-only
     - echo "Running tests"
     - dbx launch --job={{project_name}}-integration-test --as-run-submit --trace
 
@@ -38,4 +38,4 @@ release:
     - pip install -r unit-requirements.txt
     - pip install -e .
     - echo "Deploying Job"
-    - dbx deploy --deployment-file conf/deployment.yml --jobs={{project_name}}-sample
+    - dbx deploy --jobs={{project_name}}-sample
diff --git a/dbx/templates/projects/python_basic/components/azure-pipelines.yml b/dbx/templates/projects/python_basic/components/azure-pipelines.yml
@@ -44,7 +44,7 @@ stages:
       displayName: 'Run Unit tests'
 
     - script: |
-        dbx deploy --deployment-file conf/deployment.yml --jobs={{project_name}}-sample-integration-test --files-only
+        dbx deploy --jobs={{project_name}}-sample-integration-test --files-only
       displayName: 'Deploy integration test'
 
     - script: |
@@ -94,7 +94,7 @@ stages:
         displayName: 'Run Unit tests'
 
       - script: |
-          dbx deploy --deployment-file conf/deployment.yml --jobs={{project_name}}-sample
+          dbx deploy --jobs={{project_name}}-sample
         displayName: 'Deploy the job'
 
       - task: PublishTestResults@2

diff --git a/tests/unit/test_deploy.py b/tests/unit/test_deploy.py
@@ -282,8 +282,6 @@ def test_deploy_path_adjustment_yaml(self, *_):
                     [
                         "--environment",
                         "default",
-                        "--deployment-file",
-                        "conf/deployment.yml",
                         "--write-specs-to-file",
                         ".dbx/deployment-result.json",
                     ],
@@ -304,7 +302,7 @@ def test_deploy_path_adjustment_yaml(self, *_):
                         "/dbfs/Shared/dbx-testing"
                     )
                 )
-                # self.assertEqual(deploy_result.exit_code, 0)
+                self.assertEqual(deploy_result.exit_code, 0)
 
     @patch("databricks_cli.sdk.service.DbfsService.get_status", return_value=None)
     @patch(