Add parameters -p support

prmohanty · Feb 3, 2019 · 14a3f17 · 14a3f17
1 parent ae0073a
commit 14a3f17
Showing 1 changed file with 26 additions and 25 deletions.
diff --git a/dlvm/tools/scripts/notebook_executor.sh b/dlvm/tools/scripts/notebook_executor.sh
@@ -55,36 +55,37 @@ function validate_metadata() {
 
 function run_notebook() {
 
-    # Papermill output information.
-    NOTEBOOKS_FOLDER="/tmp"
-    OUTPUT_NOTEBOOK_NAME="notebook.ipynb"
-    OUTPUT_NOTEBOOK_PATH="${NOTEBOOKS_FOLDER}/${OUTPUT_NOTEBOOK_NAME}"
-
-    # Add metadata attributes.
-    INPUT_NOTEBOOK_GCS_FILE=$(curl http://metadata.google.internal/computeMetadata/v1/instance/attributes/input_notebook -H "Metadata-Flavor: Google")
-    OUTPUT_NOTEBOOK_GCS_FOLDER=$(curl http://metadata.google.internal/computeMetadata/v1/instance/attributes/output_notebook -H "Metadata-Flavor: Google")
-
-    # Copy GCS input notebook to temporary folder and extract location.
-    gsutil cp "${INPUT_NOTEBOOK_GCS_FILE}" "${NOTEBOOKS_FOLDER}/"
-    INPUT_NOTEBOOK_PATH=`find ${NOTEBOOKS_FOLDER}/ | grep ipynb`
-
-    # Run Notebook using Papermill. https://github.com/nteract/papermill. Check if parameters option exists.
-    metadata_exists parameters_file
-    exit_status=$?
-    if [[ ${exit_status} -eq 1 ]]; then
-        echo "Parameters file does not exist as parameter. Running notebook now..."
-        papermill "${INPUT_NOTEBOOK_PATH}" "${OUTPUT_NOTEBOOK_PATH}" --log-output
+  # Add metadata attributes.
+  INPUT_NOTEBOOK_PATH=$(curl http://metadata.google.internal/computeMetadata/v1/instance/attributes/input_notebook -H "Metadata-Flavor: Google")
+  OUTPUT_NOTEBOOK_PATH=$(curl http://metadata.google.internal/computeMetadata/v1/instance/attributes/output_notebook -H "Metadata-Flavor: Google")
+
+  # Run Notebook using Papermill. https://github.com/nteract/papermill. Check if parameters option exists.
+  metadata_exists parameters_file
+  parameters_file_exists=$?
+  if [[ ${parameters_file_exists} -eq 1 ]]; then
+    metadata_exists parameters
+    parameters_exists=$?
+    if [[ ${parameters_exists} -eq 1 ]]; then
+      # No parameters
+      echo "Running notebook now..."
+      papermill "${INPUT_NOTEBOOK_PATH}" "${OUTPUT_NOTEBOOK_PATH}" --log-output
     else
-        PARAMETERS_GCS_FILE=$(curl http://metadata.google.internal/computeMetadata/v1/instance/attributes/parameters_file -H "Metadata-Flavor: Google")
-        gsutil cp "${PARAMETERS_GCS_FILE}" params.yaml
-        papermill "${INPUT_NOTEBOOK_PATH}" "${OUTPUT_NOTEBOOK_PATH}" -f params.yaml --log-output
+      # Parameters as -p key value
+      echo "Manual parameters defined, running notebook now..."
+      PARAMETERS=$(curl http://metadata.google.internal/computeMetadata/v1/instance/attributes/parameters -H "Metadata-Flavor: Google")
+      papermill "${INPUT_NOTEBOOK_PATH}" "${OUTPUT_NOTEBOOK_PATH}" "${PARAMETERS}" --log-output
     fi
-
-    # Copy result to output folder.
-    gsutil cp "${OUTPUT_NOTEBOOK_PATH}" "${OUTPUT_NOTEBOOK_GCS_FOLDER}"
+  else
+    # Passing parameters file
+    echo "Parameters file exists, running notebook now..."
+    PARAMETERS_FILE=$(curl http://metadata.google.internal/computeMetadata/v1/instance/attributes/parameters_file -H "Metadata-Flavor: Google")
+    gsutil cp "${PARAMETERS_FILE}" params.yaml
+    papermill "${INPUT_NOTEBOOK_PATH}" "${OUTPUT_NOTEBOOK_PATH}" -f params.yaml --log-output
+  fi
 }
 
 
+
 function delete_instance(){
     # Delete Virtual Machine.
     INSTANCE_NAME=$(curl http://metadata.google.internal/computeMetadata/v1/instance/name -H "Metadata-Flavor: Google")