diff --git a/dlvm/tools/scripts/notebook_executor.sh b/dlvm/tools/scripts/notebook_executor.sh index 4fa1672c..d3a91518 100644 --- a/dlvm/tools/scripts/notebook_executor.sh +++ b/dlvm/tools/scripts/notebook_executor.sh @@ -55,36 +55,37 @@ function validate_metadata() { function run_notebook() { - # Papermill output information. - NOTEBOOKS_FOLDER="/tmp" - OUTPUT_NOTEBOOK_NAME="notebook.ipynb" - OUTPUT_NOTEBOOK_PATH="${NOTEBOOKS_FOLDER}/${OUTPUT_NOTEBOOK_NAME}" - - # Add metadata attributes. - INPUT_NOTEBOOK_GCS_FILE=$(curl http://metadata.google.internal/computeMetadata/v1/instance/attributes/input_notebook -H "Metadata-Flavor: Google") - OUTPUT_NOTEBOOK_GCS_FOLDER=$(curl http://metadata.google.internal/computeMetadata/v1/instance/attributes/output_notebook -H "Metadata-Flavor: Google") - - # Copy GCS input notebook to temporary folder and extract location. - gsutil cp "${INPUT_NOTEBOOK_GCS_FILE}" "${NOTEBOOKS_FOLDER}/" - INPUT_NOTEBOOK_PATH=`find ${NOTEBOOKS_FOLDER}/ | grep ipynb` - - # Run Notebook using Papermill. https://github.com/nteract/papermill. Check if parameters option exists. - metadata_exists parameters_file - exit_status=$? - if [[ ${exit_status} -eq 1 ]]; then - echo "Parameters file does not exist as parameter. Running notebook now..." - papermill "${INPUT_NOTEBOOK_PATH}" "${OUTPUT_NOTEBOOK_PATH}" --log-output + # Add metadata attributes. + INPUT_NOTEBOOK_PATH=$(curl http://metadata.google.internal/computeMetadata/v1/instance/attributes/input_notebook -H "Metadata-Flavor: Google") + OUTPUT_NOTEBOOK_PATH=$(curl http://metadata.google.internal/computeMetadata/v1/instance/attributes/output_notebook -H "Metadata-Flavor: Google") + + # Run Notebook using Papermill. https://github.com/nteract/papermill. Check if parameters option exists. + metadata_exists parameters_file + parameters_file_exists=$? + if [[ ${parameters_file_exists} -eq 1 ]]; then + metadata_exists parameters + parameters_exists=$? + if [[ ${parameters_exists} -eq 1 ]]; then + # No parameters + echo "Running notebook now..." + papermill "${INPUT_NOTEBOOK_PATH}" "${OUTPUT_NOTEBOOK_PATH}" --log-output else - PARAMETERS_GCS_FILE=$(curl http://metadata.google.internal/computeMetadata/v1/instance/attributes/parameters_file -H "Metadata-Flavor: Google") - gsutil cp "${PARAMETERS_GCS_FILE}" params.yaml - papermill "${INPUT_NOTEBOOK_PATH}" "${OUTPUT_NOTEBOOK_PATH}" -f params.yaml --log-output + # Parameters as -p key value + echo "Manual parameters defined, running notebook now..." + PARAMETERS=$(curl http://metadata.google.internal/computeMetadata/v1/instance/attributes/parameters -H "Metadata-Flavor: Google") + papermill "${INPUT_NOTEBOOK_PATH}" "${OUTPUT_NOTEBOOK_PATH}" "${PARAMETERS}" --log-output fi - - # Copy result to output folder. - gsutil cp "${OUTPUT_NOTEBOOK_PATH}" "${OUTPUT_NOTEBOOK_GCS_FOLDER}" + else + # Passing parameters file + echo "Parameters file exists, running notebook now..." + PARAMETERS_FILE=$(curl http://metadata.google.internal/computeMetadata/v1/instance/attributes/parameters_file -H "Metadata-Flavor: Google") + gsutil cp "${PARAMETERS_FILE}" params.yaml + papermill "${INPUT_NOTEBOOK_PATH}" "${OUTPUT_NOTEBOOK_PATH}" -f params.yaml --log-output + fi } + function delete_instance(){ # Delete Virtual Machine. INSTANCE_NAME=$(curl http://metadata.google.internal/computeMetadata/v1/instance/name -H "Metadata-Flavor: Google")