feat: Make repo mybinder.org compatible (microsoft#1487)

* feat: Add binder configuration files

* fix: Fix references to new environment filename extension (yaml->yml)

* trim developer dockerfile

Co-authored-by: Puneet Pruthi <[email protected]>

.github/workflows/ci-publish-artifacts.yml

@@ -37,7 +37,7 @@ jobs:
       uses: conda-incubator/[email protected]
       with:
         python-version: 3.8.8
-        environment-file: environment.yaml
+        environment-file: environment.yml
         activate-environment: synapseml
     - name: Publish Artifact
       shell: bash -l {0}

.github/workflows/ci-publish-websiteautodeployment.yml

@@ -46,7 +46,7 @@ jobs:
       uses: conda-incubator/[email protected]
       with:
         python-version: 3.8.8
-        environment-file: environment.yaml
+        environment-file: environment.yml
         activate-environment: synapseml
     - name: Convert notebooks to markdowns
       shell: bash -l {0}

.github/workflows/ci-release-sonatype.yml

@@ -66,7 +66,7 @@ jobs:
       uses: conda-incubator/[email protected]
       with:
         python-version: 3.8.8
-        environment-file: environment.yaml
+        environment-file: environment.yml
         activate-environment: synapseml
     - name: publish python package to pypi
       if: startsWith(steps.getGitTag.outputs.gittag, 'v')
@@ -95,4 +95,4 @@ jobs:
         NEXUS-PW: ${{ steps.GetKeyVaultSecrets.outputs.nexus-pw }}
         PGP-PRIVATE: ${{ steps.GetKeyVaultSecrets.outputs.pgp-private }}
         PGP-PUBLIC: ${{ steps.GetKeyVaultSecrets.outputs.pgp-public }}
-        PGP-PW: ${{ steps.GetKeyVaultSecrets.outputs.pgp-pw }}
+        PGP-PW: ${{ steps.GetKeyVaultSecrets.outputs.pgp-pw }}

.github/workflows/ci-test-synapsee2e.yml

@@ -36,7 +36,7 @@ jobs:
       uses: conda-incubator/[email protected]
       with:
         python-version: 3.8.8
-        environment-file: environment.yaml
+        environment-file: environment.yml
         activate-environment: synapseml
     - name: Publish Blob Artifacts
       shell: bash -l {0}
@@ -63,4 +63,4 @@ jobs:
         files: '**/test-reports/TEST-*.xml'
         check_name: "SynapseE2E Test Results"
         comment_title: "SynapseE2E Test Results"
-
+

.github/workflows/ci-tests-databrickse2e.yml

@@ -38,7 +38,7 @@ jobs:
       uses: conda-incubator/[email protected]
       with:
         python-version: 3.8.8
-        environment-file: environment.yaml
+        environment-file: environment.yml
         activate-environment: synapseml
     - name: Publish Blob Artifacts
       shell: bash -l {0}
@@ -63,4 +63,4 @@ jobs:
       with:
         files: '**/test-reports/TEST-*.xml'
         check_name: "DatabricksE2E Test Results"
-        comment_title: "DatabricksE2E Test Results"
+        comment_title: "DatabricksE2E Test Results"

.github/workflows/ci-tests-python.yml

@@ -41,7 +41,7 @@ jobs:
       uses: conda-incubator/[email protected]
       with:
         python-version: 3.8.8
-        environment-file: environment.yaml
+        environment-file: environment.yml
         activate-environment: synapseml
     - name: Install Pip Package
       shell: bash -l {0}

.github/workflows/ci-tests-r.yml

@@ -37,7 +37,7 @@ jobs:
       uses: conda-incubator/[email protected]
       with:
         python-version: 3.8.8
-        environment-file: environment.yaml
+        environment-file: environment.yml
         activate-environment: synapseml
     - name: Download Spark
       shell: bash -l {0}
@@ -72,4 +72,4 @@ jobs:
         curl -s https://codecov.io/bash > .codecov
         chmod +x .codecov
         echo "Starting Codecov Upload"
-        ./.codecov -t ${{ steps.GetKeyVaultSecrets.outputs.codecov-token }}
+        ./.codecov -t ${{ steps.GetKeyVaultSecrets.outputs.codecov-token }}

.github/workflows/ci-tests-websitesamples.yml

@@ -37,7 +37,7 @@ jobs:
       uses: conda-incubator/[email protected]
       with:
         python-version: 3.8.8
-        environment-file: environment.yaml
+        environment-file: environment.yml
         activate-environment: synapseml
     - name: Test Website Samples
       shell: bash -l {0}
@@ -71,4 +71,4 @@ jobs:
         curl -s https://codecov.io/bash > .codecov
         chmod +x .codecov
         echo "Starting Codecov Upload"
-        ./.codecov -t ${{ steps.GetKeyVaultSecrets.outputs.codecov-token }}
+        ./.codecov -t ${{ steps.GetKeyVaultSecrets.outputs.codecov-token }}

apt.txt

@@ -0,0 +1,3 @@
+default-jre
+default-jdk
+openmpi-bin

pipeline.yaml

@@ -203,23 +203,6 @@ jobs:
         repository: 'public/mmlspark/build-minimal'
         command: 'push'
         tags: $(version)
-    - task: Docker@2
-      displayName: Developer Image Build
-      inputs:
-        containerRegistry: 'mmlspark-mcr-connection-1'
-        repository: 'public/mmlspark/build-developer'
-        command: 'build'
-        buildContext: "."
-        Dockerfile: 'tools/docker/developer/Dockerfile'
-        tags: $(version)
-        arguments: --build-arg MMLSPARK_VERSION=$(version)
-    - task: Docker@2
-      displayName: Developer Image Push
-      inputs:
-        containerRegistry: 'mmlspark-mcr-connection-1'
-        repository: 'public/mmlspark/build-developer'
-        command: 'push'
-        tags: $(version)
     - task: Docker@2
       condition: startsWith(variables['gittag'], 'v')
       displayName: Release Image Build
@@ -280,14 +263,14 @@ jobs:
       displayName: Use cached Anaconda environment
       condition: startsWith(variables['tag'], 'v')
       inputs:
-        key: 'conda | "$(Agent.OS)" | environment.yaml'
+        key: 'conda | "$(Agent.OS)" | environment.yml'
         restoreKeys: |
           python | "$(Agent.OS)"
           python
         path: $(CONDA_CACHE_DIR)
         cacheHitVar: CONDA_CACHE_RESTORED
     - bash: |
-        conda env create -f environment.yaml -v -v -v
+        conda env create -f environment.yml -v -v -v
       condition: and(startsWith(variables['tag'], 'v'), eq(variables.CONDA_CACHE_RESTORED, 'false'))
       displayName: Create Anaconda environment
     - task: AzureKeyVault@1

postBuild

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+mkdir -p $HOME/.ipython/profile_default/startup
+cp tools/docker/demo/init_notebook.py $HOME/.ipython/profile_default/startup/init_notebook.py

project/CondaPlugin.scala

@@ -31,7 +31,7 @@ object CondaPlugin extends AutoPlugin {
       val hasEnv = Process("conda env list").lineStream.toList
         .map(_.split("\\s+").head).contains(condaEnvName)
       if (!hasEnv) {
-        runCmd(Seq("conda", "env", "create", "-f", "environment.yaml"))
+        runCmd(Seq("conda", "env", "create", "-f", "environment.yml"))
       } else {
         println("Found conda env " + condaEnvName)
       }
@@ -41,4 +41,4 @@ object CondaPlugin extends AutoPlugin {
   override def requires: Plugins = sbt.Plugins.empty
 
   override lazy val projectSettings: Seq[Setting[_]] = Seq()
-}
+}

start

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+export OPENMPI_VERSION="3.1.2"
+export SPARK_VERSION="3.2.1"
+export HADOOP_VERSION="2.7"
+export MMLSPARK_VERSION="0.9.5"
+
+echo "Beginning Spark Session..."
+exec "$@"

templates/conda.yml

@@ -6,14 +6,14 @@ steps:
   - task: Cache@2
     displayName: Use cached Anaconda environment
     inputs:
-      key: 'conda | "$(Agent.OS)" | environment.yaml'
+      key: 'conda | "$(Agent.OS)" | environment.yml'
       restoreKeys: |
         python | "$(Agent.OS)"
         python
       path: $(CONDA_CACHE_DIR)
       cacheHitVar: CONDA_CACHE_RESTORED
   - bash: |
-      conda env create -f environment.yaml  -v -v -v
+      conda env create -f environment.yml  -v -v -v
     displayName: Create Anaconda environment
     condition: eq(variables.CONDA_CACHE_RESTORED, 'false')
 

website/docs/reference/developer-readme.md

@@ -29,7 +29,7 @@ description: SynapseML Development Setup
 9) Prepare your Python Environment
     - Install [Miniconda](https://docs.conda.io/en/latest/miniconda.html)
     - Note: if you want to run conda commands from IntelliJ, you may need to select the option to add conda to PATH during installation.
-    - Activate the `synapseml` conda environment by running `conda env create -f environment.yaml` from the `synapseml` directory.
+    - Activate the `synapseml` conda environment by running `conda env create -f environment.yml` from the `synapseml` directory.
 
 > NOTE
 > 
@@ -67,7 +67,7 @@ Generates documentation for scala sources
 
 ### `createCondaEnv`
 
-Creates a conda environment `synapseml` from `environment.yaml` if it does not already exist. 
+Creates a conda environment `synapseml` from `environment.yml` if it does not already exist. 
 This env is used for python testing. **Activate this env before using python build commands.**
 
 ### `cleanCondaEnv`