Auto genreate notebooks.md (probml#992)

* autogenerate notebooks.md in workflow

* probml_utils dependency removed

* branch name changed

* moved `generate_notebooks_md.py` to `.github/scripts`

* add `notebooks_md` in .gitignore

* removed

internal/generate_notebooks_md.py → .github/scripts/generate_notebooks_md.py

@@ -3,9 +3,59 @@
 import os
 import nbformat
 import numpy as np
-import probml_utils.url_utils as url_utils
 
 root_path = "notebooks"
+curr_path = ""
+
+# where to save notebooks.md
+folder = "notebooks_md"
+if os.path.exists(folder):
+    os.system("rm -rf " + folder)
+os.makedirs(folder)
+
+
+## url utils helper functions START ##
+def github_url_to_colab_url(url):
+    """
+    convert github .ipynb url to colab .ipynb url
+    """
+    if not (url.startswith("https://github.com")):
+        raise ValueError("INVALID URL: not a Github url")
+
+    if not (url.endswith(".ipynb")):
+        raise ValueError("INVALID URL: not a .ipynb file")
+
+    base_url_colab = "https://colab.research.google.com/github/"
+    base_url_github = "https://github.com/"
+
+    return url.replace(base_url_github, base_url_colab)
+
+
+def make_url_from_chapter_no_and_script_name(
+    chapter_no,
+    script_name,
+    base_url="https://github.com/probml/pyprobml/blob/master/notebooks",
+    book_no=1,
+    convert_to_which_url="github",
+):
+    """
+    create mapping between chapter_no and actual_url path
+    (chapter_no = 3,script_name=iris_plot.ipynb) converted to https://github.com/probml/pyprobml/blob/master/notebooks/book1/01/iris_plot.ipynb
+    convert_to_which_url = Union["github","colab","gihub-raw"]
+    """
+    base_url_ipynb = os.path.join(base_url, f"book{book_no}/{int(chapter_no):02d}")
+    if script_name.strip().endswith(".py"):
+        script_name = script_name[:-3] + ".ipynb"
+    github_url = os.path.join(base_url_ipynb, script_name)
+
+    if convert_to_which_url == "colab":
+        return github_url_to_colab_url(github_url)
+    elif convert_to_which_url == "github-raw":
+        return github_to_rawcontent_url(github_url)
+    return github_url
+
+
+## url utils helper functions END ##
 
 
 def get_notebook_path(book_str, chap_no, nb_name):
@@ -82,7 +132,7 @@ def is_github_notebook(url):
 
 def to_colab_md_url(github_url):
     if is_github_notebook(github_url):
-        colab_url = url_utils.github_url_to_colab_url(github_url)
+        colab_url = github_url_to_colab_url(github_url)
         return to_md_url("colab", colab_url)
     else:
         return "NA"
@@ -108,7 +158,7 @@ def to_colab_md_url(github_url):
 # Add github url
 df_pyprobml["type"] = "github"
 df_pyprobml["github_url"] = df_pyprobml.apply(
-    lambda x: url_utils.make_url_from_chapter_no_and_script_name(
+    lambda x: make_url_from_chapter_no_and_script_name(
         chapter_no=int(x["chap_no"]),
         script_name=x["Notebook"],
         book_no=int(x["book_no"][-1]),
@@ -131,7 +181,7 @@ def to_colab_md_url(github_url):
 
 
 # handle external notebooks
-df_external = pd.read_csv("internal/external_links.csv")
+df_external = pd.read_csv(curr_path + "external_links.csv")
 common_notebooks = np.intersect1d(df_pyprobml["Notebook"].values, df_external["Notebook"].values)
 
 # drop common notebooks from df_pyprobml and replace with df_external's notebooks
@@ -156,4 +206,4 @@ def to_colab_md_url(github_url):
 
 # save to .md file
 df_all = df_all[["Notebook", "github_url", "colab_url"]]
-df_all.to_markdown("notebooks.md", index=False)
+df_all.to_markdown(os.path.join(folder, "notebooks.md"), index=False)

.github/workflows/notebooks.yml

@@ -214,3 +214,25 @@ jobs:
         github_token: ${{ secrets.GITHUB_TOKEN }}
         publish_dir: workflow_testing_indicator/
         publish_branch: workflow_testing_indicator
+
+  update_notebooks_md:
+    runs-on: ubuntu-latest
+    if: always() && github.event_name == 'push'  # Don't run on pull requests
+    steps:
+    - name: Clone the reference repository
+      uses: actions/checkout@v2
+
+    - name: Install dependencies
+      run: |
+        pip install numpy pandas nbformat tabulate 
+
+    - name: Update notebooks.md
+      run: |
+        python internal/notebooks_md/generate_notebooks_md.py
+
+    - name: Push notebooks.md to notebooks_md (does not run on pull requests)
+      uses: peaceiris/[email protected]
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        publish_dir: notebooks_md/
+        publish_branch: auto_notebooks_md

.gitignore

@@ -109,3 +109,4 @@ GitHub.sublime-settings
 # GitHub Actions
 workflow_testing_indicator*
 auto_generated_figures*
+notebooks_md*