multicuda changes as well as multiversion changes (ivy-llc#13061)

* multicuda changes as well as multiversion changes

* multiversion framework directory changes

---------

Co-authored-by: Rishabh Kumar <[email protected]>

docker/DockerfileGPUMultiCuda

@@ -0,0 +1,46 @@
+FROM ubuntu:20.04
+WORKDIR /ivy
+ARG cuda=["11.2"]
+
+COPY ../docker/multicuda_framework_directory.py .
+COPY ../docker/multicuda_requirements.txt .
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TZ=Europe/Moscow
+RUN \
+    apt-get -y update && \
+    apt-get install -y gnupg  \
+    curl  \
+    wget  \
+    software-properties-common  \
+    gcc  \
+    nano \
+    pip && \
+    distribution=$(. /etc/os-release;echo $ID$VERSION_ID) \
+      && curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey |  gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \
+      && curl -s -L https://nvidia.github.io/libnvidia-container/$distribution/libnvidia-container.list | \
+            sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
+             tee /etc/apt/sources.list.d/nvidia-container-toolkit.list  &&\
+
+    apt-get update && apt-get  install -y nvidia-container-toolkit
+
+# don't install requirements immediately
+#    pip3 install --upgrade pip && \
+#    pip3 install --no-cache-dir   -r multicuda_requirements.txt &&\
+#    pip3 uninstall -y torch &&\
+#    pip3 uninstall -y tensorflow &&\
+#    pip3 uninstall -y numpy &&\
+#    pip3 uninstall -y jax
+
+## below code is for installing cuda directly through commandline ##
+#RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb &&\
+#    dpkg -i cuda-keyring_1.0-1_all.deb
+#
+#RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin && \
+#    mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600 &&\
+#    apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub &&\
+#    add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /"
+#
+#RUN apt-get update && \
+#    for i in $cuda; do apt-get install -y cuda-$i; done
+

docker/build_DockerfileGPUMultiCuda.sh

@@ -0,0 +1 @@
+docker build --progress=plain --no-cache -t unifyai/multicuda:base   -f DockerfileGPUMultiCuda ..

docker/multicuda_framework_directory.py

@@ -0,0 +1,48 @@
+# flake8: noqa
+import os
+import subprocess
+import sys
+
+
+def directory_generator(req, base="/fw/"):
+    for versions in req:
+        pkg, ver = versions.split("/")
+        path = base + pkg + "/" + ver
+        if not os.path.exists(path):
+            install_pkg(path, pkg + "==" + ver)
+
+
+def install_pkg(path, pkg, base="fw/"):
+    if pkg.split("==")[0] == "torch":
+        subprocess.run(
+            f"pip3 install {pkg} --default-timeout=100 --extra-index-url https://download.pytorch.org/whl/cu116  --no-cache-dir",
+            shell=True,
+        )
+    elif pkg.split("==")[0] == "jaxlib":
+        subprocess.run(
+            f"pip3 install {pkg} --default-timeout=100 -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html   --no-cache-dir",
+            shell=True,
+        )
+    else:
+        subprocess.run(
+            f"pip3 install {pkg} --default-timeout=100   --no-cache-dir", shell=True
+        )
+
+if __name__=="__main__":
+    arg_lis=sys.argv
+    directory_generator(arg_lis[1:],"")
+
+
+# torch_req = ["torch/1.13.1"]
+# tensorflow_req = [
+#     "tensorflow/2.11.0",
+# ]
+# jax_req = ["jax/0.4.6"]
+# jaxlib_req = ["jaxlib/0.4.6"]
+# numpy_req = ["numpy/1.24.2"]
+#
+# directory_generator(torch_req)
+# directory_generator(tensorflow_req)
+# directory_generator(jax_req)
+# directory_generator(numpy_req)
+# directory_generator(jaxlib_req)

docker/multicuda_requirements.txt

@@ -0,0 +1,23 @@
+h5py
+pytest
+networkx
+hypothesis
+pymongo
+redis
+matplotlib
+opencv-python
+tensorflow-addons
+tensorflow-probability
+functorch
+scipy
+dm-haiku
+pydriller
+tqdm
+coverage
+einops
+psutil
+termcolor
+colorama
+packaging
+nvidia-ml-py
+paddle-bfloat

docker/multiversion_framework_directory.py

@@ -15,7 +15,7 @@ def directory_generator(req, base="/opt/miniconda/fw/"):
 def install_pkg(path, pkg, base="fw/"):
     if pkg.split("==")[0] == "torch":
         subprocess.run(
-            f"pip3 install {pkg} --default-timeout=100 -f https://download.pytorch.org/whl/cpu --target={path} --no-cache-dir",
+            f"pip3 install {pkg} --default-timeout=100 --extra-index-url https://download.pytorch.org/whl/cpu --target={path} --no-cache-dir",
             shell=True,
         )
     elif pkg.split("==")[0] == "jaxlib":
@@ -35,7 +35,57 @@ def install_pkg(path, pkg, base="fw/"):
 
 if __name__=="__main__":
     arg_lis=sys.argv
-    directory_generator(arg_lis[1:])
+
+    if 'backend' in arg_lis:
+        for i in arg_lis[2:]:
+            if i.split('/')[0]=='torch':
+                subprocess.run(
+                    f"pip3 install torch=={i.split('/')[1]} --default-timeout=100 --extra-index-url https://download.pytorch.org/whl/cpu  --no-cache-dir",
+                    shell=True,
+                )
+            elif i.split('/')[0]=='tensorflow':
+                subprocess.run(
+                    f"pip3 install tensorflow-cpu=={i.split('/')[1]} --default-timeout=100   --no-cache-dir",
+                    shell=True
+                )
+            elif i.split('/')[0]=='jaxlib':
+                subprocess.run(
+                    f"pip3 install {i} --default-timeout=100 -f https://storage.googleapis.com/jax-releases/jax_releases.html  --no-cache-dir",
+                    shell=True,
+                )
+            else:
+                subprocess.run(
+                    f"pip3 install {i.split('/')[0]}=={i.split('/')[1]} --default-timeout=100   --no-cache-dir", shell=True
+                )
+        try:
+            import tensorflow
+        except:
+            subprocess.run(
+                f"pip3 install tensorflow-cpu --default-timeout=100  --no-cache-dir",
+                shell=True,
+            )
+        try:
+            import torch
+        except:
+            subprocess.run(
+                f"pip3 install torch --default-timeout=100 --extra-index-url https://download.pytorch.org/whl/cpu  --no-cache-dir",
+                shell=True,
+            )
+        try:
+            import jaxlib
+        except:
+            subprocess.run(
+                f"pip3 install jaxlib --default-timeout=100 -f https://storage.googleapis.com/jax-releases/jax_releases.html  --no-cache-dir",
+                shell=True,
+            )
+        try:
+            import jax
+        except:
+            subprocess.run(
+                f"pip3 install jax  --default-timeout=100   --no-cache-dir", shell=True
+            )
+    else:
+        directory_generator(arg_lis[1:])
 
 
 

docker/multiversion_testing_requirements.txt

@@ -20,4 +20,5 @@ termcolor
 colorama
 packaging
 nvidia-ml-py<=11.495.46 # mod_name=pynvml
-paddle-bfloat
+paddle-bfloat
+jsonpickle