diff --git a/.github/workflows/docker_push.yaml b/.github/workflows/docker_push.yaml new file mode 100644 index 0000000000..d52b7a505d --- /dev/null +++ b/.github/workflows/docker_push.yaml @@ -0,0 +1,41 @@ +name: Docker + +on: + push: + # Publish `master` as Docker `latest` image. + branches: + - master + + # Publish `v1.2.3` tags as releases. + tags: + - v* + +jobs: + # no test is required + push: + runs-on: ubuntu-latest + if: github.event_name == 'push' + + steps: + - uses: actions/checkout@v2 + + - name: Prepare the TAG + id: prepare-the-tag + run: | + # strip git ref prefix from version + TAG="" + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + if [ $VERSION = "master" ]; then + TAG=latest + fi + + echo "::set-output name=tag::$TAG" + + - name: Build image + run: TAG=${{steps.prepare-the-tag.outputs.tag}} bash docker-build/build_cluster_docker.sh all + + - name: Log into DockerHub + run: docker login -u ${{ secrets.DOCKERHUB_USERNAME }} -p ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Push image + run: TAG=${{steps.prepare-the-tag.outputs.tag}} bash docker-build/build_cluster_docker.sh push diff --git a/docker-build/build_cluster_docker.sh b/docker-build/build_cluster_docker.sh index 9ff18d5737..c53170c1a5 100644 --- a/docker-build/build_cluster_docker.sh +++ b/docker-build/build_cluster_docker.sh @@ -7,209 +7,213 @@ set -e BASEDIR=$(dirname "$0") -cd $BASEDIR -WORKINGDIR=`pwd` -source_dir=$(cd `dirname ${WORKINGDIR}`; pwd) +cd "$BASEDIR" +WORKING_DIR=$(pwd) +# fetch parent dir of current working dir +SOURCE_DIR=$( + cd "$(dirname "${WORKING_DIR}")" + pwd +) -# fetch package info -cd ${source_dir} -version=`grep "FATE=" fate.env | awk -F '=' '{print $2}'` +# fetch package info +cd "${SOURCE_DIR}" + +version="$(grep "FATE=" fate.env | awk -F '=' '{print $2}')" package_dir_name="FATE_install_"${version} -package_dir=${source_dir}/cluster-deploy/${package_dir_name} +package_dir=${SOURCE_DIR}/cluster-deploy/${package_dir_name} PREFIX="federatedai" -TAG="${version}-release" +if [ -z "$TAG" ]; then + TAG="${version}-release" +fi + BASE_TAG=${TAG} -source ${WORKINGDIR}/.env +source ${WORKING_DIR}/.env echo "[INFO] Build info" -echo "[INFO] version: "${version} -echo "[INFO] version tag: "${version_tag} +echo "[INFO] Version: "${version} echo "[INFO] Package output dir is "${package_dir} -echo "[INFO] image prefix is: "${PREFIX} -echo "[INFO] image tag is: "${TAG} -echo "[INFO] bash image tag is: "${BASE_TAG} +echo "[INFO] Image prefix is: "${PREFIX} +echo "[INFO] Image tag is: "${TAG} +echo "[INFO] Base image tag is: "${BASE_TAG} - -eggroll_git_url=`grep -A 3 '"eggroll"' .gitmodules | grep 'url' | awk -F '= ' '{print $2}'` -eggroll_git_branch=`grep -A 3 '"eggroll"' .gitmodules | grep 'branch' | awk -F '= ' '{print $2}'` -fateboard_git_url=`grep -A 3 '"fateboard"' .gitmodules | grep 'url' | awk -F '= ' '{print $2}'` -fateboard_git_branch=`grep -A 3 '"fateboard"' .gitmodules | grep 'branch' | awk -F '= ' '{print $2}'` +eggroll_git_url=$(grep -A 3 '"eggroll"' .gitmodules | grep 'url' | awk -F '= ' '{print $2}') +eggroll_git_branch=$(grep -A 3 '"eggroll"' .gitmodules | grep 'branch' | awk -F '= ' '{print $2}') +fateboard_git_url=$(grep -A 3 '"fateboard"' .gitmodules | grep 'url' | awk -F '= ' '{print $2}') +fateboard_git_branch=$(grep -A 3 '"fateboard"' .gitmodules | grep 'branch' | awk -F '= ' '{print $2}') package() { - rm -rf ${package_dir} - mkdir -p ${package_dir} - - # create package path - [ -d ${package_dir} ] && rm -rf ${package_dir} - mkdir -p ${package_dir}/python/arch - - # package python - echo "[INFO] Package fate start" - cp fate.env RELEASE.md ${package_dir}/ - cp -r bin conf python ${package_dir}/ - cp -r examples ${package_dir}/ - echo "[INFO] Package fate done" - echo "[INFO] Package fateboard start" - - cd ${source_dir} - echo "[INFO] Git clone fateboard submodule source code from ${fateboard_git_url} branch ${fateboard_git_branch}" - if [[ -e "fateboard" ]];then - while [[ true ]];do - read -p "The fateboard directory already exists, delete and re-download? [y/n] " input - case ${input} in - [yY]*) - echo "[INFO] Delete the original fateboard" - rm -rf fateboard - git clone ${fateboard_git_url} -b ${fateboard_git_branch} --depth=1 fateboard - break - ;; - [nN]*) - echo "[INFO] Use the original fateboard" - break - ;; - *) - echo "Just enter y or n, please." - ;; - esac - done - else - git clone ${fateboard_git_url} -b ${fateboard_git_branch} --depth=1 fateboard - fi - docker run --rm -u $(id -u):$(id -g) -v ${source_dir}/fateboard:/data/projects/fate/fateboard --entrypoint="" maven:3.6-jdk-8 /bin/bash -c "cd /data/projects/fate/fateboard && mvn clean package -DskipTests" - cd ./fateboard - fateboard_version=$(grep -E -m 1 -o "(.*)" ./pom.xml | tr -d '[\\-a-z<>//]' | awk -F "version" '{print $2}') - echo "[INFO] fateboard version "${fateboard_version} - mkdir -p ${package_dir}/fateboard/conf - mkdir -p ${package_dir}/fateboard/ssh - cp ./target/fateboard-${fateboard_version}.jar ${package_dir}/fateboard/ - cp ./bin/service.sh ${package_dir}/fateboard/ - cp ./src/main/resources/application.properties ${package_dir}/fateboard/conf/ - cd ${package_dir}/fateboard - touch ./ssh/ssh.properties - ln -s fateboard-${fateboard_version}.jar fateboard.jar - echo "[INFO] Package fateboard done" - - echo "[INFO] Package eggroll start" - cd ${source_dir} - echo "[INFO] Git clone eggroll submodule source code from ${eggroll_git_url} branch ${eggroll_git_branch}" - if [[ -e "eggroll" ]];then - while [[ true ]];do - read -p "The eggroll directory already exists, delete and re-download? [y/n] " input - case ${input} in - [yY]*) - echo "[INFO] Delete the original eggroll" - rm -rf eggroll - git clone ${eggroll_git_url} -b ${eggroll_git_branch} --depth=1 eggroll - break - ;; - [nN]*) - echo "[INFO] Use the original eggroll" - break - ;; - *) - echo "Just enter y or n, please." - ;; - esac - done - else - git clone ${eggroll_git_url} -b ${eggroll_git_branch} --depth=1 eggroll - fi - - eggroll_source_code_dir=${source_dir}/eggroll - docker run --rm -u $(id -u):$(id -g) -v ${eggroll_source_code_dir}:/data/projects/fate/eggroll --entrypoint="" maven:3.6-jdk-8 /bin/bash -c "cd /data/projects/fate/eggroll/deploy && bash auto-packaging.sh " - mkdir -p ${package_dir}/eggroll - mv ${source_dir}/eggroll/eggroll.tar.gz ${package_dir}/eggroll/ - cd ${package_dir}/eggroll/ - tar xzf eggroll.tar.gz - rm -rf eggroll.tar.gz - echo "[INFO] Package eggroll done" + rm -rf ${package_dir} + mkdir -p ${package_dir} + + # create package path + [ -d ${package_dir} ] && rm -rf ${package_dir} + mkdir -p ${package_dir}/python/arch + + # package python + echo "[INFO] Package fate start" + cp fate.env RELEASE.md ${package_dir}/ + cp -r bin conf python ${package_dir}/ + cp -r examples ${package_dir}/ + echo "[INFO] Package fate done" + echo "[INFO] Package fateboard start" + + cd ${SOURCE_DIR} + echo "[INFO] Git clone fateboard submodule source code from ${fateboard_git_url} branch ${fateboard_git_branch}" + if [[ -e "fateboard" ]]; then + while [[ true ]]; do + read -p "The fateboard directory already exists, delete and re-download? [y/n] " input + case ${input} in + [yY]*) + echo "[INFO] Delete the original fateboard" + rm -rf fateboard + git clone ${fateboard_git_url} -b ${fateboard_git_branch} --depth=1 fateboard + break + ;; + [nN]*) + echo "[INFO] Use the original fateboard" + break + ;; + *) + echo "Just enter y or n, please." + ;; + esac + done + else + git clone ${fateboard_git_url} -b ${fateboard_git_branch} --depth=1 fateboard + fi + docker run --rm -u $(id -u):$(id -g) -v ${SOURCE_DIR}/fateboard:/data/projects/fate/fateboard --entrypoint="" maven:3.6-jdk-8 /bin/bash -c "cd /data/projects/fate/fateboard && mvn clean package -DskipTests" + cd ./fateboard + fateboard_version=$(grep -E -m 1 -o "(.*)" ./pom.xml | tr -d '[\\-a-z<>//]' | awk -F "version" '{print $2}') + echo "[INFO] fateboard version "${fateboard_version} + mkdir -p ${package_dir}/fateboard/conf + mkdir -p ${package_dir}/fateboard/ssh + cp ./target/fateboard-${fateboard_version}.jar ${package_dir}/fateboard/ + cp ./bin/service.sh ${package_dir}/fateboard/ + cp ./src/main/resources/application.properties ${package_dir}/fateboard/conf/ + cd ${package_dir}/fateboard + touch ./ssh/ssh.properties + ln -s fateboard-${fateboard_version}.jar fateboard.jar + echo "[INFO] Package fateboard done" + + echo "[INFO] Package eggroll start" + cd ${SOURCE_DIR} + echo "[INFO] Git clone eggroll submodule source code from ${eggroll_git_url} branch ${eggroll_git_branch}" + if [[ -e "eggroll" ]]; then + while [[ true ]]; do + read -p "The eggroll directory already exists, delete and re-download? [y/n] " input + case ${input} in + [yY]*) + echo "[INFO] Delete the original eggroll" + rm -rf eggroll + git clone ${eggroll_git_url} -b ${eggroll_git_branch} --depth=1 eggroll + break + ;; + [nN]*) + echo "[INFO] Use the original eggroll" + break + ;; + *) + echo "Just enter y or n, please." + ;; + esac + done + else + git clone ${eggroll_git_url} -b ${eggroll_git_branch} --depth=1 eggroll + fi + + eggroll_source_code_dir=${SOURCE_DIR}/eggroll + docker run --rm -u $(id -u):$(id -g) -v ${eggroll_source_code_dir}:/data/projects/fate/eggroll --entrypoint="" maven:3.6-jdk-8 /bin/bash -c "cd /data/projects/fate/eggroll/deploy && bash auto-packaging.sh " + mkdir -p ${package_dir}/eggroll + mv ${SOURCE_DIR}/eggroll/eggroll.tar.gz ${package_dir}/eggroll/ + cd ${package_dir}/eggroll/ + tar xzf eggroll.tar.gz + rm -rf eggroll.tar.gz + echo "[INFO] Package eggroll done" } buildBase() { - [ -f ${source_dir}/docker-build/docker/base/requirements.txt ] && rm ${source_dir}/docker-build/docker/base/requirements.txt - ln ${source_dir}/python/requirements.txt ${source_dir}/docker-build/docker/base/requirements.txt - echo "START BUILDING BASE IMAGE" - cd ${WORKINGDIR} + [ -f ${SOURCE_DIR}/docker-build/docker/base/requirements.txt ] && rm ${SOURCE_DIR}/docker-build/docker/base/requirements.txt + ln ${SOURCE_DIR}/python/requirements.txt ${SOURCE_DIR}/docker-build/docker/base/requirements.txt + echo "START BUILDING BASE IMAGE" + cd ${WORKING_DIR} - docker build --build-arg version=${version} -f docker/base/Dockerfile -t ${PREFIX}/base-image:${BASE_TAG} ${source_dir}/docker-build/docker/base + docker build --build-arg version=${version} -f docker/base/Dockerfile -t ${PREFIX}/base-image:${BASE_TAG} ${SOURCE_DIR}/docker-build/docker/base - rm ${source_dir}/docker-build/docker/base/requirements.txt - echo "FINISH BUILDING BASE IMAGE" + rm ${SOURCE_DIR}/docker-build/docker/base/requirements.txt + echo "FINISH BUILDING BASE IMAGE" } buildModule() { - # handle python - [ -d ${source_dir}/docker-build/docker/modules/python/python ] && rm -rf ${source_dir}/docker-build/docker/modules/python/python - [ -d ${source_dir}/docker-build/docker/modules/python/eggroll ] && rm -rf ${source_dir}/docker-build/docker/modules/python/eggroll - [ -d ${source_dir}/docker-build/docker/modules/python/examples ] && rm -rf ${source_dir}/docker-build/docker/modules/python/examples - [ -d ${source_dir}/docker-build/docker/modules/python/fate.env ] && rm -rf ${source_dir}/docker-build/docker/modules/python/fate.env - cp -r ${package_dir}/python ${source_dir}/docker-build/docker/modules/python/python - cp -r ${package_dir}/eggroll ${source_dir}/docker-build/docker/modules/python/eggroll - cp -r ${package_dir}/examples ${source_dir}/docker-build/docker/modules/python/examples - cp -r ${package_dir}/fate.env ${source_dir}/docker-build/docker/modules/python/fate.env - - # handle fateboard - [ -d ${source_dir}/docker-build/docker/modules/fateboard/fateboard ] && rm -rf ${source_dir}/docker-build/docker/modules/fateboard/fateboard - cp -r ${package_dir}/fateboard ${source_dir}/docker-build/docker/modules/fateboard/fateboard - - # handle eggroll - [ -d ${source_dir}/docker-build/docker/modules/eggroll/python ] && rm -rf ${source_dir}/docker-build/docker/modules/eggroll/python - [ -d ${source_dir}/docker-build/docker/modules/eggroll/eggroll ] && rm -rf ${source_dir}/docker-build/docker/modules/eggroll/eggroll - cp -r ${package_dir}/python ${source_dir}/docker-build/docker/modules/eggroll/python - cp -r ${package_dir}/eggroll/ ${source_dir}/docker-build/docker/modules/eggroll/eggroll - - cd ${source_dir} - - for module in "python" "fateboard" "eggroll" - do - echo "### START BUILDING ${module} ###" - docker build --build-arg version=${version} --build-arg fateboard_version=${fateboard_version} --build-arg PREFIX=${PREFIX} --build-arg BASE_TAG=${BASE_TAG} --no-cache -t ${PREFIX}/${module}:${TAG} -f ${source_dir}/docker-build/docker/modules/${module}/Dockerfile ${source_dir}/docker-build/docker/modules/${module} - echo "### FINISH BUILDING ${module} ###" - echo "" - done; - - # clean up - rm -rf ${source_dir}/docker-build/docker/modules/python/python - rm -rf ${source_dir}/docker-build/docker/modules/python/eggroll - rm -rf ${source_dir}/docker-build/docker/modules/fateboard/fateboard - rm -rf ${source_dir}/docker-build/docker/modules/eggroll/eggroll - rm -rf ${source_dir}/docker-build/docker/modules/eggroll/python - - echo "" + # handle python + [ -d ${source_dir}/docker-build/docker/modules/python/python ] && rm -rf ${source_dir}/docker-build/docker/modules/python/python + [ -d ${source_dir}/docker-build/docker/modules/python/eggroll ] && rm -rf ${source_dir}/docker-build/docker/modules/python/eggroll + [ -d ${source_dir}/docker-build/docker/modules/python/examples ] && rm -rf ${source_dir}/docker-build/docker/modules/python/examples + [ -d ${source_dir}/docker-build/docker/modules/python/fate.env ] && rm -rf ${source_dir}/docker-build/docker/modules/python/fate.env + cp -r ${package_dir}/python ${source_dir}/docker-build/docker/modules/python/python + cp -r ${package_dir}/eggroll ${source_dir}/docker-build/docker/modules/python/eggroll + cp -r ${package_dir}/examples ${source_dir}/docker-build/docker/modules/python/examples + cp -r ${package_dir}/fate.env ${source_dir}/docker-build/docker/modules/python/fate.env + + # handle fateboard + [ -d ${source_dir}/docker-build/docker/modules/fateboard/fateboard ] && rm -rf ${source_dir}/docker-build/docker/modules/fateboard/fateboard + cp -r ${package_dir}/fateboard ${source_dir}/docker-build/docker/modules/fateboard/fateboard + + # handle eggroll + [ -d ${source_dir}/docker-build/docker/modules/eggroll/python ] && rm -rf ${source_dir}/docker-build/docker/modules/eggroll/python + [ -d ${source_dir}/docker-build/docker/modules/eggroll/eggroll ] && rm -rf ${source_dir}/docker-build/docker/modules/eggroll/eggroll + cp -r ${package_dir}/python ${source_dir}/docker-build/docker/modules/eggroll/python + cp -r ${package_dir}/eggroll/ ${source_dir}/docker-build/docker/modules/eggroll/eggroll + + cd ${source_dir} + + for module in "python" "fateboard" "eggroll" "python-nn"; do + echo "### START BUILDING ${module} ###" + docker build --build-arg version=${version} --build-arg fateboard_version=${fateboard_version} --build-arg PREFIX=${PREFIX} --build-arg BASE_TAG=${BASE_TAG} --no-cache -t ${PREFIX}/${module}:${TAG} -f ${source_dir}/docker-build/docker/modules/${module}/Dockerfile ${source_dir}/docker-build/docker/modules/${module} + echo "### FINISH BUILDING ${module} ###" + echo "" + done + + # clean up + rm -rf ${source_dir}/docker-build/docker/modules/python/python + rm -rf ${source_dir}/docker-build/docker/modules/python/eggroll + rm -rf ${source_dir}/docker-build/docker/modules/fateboard/fateboard + rm -rf ${source_dir}/docker-build/docker/modules/eggroll/eggroll + rm -rf ${source_dir}/docker-build/docker/modules/eggroll/python + + echo "" } pushImage() { - ## push image - for module in "python" "eggroll" "fateboard" - do - echo "### START PUSH ${module} ###" - docker push ${PREFIX}/${module}:${TAG} - echo "### FINISH PUSH ${module} ###" - echo "" - done; + ## push image + for module in "python" "eggroll" "fateboard" "python-nn"; do + echo "### START PUSH ${module} ###" + docker push ${PREFIX}/${module}:${TAG} + echo "### FINISH PUSH ${module} ###" + echo "" + done } while [ "$1" != "" ]; do - case $1 in - package) - package - ;; - base) - buildBase - ;; - modules) - buildModule - ;; - all) - package - buildBase - buildModule - ;; - push) + case $1 in + package) + package + ;; + base) + buildBase + ;; + modules) + buildModule + ;; + all) + package + buildBase + buildModule + ;; + push) pushImage ;; - esac - shift + esac + shift done diff --git a/docker-build/docker/base/Dockerfile b/docker-build/docker/base/Dockerfile index a8f1bff066..340912db29 100644 --- a/docker-build/docker/base/Dockerfile +++ b/docker-build/docker/base/Dockerfile @@ -1,9 +1,10 @@ -FROM centos:centos7.2.1511 +FROM centos/python-36-centos7 ARG version -ENV venv_dir /data/projects/python/venv -ENV miniconda3_dir /data/projects/python/miniconda3 +USER root + +WORKDIR /data/projects/python/ COPY requirements.txt /data/projects/python/ @@ -15,25 +16,8 @@ RUN set -eux && \ snappy snappy-devel zlib zlib-devel bzip2 bzip2-devel lz4-devel libasan lsof sysstat telnet psmisc && \ yum clean all -RUN set -eux && \ - cd /data/projects/python/ && \ - curl -sL https://repo.continuum.io/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh -o /data/projects/python/Miniconda3-4.5.4-Linux-x86_64.sh && \ - sh ./Miniconda3-*-Linux-x86_64.sh -b -p ${miniconda3_dir} && \ - rm -f ./Miniconda3-*-Linux-x86_64.sh - -RUN set -eux && \ - cd /data/projects/python/ && \ - curl -L https://webank-ai-1251170195.cos.ap-guangzhou.myqcloud.com/python-env-${version}-release.tar.gz -o pip-packages-fate-${version}.tar.gz && \ - tar -xvf pip-packages-fate-${version}.tar.gz && \ - rm pip-packages-fate-${version}.tar.gz && \ - set -eux && \ - cd /data/projects/python/python-env && \ - ls && \ - ${miniconda3_dir}/bin/pip install virtualenv-20.0.18-py2.py3-none-any.whl -f ./ --no-index && \ - ${miniconda3_dir}/bin/virtualenv -p ${miniconda3_dir}/bin/python3.6 --no-wheel --no-setuptools --no-download ${venv_dir} && \ - source ${venv_dir}/bin/activate && \ - pip install ./setuptools-42.0.2-py2.py3-none-any.whl && \ - pip install -r ../requirements.txt -f ./ && \ - cd ../ && \ - rm -rf ./python-env && \ - pip list | wc -l +RUN pip install --upgrade pip && \ + sed -i '/tensorflow==1.15.4/d' /data/projects/python/requirements.txt && \ + sed -i '/torch==1.4.0/d' /data/projects/python/requirements.txt && \ + sed -i '/torchvision==0.5.0/d' /data/projects/python/requirements.txt && \ + pip install -r requirements.txt diff --git a/docker-build/docker/modules/python-nn/Dockerfile b/docker-build/docker/modules/python-nn/Dockerfile new file mode 100644 index 0000000000..dcf5a44d1c --- /dev/null +++ b/docker-build/docker/modules/python-nn/Dockerfile @@ -0,0 +1,6 @@ +# runtime environment +ARG PREFIX=prefix +ARG BASE_TAG=tag +FROM ${PREFIX}/python:${BASE_TAG} + +RUN pip install torch==1.4.0 tensorflow==1.15.4 torchvision==0.5.0 diff --git a/docker-build/docker/modules/python/Dockerfile b/docker-build/docker/modules/python/Dockerfile index 1ae15d6135..5d2c3b20d8 100644 --- a/docker-build/docker/modules/python/Dockerfile +++ b/docker-build/docker/modules/python/Dockerfile @@ -12,10 +12,7 @@ COPY eggroll /data/projects/fate/eggroll COPY examples /data/projects/fate/examples COPY fate.env /data/projects/fate/ -RUN echo "source /data/projects/python/venv/bin/activate" >> ~/.bashrc && \ - mkdir -p ./fml_agent/data +RUN mkdir -p ./fml_agent/data ENV PYTHONPATH=/data/projects/fate/python:/data/projects/fate/eggroll/python ENV EGGROLL_HOME=/data/projects/fate/eggroll - -CMD sleep 5; source /data/projects/python/venv/bin/activate && python ./fate_flow/fate_flow_server.py