Skip to content

Commit

Permalink
Merge pull request FederatedAI#798 from jiahaoc1993/fix_examples
Browse files Browse the repository at this point in the history 
Package examples to the python image
  • Loading branch information
@dylan-fan
dylan-fan authored Nov 27, 2019
2 parents 2ad29d0 + 89b7b8e commit a6d299c
Show file tree
Hide file tree
Showing 3 changed files with 63 additions and 55 deletions.
4 changes: 2 additions & 2 deletions docker-build/.env
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
PREFIX=federatedai
TAG=1.1-release
BASE_TAG=1.1-release
TAG=1.1.1-release
BASE_TAG=1.1.1-release
111 changes: 59 additions & 52 deletions docker-build/build_cluster_docker.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,58 +14,64 @@ source_code_dir=$(cd `dirname ${WORKINGDIR}`; pwd)
# fetch fate-python image
source .env

# fetch package version
source ${WORKINGDIR}/../cluster-deploy/scripts/default_configurations.sh

buildBase() {
echo "START PACKAGING"
package
[ -f ${source_code_dir}/docker-build/docker/base/pip-packages-fate-1.1.tar.gz ] && rm ${source_code_dir}/docker-build/docker/base/pip-packages-fate-1.1.tar.gz
[ -f ${source_code_dir}/docker-build/docker/base/pip-packages-fate-${python_version}.tar.gz ] && rm ${source_code_dir}/docker-build/docker/base/pip-packages-fate-${python_version}.tar.gz
[ -f ${source_code_dir}/docker-build/docker/base/requirements.txt ] && rm ${source_code_dir}/docker-build/docker/base/requirements.txt
ln ${source_code_dir}/cluster-deploy/packages/pip-packages-fate-1.1.tar.gz ${source_code_dir}/docker-build/docker/base/pip-packages-fate-1.1.tar.gz
ln ${source_code_dir}/cluster-deploy/packages/pip-packages-fate-${python_version}.tar.gz ${source_code_dir}/docker-build/docker/base/pip-packages-fate-${python_version}.tar.gz
ln ${source_code_dir}/requirements.txt ${source_code_dir}/docker-build/docker/base/requirements.txt
echo "START BUILDING BASE IMAGE"
cd ${WORKINGDIR}

docker build -f docker/base/Dockerfile -t ${PREFIX}/base-image:${BASE_TAG} ${source_code_dir}/docker-build/docker/base

rm ${source_code_dir}/docker-build/docker/base/pip-packages-fate-1.1.tar.gz
rm ${source_code_dir}/docker-build/docker/base/pip-packages-fate-${python_version}.tar.gz

echo "FINISH BUILDING BASE IMAGE"
}

buildModule() {
[ -f ${source_code_dir}/docker-build/docker/modules/federation/fate-federation-1.1.tar.gz ] && rm ${source_code_dir}/docker-build/docker/modules/federation/fate-federation-1.1.tar.gz
[ -f ${source_code_dir}/docker-build/docker/modules/proxy/fate-proxy-1.1.tar.gz ] && rm ${source_code_dir}/docker-build/docker/modules/proxy/fate-proxy-1.1.tar.gz
[ -f ${source_code_dir}/docker-build/docker/modules/roll/eggroll-roll-1.1.tar.gz ] && rm ${source_code_dir}/docker-build/docker/modules/roll/eggroll-roll-1.1.tar.gz
[ -f ${source_code_dir}/docker-build/docker/modules/meta-service/eggroll-meta-service-1.1.tar.gz ] && rm ${source_code_dir}/docker-build/docker/modules/meta-service/eggroll-meta-service-1.1.tar.gz
[ -f ${source_code_dir}/docker-build/docker/modules/fateboard/fateboard-1.1.jar ] && rm ${source_code_dir}/docker-build/docker/modules/fateboard/fateboard-1.1.jar
[ -f ${source_code_dir}/docker-build/docker/modules/egg/eggroll-api-1.1.tar.gz ] && rm ${source_code_dir}/docker-build/docker/modules/egg/eggroll-api-1.1.tar.gz
[ -f ${source_code_dir}/docker-build/docker/modules/egg/eggroll-computing-1.1.tar.gz ] && rm ${source_code_dir}/docker-build/docker/modules/egg/eggroll-computing-1.1.tar.gz
[ -f ${source_code_dir}/docker-build/docker/modules/egg/eggroll-egg-1.1.tar.gz ] && rm ${source_code_dir}/docker-build/docker/modules/egg/eggroll-egg-1.1.tar.gz
[ -f ${source_code_dir}/docker-build/docker/modules/egg/eggroll-storage-service-cxx-1.1.tar.gz ] && rm ${source_code_dir}/docker-build/docker/modules/egg/eggroll-storage-service-cxx-1.1.tar.gz
[ -f ${source_code_dir}/docker-build/docker/modules/federation/fate-federation-${federation_version}.tar.gz ] && rm ${source_code_dir}/docker-build/docker/modules/federation/fate-federation-${federation_version}.tar.gz
[ -f ${source_code_dir}/docker-build/docker/modules/proxy/fate-proxy-${proxy_version}.tar.gz ] && rm ${source_code_dir}/docker-build/docker/modules/proxy/fate-proxy-${proxy_version}.tar.gz
[ -f ${source_code_dir}/docker-build/docker/modules/roll/eggroll-roll-${roll_version}.tar.gz ] && rm ${source_code_dir}/docker-build/docker/modules/roll/eggroll-roll-${roll_version}.tar.gz
[ -f ${source_code_dir}/docker-build/docker/modules/meta-service/eggroll-meta-service-${meta_service_version}.tar.gz ] && rm ${source_code_dir}/docker-build/docker/modules/meta-service/eggroll-meta-service-${meta_service_version}.tar.gz
[ -f ${source_code_dir}/docker-build/docker/modules/fateboard/fateboard-${fateboard_version}.jar ] && rm ${source_code_dir}/docker-build/docker/modules/fateboard/fateboard-${fateboard_version}.jar
[ -f ${source_code_dir}/docker-build/docker/modules/egg/eggroll-api-${egg_version}.tar.gz ] && rm ${source_code_dir}/docker-build/docker/modules/egg/eggroll-api-${egg_version}.tar.gz
[ -f ${source_code_dir}/docker-build/docker/modules/egg/eggroll-computing-${egg_version}.tar.gz ] && rm ${source_code_dir}/docker-build/docker/modules/egg/eggroll-computing-${egg_version}.tar.gz
[ -f ${source_code_dir}/docker-build/docker/modules/egg/eggroll-egg-${egg_version}.tar.gz ] && rm ${source_code_dir}/docker-build/docker/modules/egg/eggroll-egg-${egg_version}.tar.gz
[ -f ${source_code_dir}/docker-build/docker/modules/egg/eggroll-storage-service-cxx-${egg_version}.tar.gz ] && rm ${source_code_dir}/docker-build/docker/modules/egg/eggroll-storage-service-cxx-${egg_version}.tar.gz
[ -f ${source_code_dir}/docker-build/docker/modules/egg/third_party_eggrollv1.tar.gz ] && rm ${source_code_dir}/docker-build/docker/modules/egg/third_party_eggrollv1.tar.gz
[ -d ${source_code_dir}/docker-build/docker/modules/egg/fate_flow ] && rm -r ${source_code_dir}/docker-build/docker/modules/egg/fate_flow
[ -d ${source_code_dir}/docker-build/docker/modules/egg/arch ] && rm -r ${source_code_dir}/docker-build/docker/modules/egg/arch
[ -d ${source_code_dir}/docker-build/docker/modules/egg/federatedml ] && rm -r ${source_code_dir}/docker-build/docker/modules/egg/federatedml
[ -d ${source_code_dir}/docker-build/docker/modules/python/fate_flow ] && rm -r ${source_code_dir}/docker-build/docker/modules/python/fate_flow
[ -d ${source_code_dir}/docker-build/docker/modules/python/examples ] && rm -r ${source_code_dir}/docker-build/docker/modules/python/examples
[ -d ${source_code_dir}/docker-build/docker/modules/python/arch ] && rm -r ${source_code_dir}/docker-build/docker/modules/python/arch
[ -d ${source_code_dir}/docker-build/docker/modules/python/federatedml ] && rm -r ${source_code_dir}/docker-build/docker/modules/python/federatedml
[ -f ${source_code_dir}/docker-build/docker/modules/python/eggroll-api-1.1.tar.gz ] && rm ${source_code_dir}/docker-build/docker/modules/python/eggroll-api-1.1.tar.gz
[ -d ${source_code_dir}/docker-build/docker/modules/python/examples ] && rm -r ${source_code_dir}/docker-build/docker/modules/python/examples
[ -f ${source_code_dir}/docker-build/docker/modules/python/eggroll-api-${egg_version}.tar.gz ] && rm ${source_code_dir}/docker-build/docker/modules/python/eggroll-api-${egg_version}.tar.gz

ln ${source_code_dir}/cluster-deploy/packages/fate-federation-1.1.tar.gz ${source_code_dir}/docker-build/docker/modules/federation/fate-federation-1.1.tar.gz
ln ${source_code_dir}/cluster-deploy/packages/fate-proxy-1.1.tar.gz ${source_code_dir}/docker-build/docker/modules/proxy/fate-proxy-1.1.tar.gz
ln ${source_code_dir}/cluster-deploy/packages/eggroll-roll-1.1.tar.gz ${source_code_dir}/docker-build/docker/modules/roll/eggroll-roll-1.1.tar.gz
ln ${source_code_dir}/cluster-deploy/packages/eggroll-meta-service-1.1.tar.gz ${source_code_dir}/docker-build/docker/modules/meta-service/eggroll-meta-service-1.1.tar.gz
ln ${source_code_dir}/cluster-deploy/packages/fateboard-1.1.jar ${source_code_dir}/docker-build/docker/modules/fateboard/fateboard-1.1.jar
ln ${source_code_dir}/cluster-deploy/packages/fate-federation-${federation_version}.tar.gz ${source_code_dir}/docker-build/docker/modules/federation/fate-federation-${federation_version}.tar.gz
ln ${source_code_dir}/cluster-deploy/packages/fate-proxy-${proxy_version}.tar.gz ${source_code_dir}/docker-build/docker/modules/proxy/fate-proxy-${proxy_version}.tar.gz
ln ${source_code_dir}/cluster-deploy/packages/eggroll-roll-${roll_version}.tar.gz ${source_code_dir}/docker-build/docker/modules/roll/eggroll-roll-${roll_version}.tar.gz
ln ${source_code_dir}/cluster-deploy/packages/eggroll-meta-service-${meta_service_version}.tar.gz ${source_code_dir}/docker-build/docker/modules/meta-service/eggroll-meta-service-${meta_service_version}.tar.gz
ln ${source_code_dir}/cluster-deploy/packages/fateboard-${fateboard_version}.jar ${source_code_dir}/docker-build/docker/modules/fateboard/fateboard-${fateboard_version}.jar
cp -r ${source_code_dir}/fate_flow ${source_code_dir}/docker-build/docker/modules/python/fate_flow
cp -r ${source_code_dir}/arch ${source_code_dir}/docker-build/docker/modules/python/arch
cp -r ${source_code_dir}/federatedml ${source_code_dir}/docker-build/docker/modules/python/federatedml
ln ${source_code_dir}/cluster-deploy/packages/eggroll-api-1.1.tar.gz ${source_code_dir}/docker-build/docker/modules/python/eggroll-api-1.1.tar.gz
cp -r ${source_code_dir}/examples ${source_code_dir}/docker-build/docker/modules/python/examples
ln ${source_code_dir}/cluster-deploy/packages/eggroll-api-${egg_version}.tar.gz ${source_code_dir}/docker-build/docker/modules/python/eggroll-api-${egg_version}.tar.gz
cp -r ${source_code_dir}/fate_flow ${source_code_dir}/docker-build/docker/modules/egg/fate_flow
cp -r ${source_code_dir}/arch ${source_code_dir}/docker-build/docker/modules/egg/arch
cp -r ${source_code_dir}/federatedml ${source_code_dir}/docker-build/docker/modules/egg/federatedml
ln ${source_code_dir}/cluster-deploy/packages/eggroll-api-1.1.tar.gz ${source_code_dir}/docker-build/docker/modules/egg/eggroll-api-1.1.tar.gz
ln ${source_code_dir}/cluster-deploy/packages/eggroll-computing-1.1.tar.gz ${source_code_dir}/docker-build/docker/modules/egg/eggroll-computing-1.1.tar.gz
ln ${source_code_dir}/cluster-deploy/packages/eggroll-egg-1.1.tar.gz ${source_code_dir}/docker-build/docker/modules/egg/eggroll-egg-1.1.tar.gz
ln ${source_code_dir}/cluster-deploy/packages/eggroll-storage-service-cxx-1.1.tar.gz ${source_code_dir}/docker-build/docker/modules/egg/eggroll-storage-service-cxx-1.1.tar.gz
ln ${source_code_dir}/cluster-deploy/packages/eggroll-api-${egg_version}.tar.gz ${source_code_dir}/docker-build/docker/modules/egg/eggroll-api-${egg_version}.tar.gz
ln ${source_code_dir}/cluster-deploy/packages/eggroll-computing-${egg_version}.tar.gz ${source_code_dir}/docker-build/docker/modules/egg/eggroll-computing-${egg_version}.tar.gz
ln ${source_code_dir}/cluster-deploy/packages/eggroll-egg-${egg_version}.tar.gz ${source_code_dir}/docker-build/docker/modules/egg/eggroll-egg-${egg_version}.tar.gz
ln ${source_code_dir}/cluster-deploy/packages/eggroll-storage-service-cxx-${egg_version}.tar.gz ${source_code_dir}/docker-build/docker/modules/egg/eggroll-storage-service-cxx-${egg_version}.tar.gz
ln ${source_code_dir}/cluster-deploy/packages/third_party_eggrollv1.tar.gz ${source_code_dir}/docker-build/docker/modules/egg/third_party_eggrollv1.tar.gz

for module in "federation" "proxy" "roll" "meta-service" "fateboard" "egg" "python"
Expand All @@ -76,23 +82,24 @@ buildModule() {
echo ""
done;

rm ${source_code_dir}/docker-build/docker/modules/federation/fate-federation-1.1.tar.gz
rm ${source_code_dir}/docker-build/docker/modules/proxy/fate-proxy-1.1.tar.gz
rm ${source_code_dir}/docker-build/docker/modules/roll/eggroll-roll-1.1.tar.gz
rm ${source_code_dir}/docker-build/docker/modules/meta-service/eggroll-meta-service-1.1.tar.gz
rm ${source_code_dir}/docker-build/docker/modules/fateboard/fateboard-1.1.jar
rm ${source_code_dir}/docker-build/docker/modules/egg/eggroll-api-1.1.tar.gz
rm ${source_code_dir}/docker-build/docker/modules/egg/eggroll-computing-1.1.tar.gz
rm ${source_code_dir}/docker-build/docker/modules/egg/eggroll-egg-1.1.tar.gz
rm ${source_code_dir}/docker-build/docker/modules/egg/eggroll-storage-service-cxx-1.1.tar.gz
rm ${source_code_dir}/docker-build/docker/modules/federation/fate-federation-${federation_version}.tar.gz
rm ${source_code_dir}/docker-build/docker/modules/proxy/fate-proxy-${proxy_version}.tar.gz
rm ${source_code_dir}/docker-build/docker/modules/roll/eggroll-roll-${roll_version}.tar.gz
rm ${source_code_dir}/docker-build/docker/modules/meta-service/eggroll-meta-service-${meta_service_version}.tar.gz
rm ${source_code_dir}/docker-build/docker/modules/fateboard/fateboard-${fateboard_version}.jar
rm ${source_code_dir}/docker-build/docker/modules/egg/eggroll-api-${egg_version}.tar.gz
rm ${source_code_dir}/docker-build/docker/modules/egg/eggroll-computing-${egg_version}.tar.gz
rm ${source_code_dir}/docker-build/docker/modules/egg/eggroll-egg-${egg_version}.tar.gz
rm ${source_code_dir}/docker-build/docker/modules/egg/eggroll-storage-service-cxx-${egg_version}.tar.gz
rm ${source_code_dir}/docker-build/docker/modules/egg/third_party_eggrollv1.tar.gz
rm -r ${source_code_dir}/docker-build/docker/modules/egg/fate_flow
rm -r ${source_code_dir}/docker-build/docker/modules/egg/arch
rm -r ${source_code_dir}/docker-build/docker/modules/egg/federatedml
rm -r ${source_code_dir}/docker-build/docker/modules/python/fate_flow
rm -r ${source_code_dir}/docker-build/docker/modules/python/arch
rm -r ${source_code_dir}/docker-build/docker/modules/python/federatedml
rm ${source_code_dir}/docker-build/docker/modules/python/eggroll-api-1.1.tar.gz
rm -r ${source_code_dir}/docker-build/docker/modules/python/examples
rm ${source_code_dir}/docker-build/docker/modules/python/eggroll-api-${egg_version}.tar.gz
echo ""
}

Expand Down Expand Up @@ -138,38 +145,38 @@ package() {
echo "[INFO] Packaging eggroll"
cd ${eggroll_source_code_dir}
cd api
tar czf eggroll-api-1.1.tar.gz *
mv eggroll-api-1.1.tar.gz ${packages_dir}/
tar czf eggroll-api-${egg_version}.tar.gz *
mv eggroll-api-${egg_version}.tar.gz ${packages_dir}/

cd ${eggroll_source_code_dir}
cd computing
tar czf eggroll-computing-1.1.tar.gz *
mv eggroll-computing-1.1.tar.gz ${packages_dir}/
tar czf eggroll-computing-${egg_version}.tar.gz *
mv eggroll-computing-${egg_version}.tar.gz ${packages_dir}/

cd ${eggroll_source_code_dir}
cd conf
tar czf eggroll-conf-1.1.tar.gz *
mv eggroll-conf-1.1.tar.gz ${packages_dir}/
tar czf eggroll-conf-${egg_version}.tar.gz *
mv eggroll-conf-${egg_version}.tar.gz ${packages_dir}/

cd ${eggroll_source_code_dir}
cd framework/egg/target
tar czf eggroll-egg-1.1.tar.gz eggroll-egg-1.1.jar lib/
mv eggroll-egg-1.1.tar.gz ${packages_dir}/
tar czf eggroll-egg-${egg_version}.tar.gz eggroll-egg-${egg_version}.jar lib/
mv eggroll-egg-${egg_version}.tar.gz ${packages_dir}/

cd ${eggroll_source_code_dir}
cd framework/meta-service/target
tar czf eggroll-meta-service-1.1.tar.gz eggroll-meta-service-1.1.jar lib/
mv eggroll-meta-service-1.1.tar.gz ${packages_dir}/
tar czf eggroll-meta-service-${meta_service_version}.tar.gz eggroll-meta-service-${meta_service_version}.jar lib/
mv eggroll-meta-service-${meta_service_version}.tar.gz ${packages_dir}/

cd ${eggroll_source_code_dir}
cd framework/roll/target
tar czf eggroll-roll-1.1.tar.gz eggroll-roll-1.1.jar lib/
mv eggroll-roll-1.1.tar.gz ${packages_dir}/
tar czf eggroll-roll-${roll_version}.tar.gz eggroll-roll-${roll_version}.jar lib/
mv eggroll-roll-${roll_version}.tar.gz ${packages_dir}/

cd ${eggroll_source_code_dir}
cd storage/storage-service-cxx
tar czf eggroll-storage-service-cxx-1.1.tar.gz *
mv eggroll-storage-service-cxx-1.1.tar.gz ${packages_dir}/
tar czf eggroll-storage-service-cxx-${egg_version}.tar.gz *
mv eggroll-storage-service-cxx-${egg_version}.tar.gz ${packages_dir}/
echo "[INFO] Package eggroll done"

cd ${source_code_dir}
Expand Down Expand Up @@ -207,15 +214,15 @@ package() {
echo "[INFO] Compile fate done"

echo "[INFO] Packaging fate"
cp ${source_code_dir}/fateboard/target/fateboard-1.1.jar ${packages_dir}/
cp ${source_code_dir}/fateboard/target/fateboard-${fateboard_version}.jar ${packages_dir}/

cd ${source_code_dir}/arch/driver/federation/target
tar czf fate-federation-1.1.tar.gz fate-federation-1.1.jar lib/
mv fate-federation-1.1.tar.gz ${packages_dir}/
tar czf fate-federation-${federation_version}.tar.gz fate-federation-${federation_version}.jar lib/
mv fate-federation-${federation_version}.tar.gz ${packages_dir}/

cd ${source_code_dir}/arch/networking/proxy/target
tar czf fate-proxy-1.1.tar.gz fate-proxy-1.1.jar lib/
mv fate-proxy-1.1.tar.gz ${packages_dir}/
tar czf fate-proxy-${proxy_version}.tar.gz fate-proxy-${proxy_version}.jar lib/
mv fate-proxy-${proxy_version}.tar.gz ${packages_dir}/

echo "[INFO] Packaging base module"
get_module_package ${source_code_dir} "storage-service-cxx third-party" third_party_eggrollv1.tar.gz
Expand Down
3 changes: 2 additions & 1 deletion docker-build/docker/modules/python/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ COPY fate_flow /data/projects/fate/python/fate_flow
COPY arch /data/projects/fate/python/arch
COPY federatedml /data/projects/fate/python/federatedml
COPY eggroll-api-1.1.tar.gz /data/projects/fate/python
COPY examples /data/projects/fate/python/examples

RUN cd /data/projects/fate/python && \
mkdir -p eggroll/api && tar -C eggroll/api -xvf eggroll-api-1.1.tar.gz && \
Expand All @@ -17,4 +18,4 @@ WORKDIR /data/projects/fate/python

ENV PYTHONPATH=/data/projects/fate/python
CMD sleep 5; source /data/projects/python/venv/bin/activate && cd /data/projects/fate/python/fate_flow && \
python fate_flow_server.py
python fate_flow_server.py

0 comments on commit a6d299c

Please sign in to comment.