curvefs_python: add build options for python2/3

xin321 · Dec 23, 2020 · 093c220 · 093c220
1 parent ebc1506
commit 093c220
Show file tree

Hide file tree

Showing 6 changed files with 160 additions and 53 deletions.
diff --git a/curvefs_python/BUILD_bak b/curvefs_python/BUILD_bak
@@ -33,6 +33,7 @@ COPTS = [
     "-Wnon-virtual-dtor",
     "-Wno-missing-field-initializers",
     "-std=c++11",
+    "-I PYTHON_H_DIR",
 ]
 
 cc_library(

diff --git a/curvefs_python/__init__.py b/curvefs_python/__init__.py
@@ -16,4 +16,4 @@
 
 # coding: utf-8
 
-from curvefs import *
+from .curvefs import *
diff --git a/curvefs_python/configure.sh b/curvefs_python/configure.sh
@@ -30,6 +30,32 @@ cd $curve_path/curvefs_python/
 echo "Prepare bazel build file"
 cp BUILD_bak BUILD -f
 
+usage() {
+    echo "Usage: ./configure.sh [python2|python3]"
+}
+
+PYTHON_VER=python2
+
+if [ $# -ge 1 ]; then
+    PYTHON_VER=$1
+fi
+
+echo "${PYTHON_VER}"
+
+if [ "${PYTHON_VER}" = "python2" ] || [ "${PYTHON_VER}" = "python3" ]; then
+    PYTHON_H_DIR=`find /usr/include/ -name "${PYTHON_VER}*" | grep "/usr/include/${PYTHON_VER}*" | head -n1`
+else
+    usage
+    exit 1
+fi
+
+if [ -z $PYTHON_H_DIR ] || [ ! -f $PYTHON_H_DIR/Python.h ]; then
+    echo "Not found include path for Python.h"
+    exit 1
+fi
+
+sed -i "s%PYTHON_H_DIR%${PYTHON_H_DIR}%g" ${PWD}/BUILD
+
 echo "copy libs to tmplib directory"
 libs=`cat BUILD | tr -d "[:blank:]" | grep "^\"-l" | sed 's/[",]//g' | awk '{ print substr($0, 3) }'`
 

diff --git a/curvefs_python/curvefs_wrap.cxx b/curvefs_python/curvefs_wrap.cxx
@@ -183,10 +183,10 @@ template <typename T> T SwigValueInit() {
 #if defined(_DEBUG) && defined(SWIG_PYTHON_INTERPRETER_NO_DEBUG)
 /* Use debug wrappers with the Python release dll */
 # undef _DEBUG
-# include <python2.7/Python.h>
+# include <Python.h>
 # define _DEBUG
 #else
-# include <python2.7/Python.h>
+# include <Python.h>
 #endif
 
 /* -----------------------------------------------------------------------------

diff --git a/mk-deb.sh b/mk-deb.sh
@@ -54,6 +54,67 @@ fi
 
 curve_version=${tag_version}+${commit_id}${debug}
 
+function create_python_wheel() {
+    PYTHON_VER=$(basename $1)
+    curdir=$(pwd)
+    basedir="build/curvefs_${PYTHON_VER}/"
+
+    mkdir -p ${basedir}/tmplib
+    mkdir -p ${basedir}/curvefs
+
+    cp ./curvefs_python/tmplib/* ${basedir}/tmplib
+    cp ./curvefs_python/setup.py ${basedir}/setup.py
+    cp ./curvefs_python/__init__.py ${basedir}/curvefs
+    cp ./curvefs_python/curvefs.py ${basedir}/curvefs
+    cp ./bazel-bin/curvefs_python/libcurvefs.so ${basedir}/curvefs/_curvefs.so
+
+    cd ${basedir}
+    sed -i "s/version-anchor/${curve_version}/g" setup.py
+
+    deps=$(ldd curvefs/_curvefs.so | awk '{ print $1 }' | sed '/^$/d')
+    for i in $(find tmplib/ -name "lib*so"); do
+        basename=$(basename $i)
+        if [[ $deps =~ $basename ]]; then
+            echo $i
+            cp $i curvefs
+        fi
+    done
+
+    ${1} setup.py bdist_wheel
+    cp dist/*whl ${curdir}
+
+    cd ${curdir}
+}
+
+function build_curvefs_python() {
+    for bin in "/usr/bin/python2" "/usr/bin/python3"; do
+        if [ ! -f ${bin} ]; then
+            echo "${bin} not exist"
+            continue
+        fi
+
+        rm curvefs_python/BUILD
+        rm -rf curvefs_python/tmplib
+        rm -rf ./bazel-bin/curvefs_python
+
+        bash ./curvefs_python/configure.sh $(basename ${bin})
+
+        if [ "$1" = "release" ]; then
+            bazel build curvefs_python:curvefs --copt -DHAVE_ZLIB=1 --copt -O2 -s \
+                --define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google \
+                --copt -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --linkopt \
+                -L${dir}/curvefs_python/tmplib/ --copt -DCURVEVERSION=${curve_version}
+        else
+            bazel build curvefs_python:curvefs --copt -DHAVE_ZLIB=1 --compilation_mode=dbg -s \
+                --define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google \
+                --copt -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --linkopt \
+                -L${dir}/curvefs_python/tmplib/ --copt -DCURVEVERSION=${curve_version}
+        fi
+
+        create_python_wheel ${bin}
+    done
+}
+
 #step3 执行编译
 bazel_version=`bazel version | grep "Build label" | awk '{print $3}'`
 if [ -z ${bazel_version} ]
@@ -114,7 +175,7 @@ then
     echo "build phase1 failed"
     exit
 fi
-bash ./curvefs_python/configure.sh
+bash ./curvefs_python/configure.sh python2
 if [ $? -ne 0 ]
 then
     echo "configure failed"
@@ -139,7 +200,7 @@ then
     echo "build phase1 failed"
     exit
 fi
-bash ./curvefs_python/configure.sh
+bash ./curvefs_python/configure.sh python2
 if [ $? -ne 0 ]
 then
     echo "configure failed"
@@ -503,29 +564,7 @@ dpkg-deb -b build/k8s-nbd-package .
 #step6 清理libetcdclient.so编译出现的临时文件
 cd ${dir}/thirdparties/etcdclient
 make clean
-
-# step7 打包python whell
 cd ${dir}
-cp curve-sdk*deb build/
-cd build/
-dpkg-deb -X curve-sdk*deb python-wheel
-cp ${dir}/curvefs_python/setup.py python-wheel/usr
-cd python-wheel/usr
-
-# 复制依赖的so文件到curvefs
-deps=`ldd curvefs/_curvefs.so | awk '{ print $1 }' | sed '/^$/d'`
-for i in `find lib/ -name "lib*so"`
-do
-    basename=$(basename $i)
-    if [[ $deps =~ $basename ]]
-    then
-        echo $i
-        cp $i curvefs
-    fi
-done
-
-# 替换curvefs setup.py中的版本号
-sed -i "s/version-anchor/${curve_version}/g" setup.py
 
-python2 setup.py bdist_wheel
-cp dist/*whl $dir
+# step7 打包python wheel
+build_curvefs_python $1
diff --git a/mk-tar.sh b/mk-tar.sh
@@ -54,6 +54,67 @@ fi
 
 curve_version=${tag_version}+${commit_id}${debug}
 
+function create_python_wheel() {
+    PYTHON_VER=$(basename $1)
+    curdir=$(pwd)
+    basedir="build/curvefs_${PYTHON_VER}/"
+
+    mkdir -p ${basedir}/tmplib
+    mkdir -p ${basedir}/curvefs
+
+    cp ./curvefs_python/tmplib/* ${basedir}/tmplib
+    cp ./curvefs_python/setup.py ${basedir}/setup.py
+    cp ./curvefs_python/__init__.py ${basedir}/curvefs
+    cp ./curvefs_python/curvefs.py ${basedir}/curvefs
+    cp ./bazel-bin/curvefs_python/libcurvefs.so ${basedir}/curvefs/_curvefs.so
+
+    cd ${basedir}
+    sed -i "s/version-anchor/${curve_version}/g" setup.py
+
+    deps=$(ldd curvefs/_curvefs.so | awk '{ print $1 }' | sed '/^$/d')
+    for i in $(find tmplib/ -name "lib*so"); do
+        basename=$(basename $i)
+        if [[ $deps =~ $basename ]]; then
+            echo $i
+            cp $i curvefs
+        fi
+    done
+
+    ${1} setup.py bdist_wheel
+    cp dist/*whl ${curdir}
+
+    cd ${curdir}
+}
+
+function build_curvefs_python() {
+    for bin in "/usr/bin/python2" "/usr/bin/python3"; do
+        if [ ! -f ${bin} ]; then
+            echo "${bin} not exist"
+            continue
+        fi
+
+        rm curvefs_python/BUILD
+        rm -rf curvefs_python/tmplib
+        rm -rf ./bazel-bin/curvefs_python
+
+        bash ./curvefs_python/configure.sh $(basename ${bin})
+
+        if [ "$1" = "release" ]; then
+            bazel build curvefs_python:curvefs --copt -DHAVE_ZLIB=1 --copt -O2 -s \
+                --define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google \
+                --copt -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --linkopt \
+                -L${dir}/curvefs_python/tmplib/ --copt -DCURVEVERSION=${curve_version}
+        else
+            bazel build curvefs_python:curvefs --copt -DHAVE_ZLIB=1 --compilation_mode=dbg -s \
+                --define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google \
+                --copt -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --linkopt \
+                -L${dir}/curvefs_python/tmplib/ --copt -DCURVEVERSION=${curve_version}
+        fi
+
+        create_python_wheel ${bin}
+    done
+}
+
 #step3 执行编译
 bazel_version=`bazel version | grep "Build label" | awk '{print $3}'`
 if [ -z ${bazel_version} ]
@@ -123,7 +184,7 @@ then
     echo "build phase1 failed"
     exit
 fi
-bash ./curvefs_python/configure.sh
+bash ./curvefs_python/configure.sh python2
 if [ $? -ne 0 ]
 then
     echo "configure failed"
@@ -150,7 +211,7 @@ then
     echo "build phase1 failed"
     exit
 fi
-bash ./curvefs_python/configure.sh
+bash ./curvefs_python/configure.sh python2
 if [ $? -ne 0 ]
 then
     echo "configure failed"
@@ -422,30 +483,10 @@ echo "end make tarball"
 echo "start clean etcd"
 cd ${dir}/thirdparties/etcdclient
 make clean
+cd ${dir}
 echo "end clean etcd"
 
 # step7 打包python whell
 echo "start make python whell"
-cd ${dir}/build
-cp -r curve/curve-sdk python-wheel
-cp ${dir}/curvefs_python/setup.py python-wheel/
-cd python-wheel/
-
-# 复制依赖的so文件到curvefs
-deps=`ldd curvefs/_curvefs.so | awk '{ print $1 }' | sed '/^$/d'`
-for i in `find lib/ -name "lib*so"`
-do
-    basename=$(basename $i)
-    if [[ $deps =~ $basename ]]
-    then
-        echo $i
-        cp $i curvefs
-    fi
-done
-
-# 替换curvefs setup.py中的版本号
-sed -i "s/version-anchor/${curve_version}/g" setup.py
-
-python2 setup.py bdist_wheel
-cp dist/*whl $dir
+build_curvefs_python $1
 echo "end make python whell"
Original file line number	Diff line number	Diff line change
Expand Up		@@ -16,4 +16,4 @@

		# coding: utf-8

		from curvefs import *
		from .curvefs import *