updates CI tools (BinaryAnalysisPlatform#787)

* updates CI tools This PR adds 7 docker files and 3 vagrant files that are using in our CI system to test bap installability. Also this PR updates our `opam-release.sh` script, that is now capable of performing the whole update procedure of the OPAM repo in fully automated way (you need to create only PR manually) * adds a script to delete all non-master packages it keeps the conf-* packages so far, but probably we can get rid of them too, they all landed up in the upstream long time ago, and most likely we do not need to update them often, neither do they point to the git repositories.
yuroc0598 · Mar 7, 2018 · f9d7657 · f9d7657
1 parent b796fcf
commit f9d7657
Show file tree

Hide file tree

Showing 14 changed files with 319 additions and 108 deletions.
diff --git a/README.md b/README.md
@@ -44,8 +44,8 @@ derivatives. For other distributions we provide tgz archives. To
 install bap on a Debian derivative:
 
 ```bash
-wget https://github.com/BinaryAnalysisPlatform/bap/releases/download/v1.3.0/{bap,libbap,libbap-dev}_1.3.0.deb
-sudo dpkg -i {bap,libbap,libbap-dev}_1.3.0.deb
+wget https://github.com/BinaryAnalysisPlatform/bap/releases/download/v1.4.0/{bap,libbap,libbap-dev}_1.4.0.deb
+sudo dpkg -i {bap,libbap,libbap-dev}_1.4.0.deb
 ```
 
 ## From sources
@@ -58,7 +58,7 @@ install BAP and the development environment.  After you've successfully
 following commands:
 
 ```bash
-opam init --comp=4.03.0              # install the compiler
+opam init --comp=4.05.0              # install the compiler
 eval `opam config env`               # activate opam environment
 opam depext --install bap            # install bap
 ```

diff --git a/docker/debian/dev/Dockerfile b/docker/debian/dev/Dockerfile
@@ -0,0 +1,20 @@
+FROM ocaml/opam:debian
+MAINTAINER Ivan Gotovchits <[email protected]>
+RUN sudo apt-get -y update && sudo apt-get -y install \
+    llvm-3.8-dev \
+    git \
+    libcurl4-gnutls-dev \
+    libgmp-dev \
+    zlib1g-dev \
+    binutils-multiarch \
+    clang \
+    time
+RUN opam init --auto-setup --comp=4.05.0 --yes
+RUN eval `opam config env`
+RUN LLVM_CONFIG=llvm-config-3.8 opam install conf-bap-llvm --yes
+RUN opam install conf-binutils
+RUN git clone https://github.com/binaryanalysisplatform/bap
+RUN opam pin add bap ./bap --yes
+RUN sudo apt-get install python-pip --yes
+RUN sudo pip install bap
+RUN opam install merlin ocp-indent --yes
diff --git a/docker/debian/testing/Dockerfile b/docker/debian/testing/Dockerfile
@@ -0,0 +1,16 @@
+FROM ocaml/opam:debian
+MAINTAINER Ivan Gotovchits <[email protected]>
+RUN sudo apt-get -y update && sudo apt-get -y install \
+    llvm-3.8-dev \
+    git \
+    libcurl4-gnutls-dev \
+    libgmp-dev \
+    zlib1g-dev \
+    binutils-multiarch \
+    clang
+RUN opam init --auto-setup --comp=4.05.0 --yes
+RUN opam repo add bap git://github.com/BinaryAnalysisPlatform/opam-repository#testing
+RUN LLVM_CONFIG=llvm-config-3.8 opam install conf-bap-llvm --yes
+RUN OPAMJOBS=16 opam install bap --yes
+RUN sudo apt-get install python-pip --yes
+RUN sudo pip install bap
diff --git a/docker/debian/v1.4/Dockerfile b/docker/debian/v1.4/Dockerfile
@@ -0,0 +1,16 @@
+FROM ocaml/opam:debian
+MAINTAINER Ivan Gotovchits <[email protected]>
+RUN sudo apt-get -y update && sudo apt-get -y install \
+    llvm-3.8-dev \
+    git \
+    libcurl4-gnutls-dev \
+    libgmp-dev \
+    zlib1g-dev \
+    binutils-multiarch \
+    clang
+RUN opam init --auto-setup --comp=4.05.0 --yes
+RUN opam repo add bap git://github.com/BinaryAnalysisPlatform/opam-repository#bap-1.4
+RUN LLVM_CONFIG=llvm-config-3.8 opam install conf-bap-llvm --yes
+RUN OPAMJOBS=16 opam install bap --yes
+RUN sudo apt-get install python-pip --yes
+RUN sudo pip install bap
diff --git a/docker/Dockerfile → docker/trusty/testing/Dockerfile b/docker/Dockerfile → docker/trusty/testing/Dockerfile
@@ -22,7 +22,7 @@ RUN sed -i.bkp -e \
       /etc/sudoers
 USER bap
 WORKDIR /home/bap
-RUN opam init --auto-setup --comp=4.02.3 --yes
+RUN opam init --auto-setup --comp=4.05.0 --yes
 RUN opam repo add bap git://github.com/BinaryAnalysisPlatform/opam-repository#testing
 RUN opam update
 RUN OPAMJOBS=1 opam depext --install bap --yes

diff --git a/docker/trusty/v1.4/Dockerfile b/docker/trusty/v1.4/Dockerfile
@@ -0,0 +1,30 @@
+FROM ubuntu:trusty
+MAINTAINER Ivan Gotovchits <[email protected]>
+RUN apt-get -y update && apt-get -y install \
+    build-essential \
+    curl \
+    git \
+    libx11-dev \
+    m4 \
+    pkg-config \
+    python-pip \
+    software-properties-common \
+    sudo \
+    unzip \
+    wget
+RUN add-apt-repository --yes ppa:avsm/ppa && apt-get update && apt-get -y install \
+    ocaml \
+    ocaml-native-compilers \
+    opam
+RUN useradd -m bap && echo "bap:bap" | chpasswd && adduser bap sudo
+RUN sed -i.bkp -e \
+      's/%sudo\s\+ALL=(ALL\(:ALL\)\?)\s\+ALL/%sudo ALL=NOPASSWD:ALL/g' \
+      /etc/sudoers
+USER bap
+WORKDIR /home/bap
+RUN opam init --auto-setup --comp=4.05.0 --yes
+RUN opam repo add bap git://github.com/BinaryAnalysisPlatform/opam-repository#bap-1.4
+RUN opam update
+RUN OPAMJOBS=1 opam depext --install bap --yes
+RUN sudo pip install bap
+ENTRYPOINT ["opam", "config", "exec", "--"]
diff --git a/docker/xenial/testing/Dockerfile b/docker/xenial/testing/Dockerfile
@@ -0,0 +1,27 @@
+FROM ubuntu:xenial
+MAINTAINER Ivan Gotovchits <[email protected]>
+RUN apt-get -y update && apt-get -y install \
+    build-essential \
+    curl \
+    git \
+    libx11-dev \
+    m4 \
+    pkg-config \
+    python-pip \
+    software-properties-common \
+    sudo \
+    unzip \
+    wget
+RUN apt-get -y install opam
+RUN useradd -m bap && echo "bap:bap" | chpasswd && adduser bap sudo
+RUN sed -i.bkp -e \
+      's/%sudo\s\+ALL=(ALL\(:ALL\)\?)\s\+ALL/%sudo ALL=NOPASSWD:ALL/g' \
+      /etc/sudoers
+USER bap
+WORKDIR /home/bap
+RUN opam init --auto-setup --comp=4.05.0 --yes
+RUN opam repo add bap git://github.com/BinaryAnalysisPlatform/opam-repository#testing
+RUN opam update
+RUN OPAMJOBS=1 opam depext --install bap --yes
+RUN sudo pip install bap
+ENTRYPOINT ["opam", "config", "exec", "--"]
diff --git a/docker/xenial/v1.4/Dockerfile b/docker/xenial/v1.4/Dockerfile
@@ -0,0 +1,27 @@
+FROM ubuntu:xenial
+MAINTAINER Ivan Gotovchits <[email protected]>
+RUN apt-get -y update && apt-get -y install \
+    build-essential \
+    curl \
+    git \
+    libx11-dev \
+    m4 \
+    pkg-config \
+    python-pip \
+    software-properties-common \
+    sudo \
+    unzip \
+    wget
+RUN apt-get -y install opam
+RUN useradd -m bap && echo "bap:bap" | chpasswd && adduser bap sudo
+RUN sed -i.bkp -e \
+      's/%sudo\s\+ALL=(ALL\(:ALL\)\?)\s\+ALL/%sudo ALL=NOPASSWD:ALL/g' \
+      /etc/sudoers
+USER bap
+WORKDIR /home/bap
+RUN opam init --auto-setup --comp=4.05.0 --yes
+RUN opam repo add bap git://github.com/BinaryAnalysisPlatform/opam-repository#bap-1.4
+RUN opam update
+RUN OPAMJOBS=1 opam depext --install bap --yes
+RUN sudo pip install bap
+ENTRYPOINT ["opam", "config", "exec", "--"]
diff --git a/tools/binary-release.sh b/tools/binary-release.sh
@@ -3,9 +3,12 @@
 set -e
 
 
+# bap version
+export VERSION=1.4.0
+
+# OCAML versio
+export OCAML=4.05.0
 
-# use vargrant to build packages
-export VERSION=1.2.0
 
 # This script will create an rpm file, and requires sudo to operate.
 #
@@ -48,7 +51,7 @@ Vagrant.configure(2) do |config|
   config.vm.provision "shell", privileged: false, inline: <<-SHELL
 set -e
 sudo apt-get update
-sudo apt-get --yes install clang wget m4 curl autoconf dpkg-dev libgmp-dev libzip-dev libcurl4-gnutls-dev llvm-$llvm-dev time unzip git
+sudo apt-get --yes install git aspcud clang pkg-config wget m4 curl autoconf dpkg-dev libgmp-dev libzip-dev libcurl4-gnutls-dev llvm-$llvm-dev time unzip git
 export eval \`dpkg-architecture\`
 
 echo "getting opam from https://github.com/ocaml/opam/releases/download/1.2.2/opam-1.2.2-\$DEB_HOST_GNU_CPU-Linux"
@@ -66,12 +69,12 @@ end
 EOF
 vagrant destroy -f
 vagrant up
-buildrpm bap_$VERSION.deb
-buildrpm libbap_$VERSION.deb
-buildrpm libbap-dev_$VERSION.deb
-alien --to-tgz bap_$VERSION.deb
-alien --to-tgz libbap_$VERSION.deb
-alien --to-tgz libbap-dev_$VERSION.deb
+# buildrpm bap_$VERSION.deb
+# buildrpm libbap_$VERSION.deb
+# buildrpm libbap-dev_$VERSION.deb
+# alien --to-tgz bap_$VERSION.deb
+# alien --to-tgz libbap_$VERSION.deb
+# alien --to-tgz libbap-dev_$VERSION.deb
 }
 
-build ubuntu trusty64 3.4
+build ubuntu xenial64 5.0
diff --git a/tools/deb-build.sh b/tools/deb-build.sh
@@ -7,9 +7,9 @@ SOURCE=$GITHUB/bap
 BINDINGS=$GITHUB/bap-bindings
 BINARIES="bap bap-mc bapbundle"
 PREFIX=/usr/local
-VERSION=${VERSION:-1.2.0}
-OCAML=4.02.3
-LLVM_VERSION=${LLVM_VERSION:-3.4}
+VERSION=1.4.0
+OCAML=4.05.0
+LLVM_VERSION=5.0
 ARCH=$(dpkg-architecture -qDEB_BUILD_ARCH)
 
 CONFDIR=$PREFIX/etc/bap
@@ -32,20 +32,26 @@ fi
 
 eval $(opam config env)
 echo OCaml is at `which ocaml`
+echo LLVM is $LLVM_VERSION
+echo BAP version is $VERSION
 which ocaml
 
+echo "Pinning jbuilder to the not yet broken version"
+opam pin add jbuilder 1.0+beta17 -n --yes
+echo "Pinning utop to the latest not broken version"
+opam pin add utop 1.19.3 -n --yes
 echo "Looking in the dev-repo for the current list of dependencies"
 opam pin add bap --dev-repo --yes -n
 echo "Installing OCaml dependenices from OPAM"
-opam install --deps-only bap --yes
+opam install --yes --deps-only bap
 echo "Installed dependencies. Cleaning up..."
 opam pin remove bap
-echo "Cloning a fresh repo"
 
+echo "Cloning a fresh repo"
 [ -d bap-repo ] || git clone $SOURCE bap-repo
 echo "Installing ocamlfind to the system path"
 
-[ -f $PREFIX/bin/ocamlfind ] && cp $PREFIX/bin/ocamlfind $TMPDIR
+[ -f $PREFIX/bin/ocamlfind ] && cp $PREFIX/bin/ocamlfind $TMPDIR/stored-ocamlfind
 sudo cp $(which ocamlfind) $PREFIX/bin
 
 
@@ -54,6 +60,7 @@ git pull
 ./configure --enable-everything \
             --disable-ida --disable-fsi-benchmark \
             --with-llvm-version=$LLVM_VERSION \
+            --with-llvm-config=llvm-config-$LLVM_VERSION \
             --libdir=$(opam config var lib) \
             --plugindir=$PREFIX/lib/bap \
             --prefix=$PREFIX \