Merge feature/reduce-size

* Improve generate-dockerimage-info * Add baselayout for typo3-solr * Implement Arch Linux support for baselayout * Remove lsb-release * Fix dependency detection for latest images * Remove logrotate * Improve bootstrap * Add transparent wrapper for apt-add-repository (automatic installation & deinstallation) * Add prefetched apt repository lists (cleanup in docker-image-cleanup; reduce load on debian and ubuntu repositories) * Reduze size for bootstrap * Add docker image cleanup script * Add static distribution information collection (for testsuite)
vemukku · Mar 26, 2017 · 0146d00 · 0146d00
1 parent d4886ad
commit 0146d00
Show file tree

Hide file tree

Showing 429 changed files with 1,365 additions and 1,278 deletions.
diff --git a/baselayout/usr/local/bin/apt-add-repository b/baselayout/usr/local/bin/apt-add-repository
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -o nounset   ## set -u : exit the script if you try to use an uninitialised variable
+set -o errexit   ## set -e : exit the script if any statement returns a non-true return value
+
+apt-install software-properties-common
+add-apt-repository $@
+apt-get purge -y -f software-properties-common
diff --git a/baselayout/usr/local/bin/apt-install b/baselayout/usr/local/bin/apt-install
@@ -7,12 +7,18 @@ set -o errexit   ## set -e : exit the script if any statement returns a non-true
 
 export DEBIAN_FRONTEND=noninteractive
 
-# Update apt cache
-apt-get update
+if [[ -f "/tmp/.apt-update" ]]; then
+    echo "Detected prefetched 'apt-get update'"
+    # Install packages
+    apt-get install -y -f --no-install-recommends $*
+else
+    # Update apt cache
+    apt-get update
 
-# Install packages
-apt-get install -y -f --no-install-recommends $*
+    # Install packages
+    apt-get install -y -f --no-install-recommends $*
 
-# Clear files (reduce snapshot size)
-rm -rf /var/lib/apt/lists/*
-apt-get clean -y
+    # Clear files (reduce snapshot size)
+    rm -rf /var/lib/apt/lists/*
+    apt-get clean -y
+fi
diff --git a/baselayout/usr/local/bin/apt-update b/baselayout/usr/local/bin/apt-update
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -o pipefail  # trace ERR through pipes
+set -o errtrace  # trace ERR through 'time command' and other functions
+set -o nounset   ## set -u : exit the script if you try to use an uninitialised variable
+set -o errexit   ## set -e : exit the script if any statement returns a non-true return value
+
+apt-get update
+touch /tmp/.apt-update
diff --git a/baselayout/usr/local/bin/apt-upgrade b/baselayout/usr/local/bin/apt-upgrade
@@ -7,12 +7,18 @@ set -o errexit   ## set -e : exit the script if any statement returns a non-true
 
 export DEBIAN_FRONTEND=noninteractive
 
-# Update apt cache
-apt-get update
+if [[ -f "/tmp/.apt-update" ]]; then
+    echo "Detected prefetched 'apt-get update'"
+    # Update packages
+    apt-get dist-upgrade -y -f
+else
+    # Update apt cache
+    apt-get update
 
-# Install packages
-apt-get dist-upgrade -y -f
+    # Update packages
+    apt-get dist-upgrade -y -f
 
-# Clear files (reduce snapshot size)
-rm -rf /var/lib/apt/lists/*
-apt-get clean -y
+    # Clear files (reduce snapshot size)
+    rm -rf /var/lib/apt/lists/*
+    apt-get clean -y
+fi
diff --git a/baselayout/usr/local/bin/docker-image-cleanup b/baselayout/usr/local/bin/docker-image-cleanup
@@ -0,0 +1,35 @@
+#!/bin/sh
+
+set -o nounset   ## set -u : exit the script if you try to use an uninitialised variable
+set -o errexit   ## set -e : exit the script if any statement returns a non-true return value
+
+LSB_FAMILY=$(cat /etc/dockerimage_distribution_family)
+
+case "$LSB_FAMILY" in
+    Debian)
+        rm -f /tmp/.apt-update
+        apt-get autoremove -y -f
+        apt-get clean -y
+        rm -rf /var/lib/apt/lists/*
+        ;;
+
+    RedHat)
+        yum autoremove --assumeyes
+        yum clean all
+        ;;
+
+    Alpine)
+        find /var/lib/apk/ -mindepth 1 -delete
+        ;;
+
+    Arch)
+        pacman -Sc
+        ;;
+
+    *)
+        echo "ERROR: Distribution $LSB_FAMILY not supported"
+        exit 1
+        ;;
+esac
+
+find /tmp/ /var/log/ -mindepth 1 -delete
diff --git a/baselayout/usr/local/bin/generate-dockerimage-info b/baselayout/usr/local/bin/generate-dockerimage-info
@@ -0,0 +1,95 @@
+#!/bin/sh
+
+set -o nounset   ## set -u : exit the script if you try to use an uninitialised variable
+set -o errexit   ## set -e : exit the script if any statement returns a non-true return value
+
+LSB_FAMILY=""
+
+#############################
+# Distribution detection
+#############################
+
+if [ -x "/usr/bin/apt-get" ]; then
+    # Debian family
+    LSB_FAMILY="Debian"
+
+elif [ -x "/bin/yum" ]; then
+    # RedHat family
+    LSB_FAMILY="RedHat"
+
+elif [ -x "/sbin/apk" ]; then
+    # Alpine family
+    LSB_FAMILY="Alpine"
+
+elif [ -f "/etc/arch-release" ]; then
+    # Alpine family
+    LSB_FAMILY="Arch"
+
+else
+    # Unknown
+    echo "ERROR: Distribution detection failed"
+    exit 1
+fi
+
+#############################
+# Install
+#############################
+
+case "$LSB_FAMILY" in
+    Debian)
+        apt-install lsb-release
+        ;;
+
+    RedHat)
+        yum-install redhat-lsb-core
+        ;;
+esac
+
+#############################
+# Set distribution information
+#############################
+
+echo "Detected $LSB_FAMILY"
+echo "$LSB_FAMILY" > /etc/dockerimage_distribution_family
+echo "$LSB_FAMILY" > /etc/dockerimage_distribution
+
+# Create all files
+touch /etc/dockerimage_distribution_version
+touch /etc/dockerimage_lsb
+touch /etc/dockerimage_lsb_id
+touch /etc/dockerimage_lsb_id
+touch /etc/dockerimage_lsb_release
+touch /etc/dockerimage_lsb_codename
+
+# Collect distribution specific informations
+case "$LSB_FAMILY" in
+    Debian|RedHat)
+        lsb_release -i -s > /etc/dockerimage_distribution
+        lsb_release -r -s > /etc/dockerimage_distribution_version
+        lsb_release -a > /etc/dockerimage_lsb
+        lsb_release -i -s > /etc/dockerimage_lsb_id
+        lsb_release -d -s > /etc/dockerimage_lsb_id
+        lsb_release -r -s > /etc/dockerimage_lsb_release
+        lsb_release -c -s > /etc/dockerimage_lsb_codename
+        ;;
+
+    Alpine)
+        cat /etc/alpine-release > /etc/dockerimage_distribution_version
+        ;;
+esac
+
+
+#############################
+# Uninstall
+#############################
+
+case "$LSB_FAMILY" in
+    Debian)
+        apt-get purge -y -f lsb-release
+        ;;
+
+    RedHat)
+        yum erase --assumeyes redhat-lsb-core
+        yum autoremove --assumeyes
+        ;;
+esac
diff --git a/baselayout/usr/local/bin/generate-locales b/baselayout/usr/local/bin/generate-locales
@@ -1,25 +1,27 @@
 #!/bin/bash
 
-set -o pipefail  # trace ERR through pipes
-set -o errtrace  # trace ERR through 'time command' and other functions
+set -o pipefail  ## trace ERR through pipes
+set -o errtrace  ## trace ERR through 'time command' and other functions
 set -o nounset   ## set -u : exit the script if you try to use an uninitialised variable
 set -o errexit   ## set -e : exit the script if any statement returns a non-true return value
 
+LSB_DISTRIBUTION=$(cat /etc/dockerimage_distribution)
+LSB_DISTRIBUTION_VERSION=$(cat /etc/dockerimage_distribution_version)
 
 #######################################
 ## Debian
 #######################################
 
-if [[ "$(lsb_release -i -s)" == "Debian" ]]; then
+if [[ "$LSB_DISTRIBUTION" == "Debian" ]]; then
     /usr/local/bin/apt-install locales-all
 fi
 
 #######################################
 ## Ubuntu
 #######################################
 
-if [[ "$(lsb_release -i -s)" == "Ubuntu" ]]; then
-    if [[ "$(lsb_release -r -s | cut -f 1 -d .)" -ge "16" ]]; then
+if [[ "$LSB_DISTRIBUTION" == "Ubuntu" ]]; then
+    if [[ "$(echo $LSB_DISTRIBUTION_VERSION| cut -f 1 -d .)" -ge "16" ]]; then
         # Ubuntu 16.04 or later
         /usr/local/bin/apt-install locales-all
     else

diff --git a/conf/provision.yml b/conf/provision.yml
@@ -96,6 +96,8 @@ provision:
   piwik:
     configuration:
       piwik/general     : '*'
+  typo3-solr:
+    baselayout: on
   varnish:
     configuration:
       varnish/general   : '*'

diff --git a/docker/apache-dev/alpine-3/Dockerfile b/docker/apache-dev/alpine-3/Dockerfile
@@ -19,7 +19,7 @@ COPY conf/ /opt/docker/
 # Install apache
 RUN echo \
     && /opt/docker/bin/provision run --tag bootstrap --role webdevops-apache-dev \
-    && /opt/docker/bin/bootstrap.sh
-
+    && /opt/docker/bin/bootstrap.sh \
+    && /usr/local/bin/docker-image-cleanup
 
 EXPOSE 80 443
diff --git a/docker/apache-dev/centos-7/Dockerfile b/docker/apache-dev/centos-7/Dockerfile
@@ -19,7 +19,7 @@ COPY conf/ /opt/docker/
 # Install apache
 RUN echo \
     && /opt/docker/bin/provision run --tag bootstrap --role webdevops-apache-dev \
-    && /opt/docker/bin/bootstrap.sh
-
+    && /opt/docker/bin/bootstrap.sh \
+    && /usr/local/bin/docker-image-cleanup
 
 EXPOSE 80 443
diff --git a/docker/apache-dev/debian-7/Dockerfile b/docker/apache-dev/debian-7/Dockerfile
@@ -19,7 +19,7 @@ COPY conf/ /opt/docker/
 # Install apache
 RUN echo \
     && /opt/docker/bin/provision run --tag bootstrap --role webdevops-apache-dev \
-    && /opt/docker/bin/bootstrap.sh
-
+    && /opt/docker/bin/bootstrap.sh \
+    && /usr/local/bin/docker-image-cleanup
 
 EXPOSE 80 443
diff --git a/docker/apache-dev/debian-8/Dockerfile b/docker/apache-dev/debian-8/Dockerfile
@@ -19,7 +19,7 @@ COPY conf/ /opt/docker/
 # Install apache
 RUN echo \
     && /opt/docker/bin/provision run --tag bootstrap --role webdevops-apache-dev \
-    && /opt/docker/bin/bootstrap.sh
-
+    && /opt/docker/bin/bootstrap.sh \
+    && /usr/local/bin/docker-image-cleanup
 
 EXPOSE 80 443
diff --git a/docker/apache-dev/debian-9/Dockerfile b/docker/apache-dev/debian-9/Dockerfile
@@ -19,7 +19,7 @@ COPY conf/ /opt/docker/
 # Install apache
 RUN echo \
     && /opt/docker/bin/provision run --tag bootstrap --role webdevops-apache-dev \
-    && /opt/docker/bin/bootstrap.sh
-
+    && /opt/docker/bin/bootstrap.sh \
+    && /usr/local/bin/docker-image-cleanup
 
 EXPOSE 80 443
diff --git a/docker/apache-dev/ubuntu-12.04/Dockerfile b/docker/apache-dev/ubuntu-12.04/Dockerfile
@@ -19,7 +19,7 @@ COPY conf/ /opt/docker/
 # Install apache
 RUN echo \
     && /opt/docker/bin/provision run --tag bootstrap --role webdevops-apache-dev \
-    && /opt/docker/bin/bootstrap.sh
-
+    && /opt/docker/bin/bootstrap.sh \
+    && /usr/local/bin/docker-image-cleanup
 
 EXPOSE 80 443
diff --git a/docker/apache-dev/ubuntu-14.04/Dockerfile b/docker/apache-dev/ubuntu-14.04/Dockerfile
@@ -19,7 +19,7 @@ COPY conf/ /opt/docker/
 # Install apache
 RUN echo \
     && /opt/docker/bin/provision run --tag bootstrap --role webdevops-apache-dev \
-    && /opt/docker/bin/bootstrap.sh
-
+    && /opt/docker/bin/bootstrap.sh \
+    && /usr/local/bin/docker-image-cleanup
 
 EXPOSE 80 443
diff --git a/docker/apache-dev/ubuntu-15.04/Dockerfile b/docker/apache-dev/ubuntu-15.04/Dockerfile
@@ -19,7 +19,7 @@ COPY conf/ /opt/docker/
 # Install apache
 RUN echo \
     && /opt/docker/bin/provision run --tag bootstrap --role webdevops-apache-dev \
-    && /opt/docker/bin/bootstrap.sh
-
+    && /opt/docker/bin/bootstrap.sh \
+    && /usr/local/bin/docker-image-cleanup
 
 EXPOSE 80 443
diff --git a/docker/apache-dev/ubuntu-15.10/Dockerfile b/docker/apache-dev/ubuntu-15.10/Dockerfile
@@ -19,7 +19,7 @@ COPY conf/ /opt/docker/
 # Install apache
 RUN echo \
     && /opt/docker/bin/provision run --tag bootstrap --role webdevops-apache-dev \
-    && /opt/docker/bin/bootstrap.sh
-
+    && /opt/docker/bin/bootstrap.sh \
+    && /usr/local/bin/docker-image-cleanup
 
 EXPOSE 80 443
diff --git a/docker/apache-dev/ubuntu-16.04/Dockerfile b/docker/apache-dev/ubuntu-16.04/Dockerfile
@@ -19,7 +19,7 @@ COPY conf/ /opt/docker/
 # Install apache
 RUN echo \
     && /opt/docker/bin/provision run --tag bootstrap --role webdevops-apache-dev \
-    && /opt/docker/bin/bootstrap.sh
-
+    && /opt/docker/bin/bootstrap.sh \
+    && /usr/local/bin/docker-image-cleanup
 
 EXPOSE 80 443
diff --git a/docker/apache-dev/ubuntu-16.10/Dockerfile b/docker/apache-dev/ubuntu-16.10/Dockerfile
@@ -19,7 +19,7 @@ COPY conf/ /opt/docker/
 # Install apache
 RUN echo \
     && /opt/docker/bin/provision run --tag bootstrap --role webdevops-apache-dev \
-    && /opt/docker/bin/bootstrap.sh
-
+    && /opt/docker/bin/bootstrap.sh \
+    && /usr/local/bin/docker-image-cleanup
 
 EXPOSE 80 443
diff --git a/docker/apache/alpine-3/Dockerfile b/docker/apache/alpine-3/Dockerfile
@@ -27,7 +27,7 @@ RUN /usr/local/bin/apk-install \
 		s!^(\s*ErrorLog)\s+\S+!\1 /proc/self/fd/2!g; \
 		' /etc/apache2/httpd.conf \
     && /opt/docker/bin/provision run --tag bootstrap --role webdevops-apache \
-    && /opt/docker/bin/bootstrap.sh
-
+    && /opt/docker/bin/bootstrap.sh \
+    && /usr/local/bin/docker-image-cleanup
 
 EXPOSE 80 443
diff --git a/docker/apache/alpine-3/conf/provision/roles/webdevops-apache/tasks/bootstrap.yml b/docker/apache/alpine-3/conf/provision/roles/webdevops-apache/tasks/bootstrap.yml
@@ -84,7 +84,7 @@
   with_items:
    - 'a2dismod mpm_event'
    - 'a2enmod mpm_event'
-  when: (ansible_distribution == 'Ubuntu' and ansible_lsb.major_release|int >= 14) or (ansible_distribution == 'Debian' and ansible_lsb.major_release|int >= 8)
+  when: (ansible_distribution == 'Ubuntu' and dockerimage_distribution_version|int >= 14) or (ansible_distribution == 'Debian' and dockerimage_distribution_version|int >= 8)
 
 - name: Manage modules [Alpine family]
   lineinfile:

diff --git a/docker/apache/centos-7/Dockerfile b/docker/apache/centos-7/Dockerfile
@@ -25,7 +25,7 @@ RUN /usr/local/bin/yum-install \
 		s!^(\s*ErrorLog)\s+\S+!\1 /proc/self/fd/2!g; \
 		' /etc/httpd/conf/httpd.conf /etc/httpd/conf.d/ssl.conf \
     && /opt/docker/bin/provision run --tag bootstrap --role webdevops-apache \
-    && /opt/docker/bin/bootstrap.sh
-
+    && /opt/docker/bin/bootstrap.sh \
+    && /usr/local/bin/docker-image-cleanup
 
 EXPOSE 80 443
diff --git a/docker/apache/centos-7/conf/provision/roles/webdevops-apache/tasks/bootstrap.yml b/docker/apache/centos-7/conf/provision/roles/webdevops-apache/tasks/bootstrap.yml
@@ -84,7 +84,7 @@
   with_items:
    - 'a2dismod mpm_event'
    - 'a2enmod mpm_event'
-  when: (ansible_distribution == 'Ubuntu' and ansible_lsb.major_release|int >= 14) or (ansible_distribution == 'Debian' and ansible_lsb.major_release|int >= 8)
+  when: (ansible_distribution == 'Ubuntu' and dockerimage_distribution_version|int >= 14) or (ansible_distribution == 'Debian' and dockerimage_distribution_version|int >= 8)
 
 - name: Manage modules [Alpine family]
   lineinfile: