优化测试脚本

leoowu · Apr 12, 2015 · a9d48a3 · a9d48a3
1 parent 4d3b52c
commit a9d48a3
Show file tree

Hide file tree

Showing 20 changed files with 145 additions and 138 deletions.
diff --git a/common/build.sh b/common/build.sh
@@ -0,0 +1,16 @@
+docker build -t jamesmo/debian:7 mydebian
+docker build -t jamesmo/jre:7 myjre7
+docker build -t jamesmo/jdk:7 myjava7
+docker build -t jamesmo/solr:4.10.2 mysolr
+
+docker build -t jamesmo/kafka:0.8 mykafka
+docker build -t jamesmo/rabbitmq_base:3.5 myrabbitmq
+docker build -t jamesmo/rabbitmq:3.5 myrabbitmq/rabbitmq
+
+docker build -t jamesmo/myflume:latest myflume
+
+
+docker build -t jamesmo/storm_base:0.9.3 mystorm/storm
+docker build -t jamesmo/storm-nimbus:0.9.3 mystorm/storm-nimbus
+docker build -t jamesmo/storm-supervisor:0.9.3 mystorm/storm-supervisor
+docker build -t jamesmo/storm-ui:0.9.3 mystorm/storm-ui
diff --git a/common/mycloud/README.md b/common/mycloud/README.md
@@ -51,7 +51,8 @@ Pig和Hive还为HBase提供了高层语言支持，使得在HBase上进行数据
 * flume-ng_hbase会自动建表，如果有同名表会被覆盖。注意/hbase目录允许所有用户777。
 *
 *
-* 生产数据
+* 生产数据（nodeame启动慢;fig restart flume1）
+* 
 * telnet 192.168.59.103 44448
 *
 * 查看数据-hive数据(hregionserver-node)

diff --git a/common/mycloud/fig.yml b/common/mycloud/fig.yml
@@ -1,6 +1,7 @@
 #hbase集群
+#hadoop集群
 nn:
-  image: nn_base:latest
+  image: jamesmo/snappy-spark-nn:cdh5
   environment:
     HOSTNAME_NN: mynn
   ports:
@@ -10,10 +11,24 @@ nn:
     - "1022:22"
   hostname: mynn
 
+
+yarn:
+  image: jamesmo/snappy-spark-yarn:cdh5
+  links:
+    - nn:mynn
+  environment:
+    HDFSNAMENODERPC_SERVICE_HOST: mynn
+    HDFSNAMENODERPC_SERVICE_PORT: 8020
+  ports:
+    - "8088:8088"
+    - "19888:19888"
+  hostname: yarn
+
 dn1:
-  image: dn_base:latest
+  image: jamesmo/snappy-spark-dn:cdh5
   links:
     - nn:mynn
+    - yarn
   environment:
     HDFSNAMENODERPC_SERVICE_HOST: mynn
     HDFSNAMENODERPC_SERVICE_PORT: 8020
@@ -23,9 +38,10 @@ dn1:
 
 
 dn2:
-  image: dn_base:latest
+  image: jamesmo/snappy-spark-dn:cdh5
   links:
     - nn:mynn
+    - yarn
   environment:
     HDFSNAMENODERPC_SERVICE_HOST: mynn
     HDFSNAMENODERPC_SERVICE_PORT: 8020
@@ -42,7 +58,7 @@ zk:
     - "3888:3888"
 
 hb:
-  image: hmaster_base:latest
+  image: jamesmo/snappy-spark-hb:cdh5
   links:
     - nn:mynn
     - dn1:mydn1
@@ -60,7 +76,7 @@ hb:
   hostname: hbasemasteripc
 
 rs:
-  image: hregionserver_base:latest
+  image: jamesmo/snappy-spark-rs:cdh5
   links:
     - nn:mynn
     - dn1:mydn1
@@ -80,40 +96,9 @@ rs:
   hostname: hregionserver1
 
 
-#  volumes:
-#    - /Users/moyong/docker-share/data/hadoop/:/tmp/
-
-#初始化建立日志表
-#dbinit:
-#  image: hregionserver_base:latest
-#  entrypoint: /bin/bash
-#  command: -c "sudo -u hdfs sh /home/jamesmo//home/jamesmo/pre-start-hive.sh; exit 0"
-#  links:
-#    - rs
-
-#web namenode http://192.168.59.103:50075/
-#web hbase #web http://192.168.59.103:60010/
-#web regionserver http://192.168.59.103:60030/
-
-#hive 集成 hbase
-#hive 建表
-
-
-#采集数据，传输给kafka
-#flume1:
-#  image: myflume_base:latest
-#  environment:
-#    FLUME_AGENT_NAME: a1
-#    FLUME_CONF_DIR: /opt/flume/conf
-#    FLUME_CONF_FILE: /var/tmp/flume.conf.hbase
-#  ports:
-#    - "44448:44444"
-#  links:
-#    - zk:zk1
-
 
 initdb:
-  image: inithivehbase_base:latest
+  image: jamesmo/snappy-spark-initdb:cdh5
   links:
     - nn:mynn
     - dn1:mydn1
@@ -131,7 +116,7 @@ initdb:
   hostname: hregionserver1
 
 flume1:
-  image: myflume_base:latest
+  image: jamesmo/myflume:latest
   environment:
     FLUME_AGENT_NAME: a1
     FLUME_CONF_DIR: /opt/flume/conf

diff --git a/common/mycloud/flume/fig.yml b/common/mycloud/flume/fig.yml
diff --git a/common/mydebian/Dockerfile b/common/mydebian/Dockerfile
@@ -3,27 +3,32 @@ FROM    debian:latest
 #2015-01-24修订,中文语言
 MAINTAINER JamesMo <[email protected]>
 
-RUN sed -i '1,3d'   /etc/apt/sources.list
-RUN echo '#hello'>> /etc/apt/sources.list
+#RUN sed -i '1,3d'   /etc/apt/sources.list
 
-RUN sed -i '1a \
+RUN sed -i '4a \
     deb http://mirrors.163.com/debian/ wheezy main non-free contrib \n \
     deb http://mirrors.163.com/debian/ wheezy-proposed-updates main contrib non-free \n \
     deb http://mirrors.163.com/debian-security/ wheezy/updates main contrib non-free \n \
     deb-src http://mirrors.163.com/debian/ wheezy main non-free contrib \n \
     deb-src http://mirrors.163.com/debian/ wheezy-proposed-updates main contrib non-free \n \
     deb-src http://mirrors.163.com/debian-security/ wheezy/updates main contrib non-free \n \
-    deb http://ftp.sjtu.edu.cn/debian/ wheezy main non-free contrib \n \
-    deb http://ftp.sjtu.edu.cn/debian/ wheezy-proposed-updates main contrib non-free \n \
-    deb http://ftp.sjtu.edu.cn/debian-security/ wheezy/updates main contrib non-free \n \
-    deb-src http://ftp.sjtu.edu.cn/debian/ wheezy main non-free contrib \n \
-    deb-src http://ftp.sjtu.edu.cn/debian/ wheezy-proposed-updates main contrib non-free \n \
-    deb-src http://ftp.sjtu.edu.cn/debian-security/ wheezy/updates main contrib non-free \n \
     ' /etc/apt/sources.list
 
+# 慢
+#    deb http://ftp.sjtu.edu.cn/debian/ wheezy main non-free contrib \n \
+#    deb http://ftp.sjtu.edu.cn/debian/ wheezy-proposed-updates main contrib non-free \n \
+#    deb http://ftp.sjtu.edu.cn/debian-security/ wheezy/updates main contrib non-free \n \
+#    deb-src http://ftp.sjtu.edu.cn/debian/ wheezy main non-free contrib \n \
+#    deb-src http://ftp.sjtu.edu.cn/debian/ wheezy-proposed-updates main contrib non-free \n \
+#    deb-src http://ftp.sjtu.edu.cn/debian-security/ wheezy/updates main contrib non-free \n \
+
 RUN cat /etc/apt/sources.list
 
-RUN apt-get -qq update  &&  apt-get -qq upgrade
+
+RUN apt-get -y update
+#&&  apt-get  -y upgrade
+
+RUN	   apt-get install -qq -y curl wget sudo
 
 RUN     apt-get -y install locales && \
         sed -i 's/# zh_CN.UTF-8 UTF-8/zh_CN.UTF-8 UTF-8/' /etc/locale.gen && \
@@ -33,21 +38,26 @@ RUN     apt-get -y install locales && \
         export LANG=zh_CN.UTF-8 && \
         locale
 
-RUN	   apt-get install -qq curl
-RUN    apt-get -qq install wget
-RUN    apt-get -qq install sudo
 
-# grab gosu for easy step-down from root
-RUN gpg --keyserver pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4
-RUN curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/1.2/gosu-$(dpkg --print-architecture)" \
-	&& curl -o /usr/local/bin/gosu.asc -SL "https://github.com/tianon/gosu/releases/download/1.2/gosu-$(dpkg --print-architecture).asc" \
-	&& gpg --verify /usr/local/bin/gosu.asc \
-	&& rm /usr/local/bin/gosu.asc \
-	&& chmod +x /usr/local/bin/gosu
+
+RUN apt-get -y update && apt-get install -qq -y unzip
+
+#--net=none
+RUN wget https://github.com/jpetazzo/pipework/archive/master.zip
+RUN unzip master.zip
+
+RUN cp pipework-master/pipework  /usr/local/bin/
+RUN chmod +x /usr/local/bin/pipework
+#RUN pipework kbr0 test1 172.17.1.3/[email protected]
+#RUN brctl show
+#RUN ip a
+
+RUN    apt-get clean && \
+      rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 
 ENV LANG       zh_CN.UTF-8
 ENV LANGUAGE   zh_CN:zh
 
 #userage
-#docker build -t mydebian mydebian
+#docker build -t jamesmo/debian:7 mydebian
diff --git a/common/mydebian/fig.yml b/common/mydebian/fig.yml
@@ -1,17 +1,17 @@
 #源和中文环境配置
 base:
-  build: .
-  ports:
-    - "22"
-  volumes_from:
-    - data
-
-#常用工具集合
-data:
-  image: busybox
-  command: /bin/true
-  volumes:
-      - /Users/moyong/docker-share/www/:/data/www/
-      - /Users/moyong/docker-share/logs/web/:/usr/local/nginx/logs/
-      - /tmp/:/keys
+  image: jamesmo/debian:7
+#  ports:
+#    - "22"
+#  volumes_from:
+#    - data
+#
+##常用工具集合
+#data:
+#  image: busybox
+#  command: /bin/true
+#  volumes:
+#      - /Users/moyong/docker-share/www/:/data/www/
+#      - /Users/moyong/docker-share/logs/web/:/usr/local/nginx/logs/
+#      - /tmp/:/keys
 
diff --git a/common/mydns/Readme.md b/common/mydns/Readme.md
@@ -0,0 +1,15 @@
+#shell 定制启动脚本；fig 启动脚本的名称与父目录相关,不能指定
+
+#启动dns服务器，指定域名 aabb
+docker run --name dns -v /var/run/docker.sock:/docker.sock phensley/docker-dns \
+    --domain aabb.com
+docker logs -f dns
+#启动容器的时候指定DNS服务器，可选 使用--no-recursion 禁止外网访问
+docker run -it --dns $(docker inspect -f '{{.NetworkSettings.IPAddress}}' dns) \
+    --dns-search aabb.com debian:wheezy bash
+    >cat /etc/resolv.conf
+     nameserver 172.17.0.22
+     search aabb.com
+    >ping trusting_jang.aabb.com
+     PING trusting_jang.aabb.com (172.17.0.23): 48 data bytes
+     56 bytes from 172.17.0.23: icmp_seq=0 ttl=64 time=0.074 ms
diff --git a/common/myflume/fig.yml b/common/myflume/fig.yml
@@ -1,15 +1,5 @@
 base:
-  build: .
-  environment:
-    FLUME_AGENT_NAME: a1
-    FLUME_CONF_DIR: /opt/flume/conf
-    FLUME_CONF_FILE: /var/tmp/flume.conf.netcat
-  ports:
-    - "44445:44444"
-
-
-base1:
-  image: myflume_base:latest
+  image: jamesmo/myflume:latest
   environment:
     FLUME_AGENT_NAME: a1
     FLUME_CONF_DIR: /opt/flume/conf

diff --git a/common/myjava7/Dockerfile b/common/myjava7/Dockerfile
@@ -1,15 +1,17 @@
-FROM mydebian_base
+FROM jamesmo/debian:7
 
 ## Install Oracle JDK
 RUN wget --no-check-certificate --header "Cookie: oraclelicense=accept-securebackup-cookie" \
-    http://download.oracle.com/otn-pub/java/jdk/7u67-b01/jdk-7u67-linux-x64.tar.gz
+    http://download.oracle.com/otn-pub/java/jdk/7u76-b13/jdk-7u76-linux-x64.tar.gz
 
 RUN mkdir /opt/jdk
-RUN tar -zxf jdk-7u67-linux-x64.tar.gz -C /opt/jdk
-RUN rm jdk-7u67-linux-x64.tar.gz
-RUN update-alternatives --install /usr/bin/java java /opt/jdk/jdk1.7.0_67/bin/java 100
-RUN update-alternatives --install /usr/bin/javac javac /opt/jdk/jdk1.7.0_67/bin/javac 100
-ENV JAVA_HOME /opt/jdk/jdk1.7.0_67/
+RUN tar -zxf jdk-7u76-linux-x64.tar.gz -C /opt/jdk
+RUN rm jdk-7u76-linux-x64.tar.gz
+
+RUN update-alternatives --install /usr/bin/java java /opt/jdk/jdk1.7.0_76/bin/java 100
+RUN update-alternatives --install /usr/bin/javac javac /opt/jdk/jdk1.7.0_76/bin/javac 100
+
+ENV JAVA_HOME /opt/jdk/jdk1.7.0_76/
 
 RUN    apt-get clean && \
       rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
diff --git a/common/myjava7/fig.yml b/common/myjava7/fig.yml
@@ -1,6 +1,6 @@
 #源和中文环境配置
 base:
-  build: .
+  image: jamesmo/jdk:7
   ports:
     - "22"
   volumes_from:

diff --git a/common/myjre7/Dockerfile b/common/myjre7/Dockerfile
@@ -1,19 +1,20 @@
-FROM mydebian_base
+FROM jamesmo/debian:7
 
 RUN wget --no-check-certificate --header "Cookie: oraclelicense=accept-securebackup-cookie" \
-    http://download.oracle.com/otn-pub/java/jdk/7u75-b13/server-jre-7u75-linux-x64.tar.gz
+    http://download.oracle.com/otn-pub/java/jdk/7u76-b13/server-jre-7u76-linux-x64.tar.gz
 
 RUN mkdir /opt/jre
-RUN tar -zxf server-jre-7u75-linux-x64.tar.gz -C /opt/jre
-RUN rm server-jre-7u75-linux-x64.tar.gz
-RUN update-alternatives --install /usr/bin/java java /opt/jre/jdk1.7.0_75/bin/java 100
-RUN update-alternatives --install /usr/bin/javac javac /opt/jre/jdk1.7.0_75/bin/javac 100
-RUN update-alternatives --install /usr/bin/jar jar /opt/jre/jdk1.7.0_75/bin/jar 100
+RUN tar -zxf server-jre-7u76-linux-x64.tar.gz -C /opt/jre
+RUN rm server-jre-7u76-linux-x64.tar.gz
+
+RUN update-alternatives --install /usr/bin/java java /opt/jre/jdk1.7.0_76/bin/java 100
+RUN update-alternatives --install /usr/bin/javac javac /opt/jre/jdk1.7.0_76/bin/javac 100
+RUN update-alternatives --install /usr/bin/jar jar /opt/jre/jdk1.7.0_76/bin/jar 100
 
 #docker镜像不保存当前状态，状态写在配置文件里面
-RUN sed -i '1a JAVA_HOME="/opt/jre/jdk1.7.0_75/"' /etc/profile
+RUN sed -i '1a JAVA_HOME="/opt/jre/jdk1.7.0_76/"' /etc/profile
 
-ENV JAVA_HOME /opt/jre/jdk1.7.0_75/
+ENV JAVA_HOME /opt/jre/jdk1.7.0_76/
 
 RUN    apt-get clean && \
       rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
diff --git a/common/myjre7/fig.yml b/common/myjre7/fig.yml
@@ -1,6 +1,6 @@
 #源和中文环境配置
 base:
-  build: .
+  build: jamesmo/jre:7
   ports:
     - "22"
   volumes_from: