Skip to content

Commit a8e6f2c

Browse files
codeboyyongcodeboyyong
committed
init the first draft version
1 parent 1abde2e commit a8e6f2c

21 files changed

+733
-2
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,4 @@
1010

1111
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
1212
hs_err_pid*
13+
/target

‎README.md‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,32 @@
1-
distributed-shell
2-
=================
1+
distrbuted-shell
2+
================
3+
4+
A real distributed shell for yarn
5+
The target is to create a real distrbuted shell in yarn
6+
7+
simple usgae:
8+
1) mvn package
9+
2) export HADOOP_YARN_HOME=/apache2.2.0/hadoop-2.2.0
10+
export HADOOP_VERSION=2.2.0
11+
3) cp ./target/distributed-shell-1.0-SNAPSHOT.jar $HADOOP_YARN_HOME
12+
4) cd $HADOOP_YARN_HOME
13+
14+
### Unmanaged mode
15+
16+
$ bin/hadoop jar $HADOOP_YARN_HOME/share/hadoop/yarn/hadoop-yarn-applications-unmanaged-am-launcher-$HADOOP_VERSION.jar \
17+
Client -classpath distributed-shell-1.0-SNAPSHOT.jar \
18+
-cmd "java com.codeboy.simpleyarnapp.ApplicationMaster /bin/date 2"
19+
20+
You will the date out put is in the out of the yarn job.
21+
In my case, it is $HADOOP_YARN_HOME/logs/userlogs/application_1398635014957_0001/container_1398635014957_0001_01_000002
22+
23+
### Managed mode
24+
25+
$ bin/hadoop fs -copyFromLocal distributed-shell-1.0-SNAPSHOT.jar /apps/simple/distributed-shell-1.0-SNAPSHOT.jar
26+
27+
$ bin/hadoop jar distributed-shell-1.0-SNAPSHOT.jar com.hortonworks.simpleyarnapp.Client /bin/date 2 /apps/simple/distributed-shell-1.0-SNAPSHOT.jar
28+
29+
30+
Use mvn eclipse:eclipse to generate eclipse project
31+
32+

‎ROADMAP.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
road-map
2+
================
3+
1)simple version, pure java http server, socket based REST service,ditributed to certain number of selected node
4+
2)front end evolution + distributed to every node
5+
3)backend -> scala+spray
6+

‎hadoop-installtion/.DS_Store‎

6 KB
Binary file not shown.

‎hadoop-installtion/conf/.DS_Store‎

6 KB
Binary file not shown.

‎hadoop-installtion/conf/2.3.0-cdh5.0.0/core-site.xml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
3+
4+
<configuration>
5+
<property>
6+
<name>fs.default.name</name>
7+
<value>hdfs://ALPINE_HADOOP_HOST:9000</value>
8+
</property>
9+
<property>
10+
<name>hadoop.tmp.dir</name>
11+
<value>ALPINE_HADOOP_HOME/tmp</value>
12+
</property>
13+
</configuration>
14+

‎hadoop-installtion/conf/2.3.0-cdh5.0.0/hdfs-site.xml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
3+
4+
<configuration>
5+
<property>
6+
<name>dfs.name.dir</name>
7+
<value>ALPINE_HADOOP_HOME/name</value>
8+
</property>
9+
10+
<property>
11+
<name>dfs.data.dir</name>
12+
<value>ALPINE_HADOOP_HOME/data</value>
13+
</property>
14+
15+
<property>
16+
<name>dfs.replication</name>
17+
<value>1</value>
18+
</property>
19+
<property>
20+
<name>dfs.permissions</name>
21+
<value>false</value>
22+
</property>
23+
</configuration>
24+

‎hadoop-installtion/conf/2.3.0-cdh5.0.0/mapred-site.xml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
<?xml version="1.0"?>
2+
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
3+
4+
<!-- Put site-specific property overrides in this file. -->
5+
6+
<configuration>
7+
<property>
8+
<name>mapreduce.cluster.temp.dir</name>
9+
<value>ALPINE_HADOOP_HOME/tmp</value>
10+
<final>true</final>
11+
</property>
12+
13+
<property>
14+
<name>mapreduce.framework.name</name>
15+
<value>yarn</value>
16+
</property>
17+
18+
</configuration>
19+

‎hadoop-installtion/conf/2.3.0-cdh5.0.0/yarn-site.xml‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
<?xml version="1.0"?>
2+
3+
<configuration>
4+
5+
<property>
6+
<name>yarn.resourcemanager.resource-tracker.address</name>
7+
<value>ALPINE_HADOOP_HOST:8031</value>
8+
</property>
9+
<property>
10+
<name>yarn.resourcemanager.address</name>
11+
<value>ALPINE_HADOOP_HOST:8032</value>
12+
</property>
13+
<property>
14+
<name>yarn.resourcemanager.scheduler.address</name>
15+
<value>ALPINE_HADOOP_HOST:8030</value>
16+
</property>
17+
<property>
18+
<name>yarn.resourcemanager.admin.address</name>
19+
<value>ALPINE_HADOOP_HOST:8033</value>
20+
</property>
21+
<property>
22+
<name>yarn.resourcemanager.webapp.address</name>
23+
<value>ALPINE_HADOOP_HOST:8088</value>
24+
</property>
25+
<property>
26+
<description>Classpath for typical applications.</description>
27+
<name>yarn.application.classpath</name>
28+
<value>
29+
$HADOOP_CONF_DIR,
30+
$HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,
31+
$HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,
32+
$HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,
33+
$YARN_HOME/*,$YARN_HOME/lib/*,
34+
$HADOOP_COMMON_HOME/share/hadoop/common/*,
35+
$HADOOP_COMMON_HOME/share/hadoop/common/lib/*,
36+
$HADOOP_HDFS_HOME/share/hadoop/hdfs/*,
37+
$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*,
38+
$HADOOP_YARN_HOME/share/hadoop/yarn/*,
39+
$HADOOP_YARN_HOME/share/hadoop/yarn/lib/*,
40+
</value>
41+
</property>
42+
<property>
43+
<name>yarn.nodemanager.aux-services</name>
44+
<value>mapreduce_shuffle</value>
45+
</property>
46+
<property>
47+
<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
48+
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
49+
</property>
50+
<property>
51+
<name>yarn.nodemanager.local-dirs</name>
52+
<value>ALPINE_HADOOP_HOME/data/1/yarn/local</value>
53+
</property>
54+
<property>
55+
<name>yarn.nodemanager.log-dirs</name>
56+
<value>ALPINE_HADOOP_HOME/data/1/yarn/logs</value>
57+
</property>
58+
<property>
59+
<description>Where to aggregate logs</description>
60+
<name>yarn.nodemanager.remote-app-log-dir</name>
61+
<value>ALPINE_HADOOP_HOME/data/1/logs/apps</value>
62+
</property>
63+
</configuration>

‎hadoop-installtion/installation.config‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
2+
2.3.0-cdh5.0.0.image = http://archive.cloudera.com/cdh5/cdh/5/hadoop-2.3.0-cdh5.0.0.tar.gz
3+
2.3.0-cdh5.0.0.vm.host = master-cdh5
4+
2.3.0-cdh5.0.0.vm.ip = 192.168.1.181
5+

0 commit comments

Comments
 (0)