Added hadoop 2.7.1 image

anair-it · anair-it · commit 2db9ac3e46c0 · 2016-03-30T02:58:18.000-04:00
diff --git a/README.md b/README.md
@@ -7,6 +7,7 @@ Versions
 ----
 - Ubuntu: 14.04
 - Java: Oracle JDK 7
+- hadoop : 2.7.1
 - Zookeeper: 3.4.6
 - Hbase: 1.1.2
 - Kafka: 0.8.2
@@ -46,3 +47,4 @@ Inspired by
 - https://github.com/wurstmeister/storm-docker
 - http://sookocheff.com/post/docker/containerizing-zookeeper-a-guided-tour/
 - http://www.slideshare.net/fabiofumarola1/8a-setup-hbase
+- https://hub.docker.com/r/sequenceiq/hadoop-ubuntu/
diff --git a/build.sh b/build.sh
@@ -1,5 +1,6 @@
 #!/bin/sh
 cd base; ./build.sh
+cd ../hadoop-2.7.1; ./build.sh
 cd ../zookeeper-3.4.6; ./build.sh
 cd ../hbase-1.1.2; ./build.sh
 cd ../kafka-0.8.2; ./build.sh
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,3 +1,24 @@
+###hadoop
+hadoop:
+  image: anair/hadoop:2.7.1
+  container_name: hadoop
+  hostname: hadoop
+  ports:
+    - "49707:49707"
+    - "50010:50010"
+    - "50020:50020"
+    - "50030:50030" 
+    - "50070:50070"
+    - "50075:50075"
+    - "50090:50090"
+    - "8030:8030"
+    - "8031:8031"
+    - "8032:8032"
+    - "8033:8033"
+    - "8040:8040"
+    - "8042:8042"
+    - "8088:8088"
+
 ###zookeeper
 zookeeper:
   image: anair/zookeeper:3.4.6
diff --git a/hadoop-2.7.1/Dockerfile b/hadoop-2.7.1/Dockerfile
@@ -0,0 +1,75 @@
+FROM anair/hadoop_base
+MAINTAINER anair
+
+LABEL description="Build hadoop 2.7.1 image." inspiredBy="https://github.com/sequenceiq/docker-hadoop-ubuntu"
+
+USER root
+
+ENV HADOOP_VERSION 2.7.1
+
+RUN mkdir -p /usr/java/default \
+  && ln -s /usr/lib/jvm/java-7-oracle/bin /usr/java/default
+ENV PATH $PATH:$JAVA_HOME/bin
+
+# install dev tools
+RUN apt-get update \
+  && apt-get install -y curl sudo openssh-server openssh-client rsync
+
+# passwordless ssh
+RUN rm -f /etc/ssh/ssh_host_dsa_key /etc/ssh/ssh_host_rsa_key /root/.ssh/id_rsa \
+  && ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key \
+  && ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key \
+  && ssh-keygen -q -N "" -t rsa -f /root/.ssh/id_rsa \
+  && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
+
+RUN wget -O - http://mirror.reverse.net/pub/apache/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz | tar -xzf - -C /usr/local/ \
+  && cd /usr/local \
+  && ln -s ./hadoop-$HADOOP_VERSION hadoop
+
+ENV HADOOP_PREFIX /usr/local/hadoop
+RUN sed -i '/^export JAVA_HOME/ s:.*:export JAVA_HOME=/usr/java/default\nexport HADOOP_PREFIX=/usr/local/hadoop\nexport HADOOP_HOME=/usr/local/hadoop\n:' $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh \
+ && sed -i '/^export HADOOP_CONF_DIR/ s:.*:export HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop/:' $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh \
+ && mkdir $HADOOP_PREFIX/input \
+ && cp $HADOOP_PREFIX/etc/hadoop/*.xml $HADOOP_PREFIX/input
+
+# pseudo distributed
+ADD core-site.xml.template $HADOOP_PREFIX/etc/hadoop/core-site.xml.template
+RUN sed s/HOSTNAME/localhost/ /usr/local/hadoop/etc/hadoop/core-site.xml.template > /usr/local/hadoop/etc/hadoop/core-site.xml
+ADD hdfs-site.xml $HADOOP_PREFIX/etc/hadoop/hdfs-site.xml
+
+ADD mapred-site.xml $HADOOP_PREFIX/etc/hadoop/mapred-site.xml
+ADD yarn-site.xml $HADOOP_PREFIX/etc/hadoop/yarn-site.xml
+
+RUN $HADOOP_PREFIX/bin/hdfs namenode -format
+
+# fixing the libhadoop.so like a boss
+RUN rm  /usr/local/hadoop/lib/native/* \
+   && curl -Ls http://dl.bintray.com/sequenceiq/sequenceiq-bin/hadoop-native-64-2.7.0.tar|tar -x -C /usr/local/hadoop/lib/native/
+
+ADD ssh_config /root/.ssh/config
+RUN chmod 600 /root/.ssh/config \
+  && chown root:root /root/.ssh/config
+
+ADD bootstrap.sh /etc/bootstrap.sh
+RUN chown root:root /etc/bootstrap.sh \
+  && chmod 700 /etc/bootstrap.sh
+
+ENV BOOTSTRAP /etc/bootstrap.sh
+
+# workingaround docker.io build error
+RUN ls -la /usr/local/hadoop/etc/hadoop/*-env.sh \
+ && chmod +x /usr/local/hadoop/etc/hadoop/*-env.sh \
+ && ls -la /usr/local/hadoop/etc/hadoop/*-env.sh
+
+# fix the 254 error code
+RUN sed  -i "/^[^#]*UsePAM/ s/.*/#&/"  /etc/ssh/sshd_config \
+ && echo "UsePAM no" >> /etc/ssh/sshd_config \
+ && echo "Port 2122" >> /etc/ssh/sshd_config
+
+
+RUN service ssh start && $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh && $HADOOP_PREFIX/sbin/start-dfs.sh && $HADOOP_PREFIX/bin/hdfs dfs -mkdir -p /user/root
+RUN service ssh start && $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh && $HADOOP_PREFIX/sbin/start-dfs.sh && $HADOOP_PREFIX/bin/hdfs dfs -put $HADOOP_PREFIX/etc/hadoop/ input
+
+CMD ["/etc/bootstrap.sh", "-d"]
+
+EXPOSE 49707 50010 50020 50030 50070 50075 50090 8030 8031 8032 8033 8040 8042 8088 22
diff --git a/hadoop-2.7.1/Dockerfile~ b/hadoop-2.7.1/Dockerfile~
@@ -0,0 +1,75 @@
+FROM anair/hadoop_base
+MAINTAINER anair
+
+LABEL description="Build hadoop 2.7.1 image." inspiredBy="https://github.com/sequenceiq/docker-hadoop-ubuntu"
+
+USER root
+
+ENV HADOOP_VERSION 2.7.1
+
+RUN mkdir -p /usr/java/default \
+  && ln -s /usr/lib/jvm/java-7-oracle/bin /usr/java/default
+ENV PATH $PATH:$JAVA_HOME/bin
+
+# install dev tools
+RUN apt-get update \
+  && apt-get install -y curl sudo openssh-server openssh-client rsync
+
+# passwordless ssh
+RUN rm -f /etc/ssh/ssh_host_dsa_key /etc/ssh/ssh_host_rsa_key /root/.ssh/id_rsa \
+  && ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key \
+  && ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key \
+  && ssh-keygen -q -N "" -t rsa -f /root/.ssh/id_rsa \
+  && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
+
+RUN wget -O - http://mirror.reverse.net/pub/apache/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz | tar -xzf - -C /usr/local/ \
+  && cd /usr/local \
+  && ln -s ./hadoop-$HADOOP_VERSION hadoop
+
+ENV HADOOP_PREFIX /usr/local/hadoop
+RUN sed -i '/^export JAVA_HOME/ s:.*:export JAVA_HOME=/usr/java/default\nexport HADOOP_PREFIX=/usr/local/hadoop\nexport HADOOP_HOME=/usr/local/hadoop\n:' $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh \
+ && sed -i '/^export HADOOP_CONF_DIR/ s:.*:export HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop/:' $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh \
+ && mkdir $HADOOP_PREFIX/input \
+ && cp $HADOOP_PREFIX/etc/hadoop/*.xml $HADOOP_PREFIX/input
+
+# pseudo distributed
+ADD core-site.xml.template $HADOOP_PREFIX/etc/hadoop/core-site.xml.template
+RUN sed s/HOSTNAME/localhost/ /usr/local/hadoop/etc/hadoop/core-site.xml.template > /usr/local/hadoop/etc/hadoop/core-site.xml
+ADD hdfs-site.xml $HADOOP_PREFIX/etc/hadoop/hdfs-site.xml
+
+ADD mapred-site.xml $HADOOP_PREFIX/etc/hadoop/mapred-site.xml
+ADD yarn-site.xml $HADOOP_PREFIX/etc/hadoop/yarn-site.xml
+
+RUN $HADOOP_PREFIX/bin/hdfs namenode -format
+
+# fixing the libhadoop.so like a boss
+RUN rm  /usr/local/hadoop/lib/native/* \
+   && curl -Ls http://dl.bintray.com/sequenceiq/sequenceiq-bin/hadoop-native-64-2.7.0.tar|tar -x -C /usr/local/hadoop/lib/native/
+
+ADD ssh_config /root/.ssh/config
+RUN chmod 600 /root/.ssh/config \
+  && chown root:root /root/.ssh/config
+
+ADD bootstrap.sh /etc/bootstrap.sh
+RUN chown root:root /etc/bootstrap.sh \
+  && chmod 700 /etc/bootstrap.sh
+
+ENV BOOTSTRAP /etc/bootstrap.sh
+
+# workingaround docker.io build error
+RUN ls -la /usr/local/hadoop/etc/hadoop/*-env.sh \
+ && chmod +x /usr/local/hadoop/etc/hadoop/*-env.sh \
+ && ls -la /usr/local/hadoop/etc/hadoop/*-env.sh
+
+# fix the 254 error code
+RUN sed  -i "/^[^#]*UsePAM/ s/.*/#&/"  /etc/ssh/sshd_config \
+ && echo "UsePAM no" >> /etc/ssh/sshd_config \
+ && echo "Port 2122" >> /etc/ssh/sshd_config
+
+
+RUN service ssh start && $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh && $HADOOP_PREFIX/sbin/start-dfs.sh && $HADOOP_PREFIX/bin/hdfs dfs -mkdir -p /user/root
+RUN service ssh start && $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh && $HADOOP_PREFIX/sbin/start-dfs.sh && $HADOOP_PREFIX/bin/hdfs dfs -put $HADOOP_PREFIX/etc/hadoop/ input
+
+CMD ["/etc/bootstrap.sh", "-d"]
+
+EXPOSE 50020 50090 50070 50010 50075 8031 8032 8033 8040 8042 49707 22 8088 8030
diff --git a/hadoop-2.7.1/README.md b/hadoop-2.7.1/README.md
@@ -0,0 +1,16 @@
+#hadoop docker image
+This is a hadoop docker image.
+
+##Version
+2.7.1
+
+
+##Building the image
+- ```./build.sh```
+
+## Running
+- ```docker run -p 49707:49707 -p 50010:50010 -p 50020:50020 -p 50030:50030 -p 50070:50070 -p 50075:50075 -p 50090:50090 -p 8030:8030 -p 8031:8031 -p 8032:8032 -p 8033:8033 -p 8040:8040 -p 8042:8042 -p 8088:8088 -it anair/hadoop:2.7.1```
+
+##UI
+- http://localhost:8088/
+
diff --git a/hadoop-2.7.1/README.md~ b/hadoop-2.7.1/README.md~
@@ -0,0 +1,13 @@
+#hadoop docker image
+This is a hadoop docker image.
+
+##Version
+2.7.1
+
+
+##Building the image
+- ```./build.sh```
+
+## Running
+- ```docker run -p 49707:49707 -p 50010:50010 -p 50020:50020 -p 50030:50030 -p 50070:50070 -p 50075:50075 -p 50090:50090 -p 8030:8030 -p 8031:8031 -p 8032:8032 -p 8033:8033 -p 8040:8040 -p 8042:8042 -p 8088:8088 -it anair/hadoop:2.7.1```
+
diff --git a/hadoop-2.7.1/bootstrap.sh b/hadoop-2.7.1/bootstrap.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+: ${HADOOP_PREFIX:=/usr/local/hadoop}
+
+$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
+
+rm /tmp/*.pid
+
+# installing libraries if any - (resource urls added comma separated to the ACP system variable)
+cd $HADOOP_PREFIX/share/hadoop/common ; for cp in ${ACP//,/ }; do  echo == $cp; curl -LO $cp ; done; cd -
+
+# altering the core-site configuration
+sed s/HOSTNAME/$HOSTNAME/ /usr/local/hadoop/etc/hadoop/core-site.xml.template > /usr/local/hadoop/etc/hadoop/core-site.xml
+
+
+service ssh start
+$HADOOP_PREFIX/sbin/start-dfs.sh
+$HADOOP_PREFIX/sbin/start-yarn.sh
+
+if [[ $1 == "-d" ]]; then
+  while true; do sleep 1000; done
+fi
+
+if [[ $1 == "-bash" ]]; then
+  /bin/bash
+fi
diff --git a/hadoop-2.7.1/build.sh b/hadoop-2.7.1/build.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+docker build -t anair/hadoop:2.7.1 .
+
diff --git a/hadoop-2.7.1/build.sh~ b/hadoop-2.7.1/build.sh~
@@ -0,0 +1,3 @@
+#!/bin/sh
+docker build -t railinc/zookeeper:3.4.6 .
+
diff --git a/hadoop-2.7.1/core-site.xml.template b/hadoop-2.7.1/core-site.xml.template
@@ -0,0 +1,6 @@
+  <configuration>
+      <property>
+          <name>fs.defaultFS</name>
+          <value>hdfs://HOSTNAME:9000</value>
+      </property>
+  </configuration>
diff --git a/hadoop-2.7.1/hdfs-site.xml b/hadoop-2.7.1/hdfs-site.xml
@@ -0,0 +1,15 @@
+<configuration>
+    <property>
+        <name>dfs.replication</name>
+        <value>1</value>
+    </property>
+
+    <property>
+        <name>dfs.namenode.rpc-bind-host</name>
+        <value>0.0.0.0</value>
+    </property>
+    <property>
+        <name>dfs.namenode.servicerpc-bind-host</name>
+        <value>0.0.0.0</value>
+    </property>
+</configuration>
diff --git a/hadoop-2.7.1/mapred-site.xml b/hadoop-2.7.1/mapred-site.xml
@@ -0,0 +1,6 @@
+<configuration>
+    <property>
+        <name>mapreduce.framework.name</name>
+        <value>yarn</value>
+    </property>
+</configuration>
diff --git a/hadoop-2.7.1/ssh_config b/hadoop-2.7.1/ssh_config
@@ -0,0 +1,5 @@
+Host *
+  UserKnownHostsFile /dev/null
+  StrictHostKeyChecking no
+  LogLevel quiet
+  Port 2122
diff --git a/hadoop-2.7.1/yarn-site.xml b/hadoop-2.7.1/yarn-site.xml
@@ -0,0 +1,32 @@
+<configuration>
+    <property>
+        <name>yarn.nodemanager.aux-services</name>
+        <value>mapreduce_shuffle</value>
+    </property>
+
+    <property>
+      <name>yarn.application.classpath</name>
+      <value>/usr/local/hadoop/etc/hadoop, /usr/local/hadoop/share/hadoop/common/*, /usr/local/hadoop/share/hadoop/common/lib/*, /usr/local/hadoop/share/hadoop/hdfs/*, /usr/local/hadoop/share/hadoop/hdfs/lib/*, /usr/local/hadoop/share/hadoop/mapreduce/*, /usr/local/hadoop/share/hadoop/mapreduce/lib/*, /usr/local/hadoop/share/hadoop/yarn/*, /usr/local/hadoop/share/hadoop/yarn/lib/*</value>
+    </property>
+
+    <property>
+    <description>
+      Number of seconds after an application finishes before the nodemanager's
+      DeletionService will delete the application's localized file directory
+      and log directory.
+
+      To diagnose Yarn application problems, set this property's value large
+      enough (for example, to 600 = 10 minutes) to permit examination of these
+      directories. After changing the property's value, you must restart the
+      nodemanager in order for it to have an effect.
+
+      The roots of Yarn applications' work directories is configurable with
+      the yarn.nodemanager.local-dirs property (see below), and the roots
+      of the Yarn applications' log directories is configurable with the
+      yarn.nodemanager.log-dirs property (see also below).
+    </description>
+    <name>yarn.nodemanager.delete.debug-delay-sec</name>
+    <value>600</value>
+  </property>
+
+</configuration>

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+#!/bin/sh`
	`2`	`+docker build -t anair/hadoop:2.7.1 .`
	`3`	`+`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+#!/bin/sh`
	`2`	`+docker build -t railinc/zookeeper:3.4.6 .`
	`3`	`+`