Skip to content

Commit b5593ea

Browse files
author
anair-it
committed
Removed version from folder
1 parent 6429165 commit b5593ea

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+794
-5
lines changed

‎build.sh

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/bin/sh
22
cd base; ./build.sh
3-
cd ../hadoop-2.7.1; ./build.sh
4-
cd ../zookeeper-3.4.6; ./build.sh
5-
cd ../hbase-1.1.2; ./build.sh
6-
cd ../kafka-0.8.2; ./build.sh
7-
cd ../storm-0.10.0; ./build.sh
3+
cd ../hadoop; ./build.sh
4+
cd ../zookeeper; ./build.sh
5+
cd ../hbase; ./build.sh
6+
cd ../kafka; ./build.sh
7+
cd ../storm; ./build.sh

‎build.sh~

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/bin/sh
2+
cd base; ./build.sh
3+
cd ../hadoop-2.7.1; ./build.sh
4+
cd ../zookeeper-3.4.6; ./build.sh
5+
cd ../hbase-1.1.2; ./build.sh
6+
cd ../kafka-0.8.2; ./build.sh
7+
cd ../storm-0.10.0; ./build.sh

‎hadoop/Dockerfile

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
FROM anair/hadoop_base
2+
MAINTAINER anair
3+
4+
LABEL description="Build hadoop 2.7.1 image." inspiredBy="https://github.com/sequenceiq/docker-hadoop-ubuntu"
5+
6+
USER root
7+
8+
ENV HADOOP_VERSION 2.7.1
9+
10+
RUN mkdir -p /usr/java/default \
11+
&& ln -s /usr/lib/jvm/java-7-oracle/bin /usr/java/default
12+
ENV PATH $PATH:$JAVA_HOME/bin
13+
14+
# install dev tools
15+
RUN apt-get update \
16+
&& apt-get install -y curl sudo openssh-server openssh-client rsync
17+
18+
# passwordless ssh
19+
RUN rm -f /etc/ssh/ssh_host_dsa_key /etc/ssh/ssh_host_rsa_key /root/.ssh/id_rsa \
20+
&& ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key \
21+
&& ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key \
22+
&& ssh-keygen -q -N "" -t rsa -f /root/.ssh/id_rsa \
23+
&& cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
24+
25+
RUN wget -O - http://mirror.reverse.net/pub/apache/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz | tar -xzf - -C /usr/local/ \
26+
&& cd /usr/local \
27+
&& ln -s ./hadoop-$HADOOP_VERSION hadoop
28+
29+
ENV HADOOP_PREFIX /usr/local/hadoop
30+
RUN sed -i '/^export JAVA_HOME/ s:.*:export JAVA_HOME=/usr/java/default\nexport HADOOP_PREFIX=/usr/local/hadoop\nexport HADOOP_HOME=/usr/local/hadoop\n:' $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh \
31+
&& sed -i '/^export HADOOP_CONF_DIR/ s:.*:export HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop/:' $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh \
32+
&& mkdir $HADOOP_PREFIX/input \
33+
&& cp $HADOOP_PREFIX/etc/hadoop/*.xml $HADOOP_PREFIX/input
34+
35+
# pseudo distributed
36+
ADD core-site.xml.template $HADOOP_PREFIX/etc/hadoop/core-site.xml.template
37+
RUN sed s/HOSTNAME/localhost/ /usr/local/hadoop/etc/hadoop/core-site.xml.template > /usr/local/hadoop/etc/hadoop/core-site.xml
38+
ADD hdfs-site.xml $HADOOP_PREFIX/etc/hadoop/hdfs-site.xml
39+
40+
ADD mapred-site.xml $HADOOP_PREFIX/etc/hadoop/mapred-site.xml
41+
ADD yarn-site.xml $HADOOP_PREFIX/etc/hadoop/yarn-site.xml
42+
43+
RUN $HADOOP_PREFIX/bin/hdfs namenode -format
44+
45+
# fixing the libhadoop.so like a boss
46+
RUN rm /usr/local/hadoop/lib/native/* \
47+
&& curl -Ls http://dl.bintray.com/sequenceiq/sequenceiq-bin/hadoop-native-64-2.7.0.tar|tar -x -C /usr/local/hadoop/lib/native/
48+
49+
ADD ssh_config /root/.ssh/config
50+
RUN chmod 600 /root/.ssh/config \
51+
&& chown root:root /root/.ssh/config
52+
53+
ADD bootstrap.sh /etc/bootstrap.sh
54+
RUN chown root:root /etc/bootstrap.sh \
55+
&& chmod 700 /etc/bootstrap.sh
56+
57+
ENV BOOTSTRAP /etc/bootstrap.sh
58+
59+
# workingaround docker.io build error
60+
RUN ls -la /usr/local/hadoop/etc/hadoop/*-env.sh \
61+
&& chmod +x /usr/local/hadoop/etc/hadoop/*-env.sh \
62+
&& ls -la /usr/local/hadoop/etc/hadoop/*-env.sh
63+
64+
# fix the 254 error code
65+
RUN sed -i "/^[^#]*UsePAM/ s/.*/#&/" /etc/ssh/sshd_config \
66+
&& echo "UsePAM no" >> /etc/ssh/sshd_config \
67+
&& echo "Port 2122" >> /etc/ssh/sshd_config
68+
69+
70+
RUN service ssh start && $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh && $HADOOP_PREFIX/sbin/start-dfs.sh && $HADOOP_PREFIX/bin/hdfs dfs -mkdir -p /user/root
71+
RUN service ssh start && $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh && $HADOOP_PREFIX/sbin/start-dfs.sh && $HADOOP_PREFIX/bin/hdfs dfs -put $HADOOP_PREFIX/etc/hadoop/ input
72+
73+
CMD ["/etc/bootstrap.sh", "-d"]
74+
75+
EXPOSE 49707 50010 50020 50030 50070 50075 50090 8030 8031 8032 8033 8040 8042 8088 22

‎hadoop/README.md

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#hadoop docker image
2+
This is a hadoop docker image.
3+
4+
##Version
5+
2.7.1
6+
7+
8+
##Building the image
9+
- ```./build.sh```
10+
11+
## Running
12+
- ```docker run -p 49707:49707 -p 50010:50010 -p 50020:50020 -p 50030:50030 -p 50070:50070 -p 50075:50075 -p 50090:50090 -p 8030:8030 -p 8031:8031 -p 8032:8032 -p 8033:8033 -p 8040:8040 -p 8042:8042 -p 8088:8088 -it anair/hadoop:2.7.1```
13+
14+
##UI
15+
- http://localhost:8088/
16+

‎hadoop/bootstrap.sh

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/bin/bash
2+
3+
: ${HADOOP_PREFIX:=/usr/local/hadoop}
4+
5+
$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
6+
7+
rm /tmp/*.pid
8+
9+
# installing libraries if any - (resource urls added comma separated to the ACP system variable)
10+
cd $HADOOP_PREFIX/share/hadoop/common ; for cp in ${ACP//,/ }; do echo == $cp; curl -LO $cp ; done; cd -
11+
12+
# altering the core-site configuration
13+
sed s/HOSTNAME/$HOSTNAME/ /usr/local/hadoop/etc/hadoop/core-site.xml.template > /usr/local/hadoop/etc/hadoop/core-site.xml
14+
15+
16+
service ssh start
17+
$HADOOP_PREFIX/sbin/start-dfs.sh
18+
$HADOOP_PREFIX/sbin/start-yarn.sh
19+
20+
if [[ $1 == "-d" ]]; then
21+
while true; do sleep 1000; done
22+
fi
23+
24+
if [[ $1 == "-bash" ]]; then
25+
/bin/bash
26+
fi

‎hadoop/build.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/sh
2+
docker build -t anair/hadoop:2.7.1 .
3+

‎hadoop/core-site.xml.template

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
<configuration>
2+
<property>
3+
<name>fs.defaultFS</name>
4+
<value>hdfs://HOSTNAME:9000</value>
5+
</property>
6+
</configuration>

‎hadoop/hdfs-site.xml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
<configuration>
2+
<property>
3+
<name>dfs.replication</name>
4+
<value>1</value>
5+
</property>
6+
7+
<property>
8+
<name>dfs.namenode.rpc-bind-host</name>
9+
<value>0.0.0.0</value>
10+
</property>
11+
<property>
12+
<name>dfs.namenode.servicerpc-bind-host</name>
13+
<value>0.0.0.0</value>
14+
</property>
15+
</configuration>

‎hadoop/mapred-site.xml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
<configuration>
2+
<property>
3+
<name>mapreduce.framework.name</name>
4+
<value>yarn</value>
5+
</property>
6+
</configuration>

‎hadoop/ssh_config

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
Host *
2+
UserKnownHostsFile /dev/null
3+
StrictHostKeyChecking no
4+
LogLevel quiet
5+
Port 2122

‎hadoop/yarn-site.xml

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
<configuration>
2+
<property>
3+
<name>yarn.nodemanager.aux-services</name>
4+
<value>mapreduce_shuffle</value>
5+
</property>
6+
7+
<property>
8+
<name>yarn.application.classpath</name>
9+
<value>/usr/local/hadoop/etc/hadoop, /usr/local/hadoop/share/hadoop/common/*, /usr/local/hadoop/share/hadoop/common/lib/*, /usr/local/hadoop/share/hadoop/hdfs/*, /usr/local/hadoop/share/hadoop/hdfs/lib/*, /usr/local/hadoop/share/hadoop/mapreduce/*, /usr/local/hadoop/share/hadoop/mapreduce/lib/*, /usr/local/hadoop/share/hadoop/yarn/*, /usr/local/hadoop/share/hadoop/yarn/lib/*</value>
10+
</property>
11+
12+
<property>
13+
<description>
14+
Number of seconds after an application finishes before the nodemanager's
15+
DeletionService will delete the application's localized file directory
16+
and log directory.
17+
18+
To diagnose Yarn application problems, set this property's value large
19+
enough (for example, to 600 = 10 minutes) to permit examination of these
20+
directories. After changing the property's value, you must restart the
21+
nodemanager in order for it to have an effect.
22+
23+
The roots of Yarn applications' work directories is configurable with
24+
the yarn.nodemanager.local-dirs property (see below), and the roots
25+
of the Yarn applications' log directories is configurable with the
26+
yarn.nodemanager.log-dirs property (see also below).
27+
</description>
28+
<name>yarn.nodemanager.delete.debug-delay-sec</name>
29+
<value>600</value>
30+
</property>
31+
32+
</configuration>

‎hbase/Dockerfile

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
FROM anair/hadoop_base
2+
MAINTAINER anair
3+
4+
LABEL description="Build docker hbase 1.1.2 image"
5+
6+
ENV HBASE_VERSION 1.1.2
7+
8+
# Download and Install HBase
9+
10+
RUN wget -q http://archive.apache.org/dist/hbase/1.1.2/hbase-$HBASE_VERSION-bin.tar.gz && \
11+
tar xzf hbase-$HBASE_VERSION-bin.tar.gz -C /opt/ && \
12+
ln -s /opt/hbase-$HBASE_VERSION /opt/hbase && \
13+
rm hbase-$HBASE_VERSION-bin.tar.gz
14+
15+
ADD hbase-site.xml /opt/hbase/conf/hbase-site.xml
16+
ADD start-hbase.sh /opt/hbase/bin/start-hbase.sh
17+
RUN chmod a+x /opt/hbase/bin/start-hbase.sh
18+
19+
RUN mkdir /opt/hbase/data
20+
RUN mkdir /opt/hbase/logs
21+
22+
VOLUME /opt/hbase/data
23+
VOLUME /opt/hbase/logs
24+
25+
# HBase Master API port
26+
EXPOSE 60000
27+
# HBase Master Web UI
28+
EXPOSE 60010
29+
# Regionserver API port
30+
EXPOSE 60020
31+
# HBase Regionserver web UI
32+
EXPOSE 60030
33+
34+
WORKDIR /opt/hbase/bin
35+
36+
ENV PATH=$PATH:/opt/hbase/bin
37+
38+
CMD start-hbase.sh

‎hbase/README.md

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Hbase docker image
2+
This is a hbase docker image
3+
4+
##Version
5+
1.1.2
6+
7+
##Exposed ports
8+
- HBase Master API port: 60000
9+
- HBase Master Web UI: 60010
10+
- Regionserver API port: 60020
11+
- HBase Regionserver web UI: 60030
12+
13+
##Building the image
14+
```./build.sh```
15+
16+
##Usage
17+
Start a cluster with zookeeper and hbase:
18+
19+
- ```docker-compose up -d ```
20+
21+
Destroy a cluster:
22+
23+
- ```docker-compose stop```
24+
25+
26+
## Running hbase commands
27+
- ```docker exec -it hbase bash```
28+
- ```hbase shell```
29+
- Type the hbase commands
30+
31+
## Accessing Hbase UI
32+
- (http://localhost:60010)

‎hbase/build.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/sh
2+
docker build -t anair/hbase:1.1.2 .
3+

‎hbase/docker-compose.yml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
###zookeeper
2+
zookeeper:
3+
image: anair/zookeeper:3.4.6
4+
container_name: zookeeper
5+
hostname: zk
6+
ports:
7+
- "2181:2181"
8+
9+
###hbase
10+
hbase:
11+
image: anair/hbase:1.1.2
12+
container_name: hbase
13+
hostname: hbase
14+
ports:
15+
- "60000:60000"
16+
- "60010:60010"
17+
- "60020:60020"
18+
- "60030:60030"
19+
links:
20+
- zookeeper:zk
21+

‎hbase/hbase-site.xml

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
<configuration>
2+
<property>
3+
<name>hbase.cluster.distributed</name>
4+
<value>true</value>
5+
</property>
6+
<property>
7+
<name>hbase.rootdir</name>
8+
<value>file:///opt/hbase/data/hbase</value>
9+
</property>
10+
<property>
11+
<name>hbase.master.port</name>
12+
<value>60000</value>
13+
</property>
14+
<property>
15+
<name>hbase.master.info.port</name>
16+
<value>60010</value>
17+
</property>
18+
<property>
19+
<name>hbase.regionserver.port</name>
20+
<value>60020</value>
21+
</property>
22+
<property>
23+
<name>hbase.regionserver.info.port</name>
24+
<value>60030</value>
25+
</property>
26+
27+
<property>
28+
<name>hbase.regionserver.ipc.address</name>
29+
<value>0.0.0.0</value>
30+
</property>
31+
<property>
32+
<name>hbase.zookeeper.quorum</name>
33+
<value>zk:2181</value>
34+
</property>
35+
36+
</configuration>

‎hbase/start-hbase.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/bin/bash
2+
3+
/opt/hbase/bin/hbase regionserver start > logregion.log 2>&1 &
4+
/opt/hbase/bin/hbase master start --localRegionServers=0

‎kafka/Dockerfile

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
FROM anair/hadoop_base
2+
3+
MAINTAINER anair
4+
5+
LABEL description="Build kafka 0.8.2 image"
6+
7+
ENV SCALA_VERSION 2.11
8+
ENV KAFKA_VERSION 0.8.2.1
9+
ENV KAFKA_HOME /opt/kafka_$SCALA_VERSION-$KAFKA_VERSION
10+
11+
# Install Kafka
12+
RUN wget -q -O - http://apache.mirrors.spacedump.net/kafka/$KAFKA_VERSION/kafka_$SCALA_VERSION-$KAFKA_VERSION.tgz | tar -xzf - -C /opt
13+
14+
VOLUME ["/kafka"]
15+
16+
ADD start-kafka.sh /usr/bin/start-kafka.sh
17+
ADD broker-list.sh /usr/bin/broker-list.sh
18+
ADD create-topics.sh /usr/bin/create-topics.sh
19+
20+
WORKDIR $KAFKA_HOME/bin
21+
CMD /usr/bin/start-kafka.sh

0 commit comments

Comments
 (0)