Merge pull request sequenceiq#44 from pjfanning/master

Janos Matyas · Janos Matyas · commit 07e053bbbeeb · 2016-01-09T08:52:19.000+01:00
upgrade to spark 1.6.0
diff --git a/Dockerfile b/Dockerfile
@@ -2,13 +2,13 @@ FROM sequenceiq/hadoop-docker:2.6.0
 MAINTAINER SequenceIQ
 
 #support for Hadoop 2.6.0
-RUN curl -s http://d3kbcqa49mib13.cloudfront.net/spark-1.5.1-bin-hadoop2.6.tgz | tar -xz -C /usr/local/
-RUN cd /usr/local && ln -s spark-1.5.1-bin-hadoop2.6 spark
+RUN curl -s http://d3kbcqa49mib13.cloudfront.net/spark-1.6.0-bin-hadoop2.6.tgz | tar -xz -C /usr/local/
+RUN cd /usr/local && ln -s spark-1.6.0-bin-hadoop2.6 spark
 ENV SPARK_HOME /usr/local/spark
 RUN mkdir $SPARK_HOME/yarn-remote-client
 ADD yarn-remote-client $SPARK_HOME/yarn-remote-client
 
-RUN $BOOTSTRAP && $HADOOP_PREFIX/bin/hadoop dfsadmin -safemode leave && $HADOOP_PREFIX/bin/hdfs dfs -put $SPARK_HOME-1.5.1-bin-hadoop2.6/lib /spark
+RUN $BOOTSTRAP && $HADOOP_PREFIX/bin/hadoop dfsadmin -safemode leave && $HADOOP_PREFIX/bin/hdfs dfs -put $SPARK_HOME-1.6.0-bin-hadoop2.6/lib /spark
 
 ENV YARN_CONF_DIR $HADOOP_PREFIX/etc/hadoop
 ENV PATH $PATH:$SPARK_HOME/bin:$HADOOP_PREFIX/bin
diff --git a/README.md b/README.md
@@ -10,12 +10,12 @@ The base Hadoop Docker image is also available as an official [Docker image](htt
 
 ##Pull the image from Docker Repository
 ```
-docker pull sequenceiq/spark:1.5.1
+docker pull sequenceiq/spark:1.6.0
 ```
 
 ## Building the image
 ```
-docker build --rm -t sequenceiq/spark:1.5.1 .
+docker build --rm -t sequenceiq/spark:1.6.0 .
 ```
 
 ## Running the image
@@ -24,16 +24,16 @@ docker build --rm -t sequenceiq/spark:1.5.1 .
 * in your /etc/hosts file add $(boot2docker ip) as host 'sandbox' to make it easier to access your sandbox UI
 * open yarn UI ports when running container
 ```
-docker run -it -p 8088:8088 -p 8042:8042 -h sandbox sequenceiq/spark:1.5.1 bash
+docker run -it -p 8088:8088 -p 8042:8042 -h sandbox sequenceiq/spark:1.6.0 bash
 ```
 or
 ```
-docker run -d -h sandbox sequenceiq/spark:1.5.1 -d
+docker run -d -h sandbox sequenceiq/spark:1.6.0 -d
 ```
 
 ## Versions
 ```
-Hadoop 2.6.0 and Apache Spark v1.5.1 on Centos 
+Hadoop 2.6.0 and Apache Spark v1.6.0 on Centos 
 ```
 
 ## Testing
@@ -71,7 +71,7 @@ spark-submit \
 --driver-memory 1g \
 --executor-memory 1g \
 --executor-cores 1 \
-$SPARK_HOME/lib/spark-examples-1.5.1-hadoop2.6.0.jar
+$SPARK_HOME/lib/spark-examples-1.6.0-hadoop2.6.0.jar
 ```
 
 Estimating Pi (yarn-client mode):
@@ -84,5 +84,5 @@ spark-submit \
 --driver-memory 1g \
 --executor-memory 1g \
 --executor-cores 1 \
-$SPARK_HOME/lib/spark-examples-1.5.1-hadoop2.6.0.jar
+$SPARK_HOME/lib/spark-examples-1.6.0-hadoop2.6.0.jar
 ```
diff --git a/bootstrap.sh b/bootstrap.sh
@@ -13,7 +13,7 @@ cd $HADOOP_PREFIX/share/hadoop/common ; for cp in ${ACP//,/ }; do  echo == $cp;
 sed s/HOSTNAME/$HOSTNAME/ /usr/local/hadoop/etc/hadoop/core-site.xml.template > /usr/local/hadoop/etc/hadoop/core-site.xml
 
 # setting spark defaults
-echo spark.yarn.jar hdfs:///spark/spark-assembly-1.5.1-hadoop2.6.0.jar > $SPARK_HOME/conf/spark-defaults.conf
+echo spark.yarn.jar hdfs:///spark/spark-assembly-1.6.0-hadoop2.6.0.jar > $SPARK_HOME/conf/spark-defaults.conf
 cp $SPARK_HOME/conf/metrics.properties.template $SPARK_HOME/conf/metrics.properties
 
 service sshd start
