Update Hadoop and Python

dylanmei · dylanmei · commit fcf30cfabb03 · 2016-03-10T16:05:04.000-08:00
diff --git a/.gitignore b/.gitignore
@@ -1,2 +1,3 @@
 *.tsv
 *.csv
+*.py
diff --git a/Dockerfile b/Dockerfile
@@ -2,13 +2,30 @@ FROM debian:jessie
 MAINTAINER Getty Images "https://github.com/gettyimages"
 
 RUN apt-get update \
-  && apt-get install -y curl net-tools unzip \
+ && apt-get install -y locales \
+ && dpkg-reconfigure -f noninteractive locales \
+ && locale-gen C.UTF-8 \
+ && /usr/sbin/update-locale LANG=C.UTF-8 \
+ && echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen \
+ && locale-gen \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+# Users with other locales should set this in their derivative image
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
+RUN apt-get update \
+ && apt-get install -y curl unzip \
     python3 python3-setuptools \
-    python  python-setuptools \
-  && easy_install3 pip py4j \
-  && easy_install  pip py4j \
-  && apt-get clean \
-  && rm -rf /var/lib/apt/lists/*
+ && easy_install3 pip py4j \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+# http://blog.stuart.axelbrooke.com/python-3-on-spark-return-of-the-pythonhashseed
+ENV PYTHONHASHSEED 0
+ENV PYTHONIOENCODING UTF-8
 
 # JAVA
 ENV JAVA_HOME /usr/jdk1.8.0_31
@@ -21,23 +38,30 @@ RUN curl -sL --retry 3 --insecure \
   && ln -s $JAVA_HOME /usr/java \
   && rm -rf $JAVA_HOME/man
 
+# HADOOP
+ENV HADOOP_VERSION 2.6.3
+ENV HADOOP_HOME /usr/hadoop-$HADOOP_VERSION
+ENV HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
+ENV PATH $PATH:$HADOOP_HOME/bin
+RUN curl -sL --retry 3 \
+  "http://archive.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz" \
+  | gunzip \
+  | tar -x -C /usr/ \
+ && rm -rf $HADOOP_HOME/share/doc
+
 # SPARK
 ENV SPARK_VERSION 1.6.0
-ENV HADOOP_VERSION 2.6
-ENV SPARK_PACKAGE spark-$SPARK_VERSION-bin-hadoop$HADOOP_VERSION
-ENV SPARK_HOME /usr/$SPARK_PACKAGE
+ENV SPARK_PACKAGE spark-$SPARK_VERSION-bin-without-hadoop
+ENV SPARK_HOME /usr/spark-$SPARK_VERSION
+ENV PYSPARK_PYTHON python3
+ENV SPARK_DIST_CLASSPATH="$HADOOP_HOME/etc/hadoop/*:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/hdfs/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/yarn/lib/*:$HADOOP_HOME/share/hadoop/yarn/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_HOME/share/hadoop/mapreduce/*:$HADOOP_HOME/share/hadoop/tools/lib/*"
 ENV PATH $PATH:$SPARK_HOME/bin
 RUN curl -sL --retry 3 \
   "http://d3kbcqa49mib13.cloudfront.net/$SPARK_PACKAGE.tgz" \
   | gunzip \
   | tar x -C /usr/ \
-  && ln -s $SPARK_HOME /usr/spark
-
-# HADOOP/S3
-RUN curl -sL --retry 3 "http://central.maven.org/maven2/org/apache/hadoop/hadoop-aws/2.6.0/hadoop-aws-2.6.0.jar" -o $SPARK_HOME/lib/hadoop-aws-2.6.0.jar \
- && curl -sL --retry 3 "http://central.maven.org/maven2/com/amazonaws/aws-java-sdk/1.7.14/aws-java-sdk-1.7.14.jar" -o $SPARK_HOME/lib/aws-java-sdk-1.7.14.jar \
- && curl -sL --retry 3 "http://central.maven.org/maven2/com/google/collections/google-collections/1.0/google-collections-1.0.jar" -o $SPARK_HOME/lib/google-collections-1.0.jar \
- && curl -sL --retry 3 "http://central.maven.org/maven2/joda-time/joda-time/2.8.2/joda-time-2.8.2.jar" -o $SPARK_HOME/lib/joda-time-2.8.2.jar
+  && mv /usr/$SPARK_PACKAGE $SPARK_HOME \
+  && rm -rf $SPARK_HOME/examples $SPARK_HOME/ec2
 
 WORKDIR $SPARK_HOME
 CMD ["bin/spark-class", "org.apache.spark.deploy.master.Master"]
diff --git a/README.md b/README.md
@@ -3,21 +3,36 @@
 
 A `debian:jessie` based [Spark](http://spark.apache.org) container. Use it in a standalone cluster with the accompanying `docker-compose.yml`, or as a base for more complex recipes.
 
-## example
+## docker example
+
+To run `SparkPi`, run the image with Docker:
+
+    docker run --rm -it gettyimages/spark bin/run-example SparkPi 10
+
+To start `spark-shell` with your AWS credentials:
+
+    docker run --rm -it -e "AWS_ACCESS_KEY_ID=YOURKEY" -e "AWS_SECRET_ACCESS_KEY=YOURSECRET" gettyimages/spark bin/spark-shell
+
+To do a thing with Pyspark
+
+    echo "import pyspark\nprint(pyspark.SparkContext().parallelize(range(0, 10)).count())" > count.py
+    docker run --rm -it -v $(pwd)/count.py:/count.py gettyimages/spark bin/pyspark /count.py
+
+## docker-compose example
 
 To create a standalone cluster with [docker-compose](http://docs.docker.com/compose):
 
     docker-compose up
 
-The SparkUI will be running at `http://${YOUR_DOCKER_HOST}:8080` with one worker listed. To run `spark-shell`, exec into a container:
+The SparkUI will be running at `http://${YOUR_DOCKER_HOST}:8080` with one worker listed. To run `pyspark`, exec into a container:
 
     docker exec -it dockerspark_master_1 /bin/bash
-    /usr/spark/bin/spark-shell
+    bin/pyspark
 
 To run `SparkPi`, exec into a container:
 
     docker exec -it dockerspark_master_1 /bin/bash
-    /usr/spark/bin/run-example SparkPi 10
+    bin/run-example SparkPi 10
 
 ## license
 
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,11 +1,10 @@
 master:
   image: gettyimages/spark
-  command: /usr/spark/bin/spark-class org.apache.spark.deploy.master.Master -h master
+  command: bin/spark-class org.apache.spark.deploy.master.Master -h master
   hostname: master
   environment:
     MASTER: spark://master:7077
     SPARK_CONF_DIR: /conf
-    PYSPARK_PYTHON: python3
   expose:
     - 7001
     - 7002
@@ -26,15 +25,14 @@ master:
 
 worker:
   image: gettyimages/spark
-  command: /usr/spark/bin/spark-class org.apache.spark.deploy.worker.Worker spark://master:7077
+  command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://master:7077
   hostname: worker
   environment:
     SPARK_CONF_DIR: /conf
     SPARK_WORKER_CORES: 2
     SPARK_WORKER_MEMORY: 1g
     SPARK_WORKER_PORT: 8881
     SPARK_WORKER_WEBUI_PORT: 8081
-    PYSPARK_PYTHON: python3
   links:
     - master
   expose:
