Configure Minio/S3

fhueske · fhueske · commit 4732ca7ff5eb · 2020-04-03T00:44:00.000+02:00
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -20,7 +20,8 @@ services:
     build: ./data-image
     command: "java -classpath /opt/data/data-producer.jar com.ververica.sql_training.data_producer.TaxiRecordProducer --input file /opt/data --output kafka kafka:9092 --speedup 10.0"
   jobmanager:
-    image: flink:1.10.0-scala_2.11
+    image: fhueske/flink-s3:1.10.0-scala_2.11
+    build: ./flink-image
     hostname: "jobmanager"
     expose:
       - "6123"
@@ -30,7 +31,8 @@ services:
     environment:
       - JOB_MANAGER_RPC_ADDRESS=jobmanager
   taskmanager:
-    image: flink:1.10.0-scala_2.11
+    image: fhueske/flink-s3:1.10.0-scala_2.11
+    build: ./flink-image
     expose:
       - "6121"
       - "6122"
@@ -48,7 +50,8 @@ services:
       - "9083"
   minio:
     image: minio/minio:latest
-    command: server /data
+    entrypoint: sh
+    command: -c 'mkdir -p /data/sql-demo && /usr/bin/minio server /data'
     environment:
       - MINIO_ACCESS_KEY=sql-demo
       - MINIO_SECRET_KEY=demo-sql
diff --git a/flink-image/Dockerfile b/flink-image/Dockerfile
@@ -0,0 +1,32 @@
+###############################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+###############################################################################
+# Build Flink images with configured S3 plugin
+###############################################################################
+
+FROM flink:1.10.0-scala_2.11
+
+RUN mkdir /opt/flink/plugins/s3; \
+    mv /opt/flink/opt/flink-s3-fs-hadoop-1.10.0.jar /opt/flink/plugins/s3; \
+    echo "s3.access-key: sql-demo" >> /opt/flink/conf/flink-conf.yaml; \
+    echo "s3.secret-key: demo-sql" >> /opt/flink/conf/flink-conf.yaml; \
+    echo "fs.s3a.access.key: sql-demo" >> /opt/flink/conf/flink-conf.yaml; \
+    echo "fs.s3a.secret.key: demo-sql" >> /opt/flink/conf/flink-conf.yaml; \
+    echo "fs.s3a.endpoint: http://minio:9000" >> /opt/flink/conf/flink-conf.yaml; \
+    echo "fs.s3a.path.style.access: true" >> /opt/flink/conf/flink-conf.yaml;
