MINOR: add tutorial of Airflow docker image extending and customising

coder2j · coder2j · commit b61895a78087 · 2022-02-19T14:46:40.000+01:00
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,4 @@
+FROM apache/airflow:2.0.1
+COPY requirements.txt /requirements.txt
+RUN pip install --user --upgrade pip
+RUN pip install --no-cache-dir --user -r /requirements.txt
diff --git a/dags/dag_with_python_dependencies.py b/dags/dag_with_python_dependencies.py
@@ -0,0 +1,40 @@
+from datetime import datetime, timedelta
+
+from airflow import DAG
+from airflow.operators.python import PythonOperator
+
+
+default_args = {
+    'owner': 'coder2j',
+    'retry': 5,
+    'retry_delay': timedelta(minutes=5)
+}
+
+
+def get_sklearn():
+    import sklearn
+    print(f"sklearn with version: {sklearn.__version__} ")
+
+
+def get_matplotlib():
+    import matplotlib
+    print(f"matplotlib with version: {matplotlib.__version__}")
+
+
+with DAG(
+    default_args=default_args,
+    dag_id="dag_with_python_dependencies_v03",
+    start_date=datetime(2021, 10, 12),
+    schedule_interval='@daily'
+) as dag:
+    task1 = PythonOperator(
+        task_id='get_sklearn',
+        python_callable=get_sklearn
+    )
+    
+    task2 = PythonOperator(
+        task_id='get_matplotlib',
+        python_callable=get_matplotlib
+    )
+
+    task1 >> task2
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -39,7 +39,7 @@
 version: '3'
 x-airflow-common:
   &airflow-common
-  image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.0.1}
+  image: ${AIRFLOW_IMAGE_NAME:-extending_airflow:latest}
   environment:
     &airflow-common-env
     AIRFLOW__CORE__EXECUTOR: LocalExecutor
diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1,2 @@
+scikit-learn==0.24.2
+matplotlib==3.3.3

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+scikit-learn==0.24.2`
	`2`	`+matplotlib==3.3.3`