Created using Colab

cvelac4 · cvelac4 · commit ba71a29cae30 · 2025-03-03T22:28:06.000-08:00
diff --git a/PySpark.ipynb b/PySpark.ipynb
@@ -4,7 +4,7 @@
   "metadata": {
     "colab": {
       "provenance": [],
-      "authorship_tag": "ABX9TyNxAkUJklJhGRAUU039mAKM",
+      "authorship_tag": "ABX9TyO+U61SsUd5D9aYKQLGAMPF",
       "include_colab_link": true
     },
     "kernelspec": {
@@ -353,6 +353,135 @@
           ]
         }
       ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "#Featured Engineering"
+      ],
+      "metadata": {
+        "id": "Cb2BpKZUuy6Q"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from pyspark.sql.functions import dayofmonth, month, year, lag\n",
+        "from pyspark.sql.window import Window\n",
+        "\n",
+        "#Window\n",
+        "wind_spec = Window.orderBy('Order_Date')\n",
+        "\n",
+        "#Add Lag (previous day sales)\n",
+        "daily_sales = daily_sales.withColumn('Prev_Day_Sales', lag('Daily_Sales').over(wind_spec))\n",
+        "\n",
+        "daily_sales = daily_sales.withColumn('Day', dayofmonth(col('Order_Date')))\n",
+        "daily_sales = daily_sales.withColumn('Month', month(col('Order_Date')))\n",
+        "daily_sales = daily_sales.withColumn('Year', year(col('Order_Date')))\n",
+        "\n",
+        "#Drop NA values\n",
+        "daily_sales = daily_sales.na.drop()\n",
+        "\n",
+        "daily_sales.show(10)"
+      ],
+      "metadata": {
+        "id": "_TvbaBNVu50q",
+        "outputId": "6440540e-b667-46c3-9179-f185a70040e2",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
+      },
+      "execution_count": 30,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "+----------+-----------------+-----------------+---+-----+----+\n",
+            "|Order_Date|      Daily_Sales|   Prev_Day_Sales|Day|Month|Year|\n",
+            "+----------+-----------------+-----------------+---+-----+----+\n",
+            "|2021-01-05|           19.536|           288.06|  5|    1|2021|\n",
+            "|2021-01-06|           4407.1|           19.536|  6|    1|2021|\n",
+            "|2021-01-07|87.15799999999999|           4407.1|  7|    1|2021|\n",
+            "|2021-01-09|           40.544|87.15799999999999|  9|    1|2021|\n",
+            "|2021-01-10|            54.83|           40.544| 10|    1|2021|\n",
+            "|2021-01-11|             9.94|            54.83| 11|    1|2021|\n",
+            "|2021-01-13|         3553.795|             9.94| 13|    1|2021|\n",
+            "|2021-01-14|            61.96|         3553.795| 14|    1|2021|\n",
+            "|2021-01-15|           149.95|            61.96| 15|    1|2021|\n",
+            "|2021-01-16|          299.964|           149.95| 16|    1|2021|\n",
+            "+----------+-----------------+-----------------+---+-----+----+\n",
+            "only showing top 10 rows\n",
+            "\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "#Training and Testing"
+      ],
+      "metadata": {
+        "id": "gvQchto0yyZX"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from pyspark.ml.feature import VectorAssembler\n",
+        "\n",
+        "#Assemble the fetures into singal vector\n",
+        "feature_col = ['Day', 'Month', 'Year', 'Prev_Day_Sales']\n",
+        "assembler = VectorAssembler(inputCols = feature_col, outputCol = 'features')\n",
+        "\n",
+        "daily_sales_update = assembler.transform(daily_sales).select('features', 'Daily_Sales')\n",
+        "\n",
+        "daily_sales_update.show(10)"
+      ],
+      "metadata": {
+        "id": "wSLIMTLjy3pl",
+        "outputId": "ec1d4068-8faf-4097-b49b-3979149cc836",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        }
+      },
+      "execution_count": 31,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "+--------------------+-----------------+\n",
+            "|            features|      Daily_Sales|\n",
+            "+--------------------+-----------------+\n",
+            "|[5.0,1.0,2021.0,2...|           19.536|\n",
+            "|[6.0,1.0,2021.0,1...|           4407.1|\n",
+            "|[7.0,1.0,2021.0,4...|87.15799999999999|\n",
+            "|[9.0,1.0,2021.0,8...|           40.544|\n",
+            "|[10.0,1.0,2021.0,...|            54.83|\n",
+            "|[11.0,1.0,2021.0,...|             9.94|\n",
+            "|[13.0,1.0,2021.0,...|         3553.795|\n",
+            "|[14.0,1.0,2021.0,...|            61.96|\n",
+            "|[15.0,1.0,2021.0,...|           149.95|\n",
+            "|[16.0,1.0,2021.0,...|          299.964|\n",
+            "+--------------------+-----------------+\n",
+            "only showing top 10 rows\n",
+            "\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "train, test = daily_sales_update.randomSplit([0.8, 0.2], seed= 42)"
+      ],
+      "metadata": {
+        "id": "cVHQJ5Er06Pl"
+      },
+      "execution_count": 32,
+      "outputs": []
     }
   ]
 }
