update endpoint for gas prices dataset (GoogleCloudPlatform#8376)

leahecole · web-flow · commit e8bb8a4ad4b1 · 2022-10-04T10:47:17.000-04:00
diff --git a/data-science-onramp/data-ingestion/ingestion.py b/data-science-onramp/data-ingestion/ingestion.py
@@ -35,19 +35,27 @@
 CITIBIKE_TABLE_NAME = "RAW_DATA"
 EXTERNAL_TABLES = {
     "gas_prices": {
-        "url": "https://data.ny.gov/api/views/wuxr-ni2i/rows.csv",
+        "url": "https://data.ny.gov/api/views/nqur-w4p7/rows.csv",
         "schema": StructType(
             [
                 StructField("Date", StringType(), True),
                 StructField("New_York_State_Average_USD_per_Gal", FloatType(), True),
                 StructField("Albany_Average_USD_per_Gal", FloatType(), True),
-                StructField("Blinghamton_Average_USD_per_Gal", FloatType(), True),
+                StructField("Batavia_Average_USD_per_Gal", FloatType(), True),
+                StructField("Binghamton_Average_USD_per_Gal", FloatType(), True),
                 StructField("Buffalo_Average_USD_per_Gal", FloatType(), True),
+                StructField("Dutchess_Average_USD_per_Gal", FloatType(), True),
+                StructField("Elmira_Average_USD_per_Gal", FloatType(), True),
+                StructField("Glens_Falls_Average_USD_per_Gal", FloatType(), True),
+                StructField("Ithaca_Average_USD_per_Gal", FloatType(), True),
+                StructField("Kingston_Average_USD_per_Gal", FloatType(), True),
                 StructField("Nassau_Average_USD_per_Gal", FloatType(), True),
                 StructField("New_York_City_Average_USD_per_Gal", FloatType(), True),
                 StructField("Rochester_Average_USD_per_Gal", FloatType(), True),
                 StructField("Syracuse_Average_USD_per_Gal", FloatType(), True),
                 StructField("Utica_Average_USD_per_Gal", FloatType(), True),
+                StructField("Watertown_Average_USD_per_Gal", FloatType(), True),
+                StructField("White_Plains_Average_USD_per_Gal", FloatType(), True),
             ]
         ),
     },
diff --git a/data-science-onramp/feature-engineering/feature_engineering.ipynb b/data-science-onramp/feature-engineering/feature_engineering.ipynb
@@ -76,13 +76,22 @@
     "    bigquery.SchemaField(\"Date\", \"STRING\"),\n",
     "    bigquery.SchemaField(\"New_York_State_Average_USD_per_Gal\", \"FLOAT\"),\n",
     "    bigquery.SchemaField(\"Albany_Average_USD_per_Gal\", \"FLOAT\"),\n",
+    "    bigquery.SchemaField(\"Batavia_Average_USD_per_Gal\", \"FLOAT\"),\n",
     "    bigquery.SchemaField(\"Binghamton_Average_USD_per_Gal\", \"FLOAT\"),\n",
     "    bigquery.SchemaField(\"Buffalo_Average_USD_per_Gal\", \"FLOAT\"),\n",
+    "    bigquery.SchemaField(\"Dutchess_Average_USD_per_Gal\"),\n",
+    "    bigquery.SchemaField(\"Elmira_Average_USD_per_Gal\", \"FLOAT\"),\n",
+    "    bigquery.SchemaField(\"Glens_Falls_Average_USD_per_Gal\", \"FLOAT\"),\n",
+    "    bigquery.SchemaField(\"Ithaca_Average_USD_per_Gal\", \"FLOAT\"),\n",
+    "    bigquery.SchemaField(\"Kingston_Average_USD_per_Gal\", \"FLOAT\"),\n",
     "    bigquery.SchemaField(\"Nassau_Average_USD_per_Gal\", \"FLOAT\"),\n",
     "    bigquery.SchemaField(\"New_York_City_Average_USD_per_Gal\", \"FLOAT\"),\n",
     "    bigquery.SchemaField(\"Rochester_Average_USD_per_Gal\", \"FLOAT\"),\n",
     "    bigquery.SchemaField(\"Syracuse_Average_USD_per_Gal\", \"FLOAT\"),\n",
     "    bigquery.SchemaField(\"Utica_Average_USD_per_Gal\", \"FLOAT\"),\n",
+    "    bigquery.SchemaField(\"Watertown_Average_USD_per_Gal\", \"FLOAT\"),\n",
+    "    bigquery.SchemaField(\"White_Plains_Average_USD_per_Gal\", \"FLOAT\"),\n",
+    "\n",
     "]\n",
     "write_to_bq('gas_prices', gasprices_uri, gasprices_schema)"
    ]
@@ -1003,16 +1012,19 @@
    "type": "gcloud",
    "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-3:mnightly-2021-02-12-debian-10-test"
   },
-  "interpreter": {
-   "hash": "1b0ec9b1916cf0489ce214584e3f7f99b183b01cc4dc6fb00e756ec75ed889c4"
-  },
   "kernelspec": {
-   "display_name": "Python 3.8.0 64-bit ('3.8.0')",
+   "display_name": "Python 3.8.13 64-bit ('3.8.13')",
+   "language": "python",
    "name": "python3"
   },
   "language_info": {
    "name": "python",
-   "version": ""
+   "version": "3.8.13"
+  },
+  "vscode": {
+   "interpreter": {
+    "hash": "26864ca88ce905de82797da54d8873376f137ee19eedffe0cbf7a315f936dc83"
+   }
   }
  },
  "nbformat": 4,
diff --git a/data-science-onramp/feature-engineering/testing_data/gasprices.csv b/data-science-onramp/feature-engineering/testing_data/gasprices.csv
