From 8b9618d9fdc17d8d9d0f6e79488d67829f12b68e Mon Sep 17 00:00:00 2001 From: RamiKrispin Date: Wed, 30 Oct 2024 12:11:05 +0000 Subject: [PATCH] Auto update of the data --- csv/ciso_grid_py.csv | 48 +++ docs/data_refresh_python/data_refresh_py.html | 302 +++++++++--------- docs/index.html | 224 ++++++------- metadata/ciso_log_py.csv | 8 +- 4 files changed, 315 insertions(+), 267 deletions(-) diff --git a/csv/ciso_grid_py.csv b/csv/ciso_grid_py.csv index b33eaedee..80e9397ab 100644 --- a/csv/ciso_grid_py.csv +++ b/csv/ciso_grid_py.csv @@ -218807,3 +218807,51 @@ period,subba,subba-name,parent,parent-name,value,value-units 2024-10-29 05:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,60.0,megawatthours 2024-10-29 06:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,56.0,megawatthours 2024-10-29 07:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,53.0,megawatthours +2024-10-29 08:00:00,SDGE,San Diego Gas and Electric - CISO,CISO,California Independent System Operator,2059.0,megawatthours +2024-10-29 09:00:00,SDGE,San Diego Gas and Electric - CISO,CISO,California Independent System Operator,2122.0,megawatthours +2024-10-29 10:00:00,SDGE,San Diego Gas and Electric - CISO,CISO,California Independent System Operator,2027.0,megawatthours +2024-10-29 11:00:00,SDGE,San Diego Gas and Electric - CISO,CISO,California Independent System Operator,1943.0,megawatthours +2024-10-29 12:00:00,SDGE,San Diego Gas and Electric - CISO,CISO,California Independent System Operator,1927.0,megawatthours +2024-10-29 13:00:00,SDGE,San Diego Gas and Electric - CISO,CISO,California Independent System Operator,1904.0,megawatthours +2024-10-29 14:00:00,SDGE,San Diego Gas and Electric - CISO,CISO,California Independent System Operator,1979.0,megawatthours +2024-10-29 15:00:00,SDGE,San Diego Gas and Electric - CISO,CISO,California Independent System Operator,2149.0,megawatthours +2024-10-29 16:00:00,SDGE,San Diego Gas and Electric - CISO,CISO,California Independent System Operator,2299.0,megawatthours +2024-10-29 17:00:00,SDGE,San Diego Gas and Electric - CISO,CISO,California Independent System Operator,2314.0,megawatthours +2024-10-29 18:00:00,SDGE,San Diego Gas and Electric - CISO,CISO,California Independent System Operator,2260.0,megawatthours +2024-10-29 19:00:00,SDGE,San Diego Gas and Electric - CISO,CISO,California Independent System Operator,2147.0,megawatthours +2024-10-29 20:00:00,SDGE,San Diego Gas and Electric - CISO,CISO,California Independent System Operator,1762.0,megawatthours +2024-10-29 21:00:00,SDGE,San Diego Gas and Electric - CISO,CISO,California Independent System Operator,1622.0,megawatthours +2024-10-29 22:00:00,SDGE,San Diego Gas and Electric - CISO,CISO,California Independent System Operator,1635.0,megawatthours +2024-10-29 23:00:00,SDGE,San Diego Gas and Electric - CISO,CISO,California Independent System Operator,1635.0,megawatthours +2024-10-30 00:00:00,SDGE,San Diego Gas and Electric - CISO,CISO,California Independent System Operator,1671.0,megawatthours +2024-10-30 01:00:00,SDGE,San Diego Gas and Electric - CISO,CISO,California Independent System Operator,2056.0,megawatthours +2024-10-30 02:00:00,SDGE,San Diego Gas and Electric - CISO,CISO,California Independent System Operator,2474.0,megawatthours +2024-10-30 03:00:00,SDGE,San Diego Gas and Electric - CISO,CISO,California Independent System Operator,2704.0,megawatthours +2024-10-30 04:00:00,SDGE,San Diego Gas and Electric - CISO,CISO,California Independent System Operator,2681.0,megawatthours +2024-10-30 05:00:00,SDGE,San Diego Gas and Electric - CISO,CISO,California Independent System Operator,2453.0,megawatthours +2024-10-30 06:00:00,SDGE,San Diego Gas and Electric - CISO,CISO,California Independent System Operator,2342.0,megawatthours +2024-10-30 07:00:00,SDGE,San Diego Gas and Electric - CISO,CISO,California Independent System Operator,2181.0,megawatthours +2024-10-29 08:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,50.0,megawatthours +2024-10-29 09:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,48.0,megawatthours +2024-10-29 10:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,46.0,megawatthours +2024-10-29 11:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,46.0,megawatthours +2024-10-29 12:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,46.0,megawatthours +2024-10-29 13:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,48.0,megawatthours +2024-10-29 14:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,53.0,megawatthours +2024-10-29 15:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,59.0,megawatthours +2024-10-29 16:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,63.0,megawatthours +2024-10-29 17:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,113.0,megawatthours +2024-10-29 18:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,166.0,megawatthours +2024-10-29 19:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,162.0,megawatthours +2024-10-29 20:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,187.0,megawatthours +2024-10-29 21:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,172.0,megawatthours +2024-10-29 22:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,142.0,megawatthours +2024-10-29 23:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,75.0,megawatthours +2024-10-30 00:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,55.0,megawatthours +2024-10-30 01:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,55.0,megawatthours +2024-10-30 02:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,60.0,megawatthours +2024-10-30 03:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,65.0,megawatthours +2024-10-30 04:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,64.0,megawatthours +2024-10-30 05:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,61.0,megawatthours +2024-10-30 06:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,58.0,megawatthours +2024-10-30 07:00:00,VEA,Valley Electric Association - CISO,CISO,California Independent System Operator,55.0,megawatthours diff --git a/docs/data_refresh_python/data_refresh_py.html b/docs/data_refresh_python/data_refresh_py.html index 0ef45fa57..f7fbb348d 100644 --- a/docs/data_refresh_python/data_refresh_py.html +++ b/docs/data_refresh_python/data_refresh_py.html @@ -117,7 +117,7 @@

EIA API - Data Refrsh (Python Version)

Load libraries

-
+
import eia_api as api
 import eia_data 
 import pandas as pd
@@ -130,7 +130,7 @@ 
Load libraries

 import great_tables as gt

API Settings:

-
+
raw_json = open("../metadata/series.json")
 meta_json = json.load(raw_json)
 series = pd.DataFrame(meta_json["series"])
@@ -148,7 +148,7 @@ 
Load libraries

 meta_path = "../metadata/ciso_log_py.csv"
 data_path = "../csv/ciso_grid_py.csv"
-
+
# api_metadata = api.eia_metadata(api_key = eia_api_key, api_path = api_path)
 
 
@@ -156,55 +156,55 @@ 
Load libraries

 # end = pd.to_datetime(api_metadata.meta["endPeriod"])
 # print(end)
-
+
meta_obj = eia_data.get_metadata(api_key = eia_api_key, api_path = api_path, meta_path = meta_path, series = series)
 
 gt.GT(meta_obj.request_meta,)
-
+
@@ -225,7 +225,7 @@

Load libraries

PGAE 2024-08-26 07:00:00 2024-08-26 08:00:00 -2024-10-29 07:00:00 +2024-10-30 07:00:00 True @@ -233,7 +233,7 @@

Load libraries

SCE 2024-08-23 07:00:00 2024-08-23 08:00:00 -2024-10-29 07:00:00 +2024-10-30 07:00:00 True @@ -241,16 +241,16 @@

Load libraries

SDGE 2024-10-29 07:00:00 2024-10-29 08:00:00 -2024-10-29 07:00:00 -False +2024-10-30 07:00:00 +True CISO VEA 2024-10-29 07:00:00 2024-10-29 08:00:00 -2024-10-29 07:00:00 -False +2024-10-30 07:00:00 +True @@ -260,7 +260,7 @@

Load libraries

-
+
m = meta_obj.request_meta
 index = meta_obj.last_index + 1
 data = None
@@ -321,64 +321,60 @@ 
Load libraries

 
{'parent': 'CISO', 'subba': 'PGAE'}
 {'parent': 'CISO', 'subba': 'SCE'}
 {'parent': 'CISO', 'subba': 'SDGE'}
-No new data is available
+Append the new data
+Save the data to CSV file
 {'parent': 'CISO', 'subba': 'VEA'}
-No new data is available

-
-
-
/tmp/ipykernel_144/2250529845.py:56: FutureWarning:
-
-The behavior of DataFrame concatenation with empty or all-NA entries is deprecated. In a future version, this will no longer exclude empty or all-NA columns when determining the result dtypes. To retain the old behavior, exclude the relevant entries before the concat operation.
-
+Append the new data +Save the data to CSV file
-
-
gt.GT(meta_new,rowname_col = "index")
+
+
gt.GT(meta_new,rowname_col = "index")
-
+
@@ -392,8 +388,8 @@

Load libraries

end start_act end_act -start_match -end_match +start_match +end_match n_obs na type @@ -408,14 +404,14 @@

Load libraries

CISO PGAE -2024-10-30 00:25:42.449231+00:00 +2024-10-30 12:10:45.032488+00:00 2024-08-26 08:00:00 -2024-10-29 07:00:00 +2024-10-30 07:00:00 2024-08-26 08:00:00 -2024-10-29 07:00:00 -True -True -1536 +2024-10-30 07:00:00 +True +True +1560 5 refresh False @@ -426,14 +422,14 @@

Load libraries

CISO SCE -2024-10-30 00:25:43.505746+00:00 +2024-10-30 12:10:46.398644+00:00 2024-08-23 08:00:00 -2024-10-29 07:00:00 +2024-10-30 07:00:00 2024-08-23 08:00:00 -2024-10-29 07:00:00 -True -True -1608 +2024-10-30 07:00:00 +True +True +1632 14 refresh False @@ -444,37 +440,37 @@

Load libraries

CISO SDGE -2024-10-30 00:25:43.508390+00:00 +2024-10-30 12:10:47.515482+00:00 2024-10-29 08:00:00 -2024-10-29 07:00:00 - - - - - - +2024-10-30 07:00:00 +2024-10-29 08:00:00 +2024-10-30 07:00:00 +True +True +24 +0 refresh -False -False -No new data is available; The data refresh failed, please check the log; +True +True + CISO VEA -2024-10-30 00:25:43.510200+00:00 +2024-10-30 12:10:49.033546+00:00 2024-10-29 08:00:00 -2024-10-29 07:00:00 - - - - - - +2024-10-30 07:00:00 +2024-10-29 08:00:00 +2024-10-30 07:00:00 +True +True +24 +0 refresh -False -False -No new data is available; The data refresh failed, please check the log; +True +True + @@ -484,23 +480,23 @@

Load libraries

-
-
meta_updated = eia_data.append_metadata(meta_path = meta_path, meta = meta_new, save = True, init = False)
+
+
meta_updated = eia_data.append_metadata(meta_path = meta_path, meta = meta_new, save = True, init = False)

Plot the Series

We will use Plotly to visualize the series:

-
-
if data is not None:
-    d = data.sort_values(by = ["subba", "period"])
-    p = px.line(data, x="period", y="value", color="subba")
-    p.show()
-else: 
-    print("No new data is available")
+
+
if data is not None:
+    d = data.sort_values(by = ["subba", "period"])
+    p = px.line(data, x="period", y="value", color="subba")
+    p.show()
+else: 
+    print("No new data is available")
-
-
-
full_data = pd.read_csv(data_path)
-full_data.head()
-
-full_data["period"] = pd.to_datetime(full_data["period"])
-p = px.line(full_data, x="period", y="value", color="subba")
-p.show()
+
+
full_data = pd.read_csv(data_path)
+full_data.head()
+
+full_data["period"] = pd.to_datetime(full_data["period"])
+p = px.line(full_data, x="period", y="value", color="subba")
+p.show()
-