AbyelT commited on
Commit
ae2aee0
·
1 Parent(s): 9daba69

price prediction v1

Browse files
Files changed (3) hide show
  1. .hw_api_key +1 -0
  2. app.py +167 -0
  3. requirements.txt +7 -0
.hw_api_key ADDED
@@ -0,0 +1 @@
 
 
1
+ e76DLTFJzKMcADkC.BEKfBjsL91EXI2qgX9PPOcryLnsqaXWlME4EBWIPrMH8wLLUpdcmN6H3uqZuSO7J
app.py ADDED
@@ -0,0 +1,167 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ import pandas as pd
3
+ import numpy as np
4
+ import hopsworks
5
+ import joblib
6
+ import os
7
+ import json
8
+ from entsoe import EntsoePandasClient
9
+ from datetime import datetime, timedelta, date
10
+ from pandas import json_normalize
11
+ import tensorflow as tf
12
+ from keras.layers import LSTM
13
+ from urllib.request import urlopen
14
+ from sklearn.preprocessing import LabelEncoder, StandardScaler
15
+
16
+
17
+ # from keras.layers import *
18
+ # from keras.models import Sequential
19
+ # from keras.layers import Dense
20
+ # from keras.layers import LSTM
21
+ # from keras.layers import Dropout
22
+
23
+ project = hopsworks.login()
24
+ fs = project.get_feature_store()
25
+
26
+ mr = project.get_model_registry()
27
+ model = mr.get_model("SE3_elec_price_model", version=2)
28
+ model_dir = model.download()
29
+ model = joblib.load(model_dir + "/electricity_price.pkl")
30
+
31
+ def get_price_forecast():
32
+ today, tomorrow = get_date()
33
+
34
+ df_entsoe = get_entsoe_data(today, tomorrow)
35
+
36
+ # get timestamps that temp dataset should match on
37
+ entsoe_earliest = df_entsoe["datetime"].iloc[0]
38
+ entsoe_latest = df_entsoe["datetime"].iloc[-1]
39
+ df_temp = get_temp(entsoe_earliest, entsoe_latest)
40
+
41
+ df = df_entsoe.merge(df_temp, how='inner', on='datetime')
42
+ df.set_index('datetime',inplace=True)
43
+
44
+ ## pre-process before predict
45
+ sc_x=StandardScaler()
46
+ df_scaled=sc_x.fit_transform(df)
47
+ sc_y=StandardScaler()
48
+ sc_y=sc_y.fit(df[['day_ahead_price']])
49
+
50
+ step_back=24
51
+ no_records=len(df_scaled)
52
+ no_cols=4
53
+ X_train_shape_pred=[]
54
+ for i in range(step_back,no_records):
55
+ X_train_shape_pred.append(df_scaled[i-step_back:i])
56
+ X_train_shape_pred=np.array(X_train_shape_pred)
57
+ print(X_train_shape_pred.shape)
58
+
59
+ ## predict
60
+ pred_price = model.predict(X_train_shape_pred)
61
+ final_pred=sc_y.inverse_transform(pred_price)
62
+ print(final_pred.shape)
63
+
64
+ # append time for prediction
65
+ predict_time_from = datetime.fromtimestamp(entsoe_latest / 1e3)
66
+ # calculating timestamps for the next 24 h
67
+ timestamp_list = [predict_time_from + timedelta(hours=x) for x in range(len(final_pred))]
68
+
69
+ # iterating through timestamp_list
70
+ # for i, x in enumerate(timestamp_list):
71
+ # print(x, final_pred[i])
72
+ # print(final_pred.shape)
73
+ df_prediction = pd.DataFrame(
74
+ {'Datetime': timestamp_list,
75
+ 'Price forecast [EUR/MWh]': final_pred.flatten(),
76
+ })
77
+
78
+ #df_predictions = pd.DataFrame([timestamp_list, final_pred], columns=["datetime", "Price prediction"])
79
+ # print(len(final_pred), len(timestamp_list), len(final_pred), len(final_pred[0]))
80
+
81
+ return df_prediction
82
+ #[today, temp, day_ahead_price, pred_price, total_load, total_generation]
83
+
84
+ # # Returns yesterday and tomorrows date
85
+ def get_date():
86
+ # yesterday = datetime.today() - timedelta(days=1)
87
+ # yesterday = yesterday.date().strftime('%Y%m%d')
88
+ # tomorrow = (datetime.strptime(yesterday, '%Y%m%d') + timedelta(days=2)).strftime('%Y%m%d')
89
+
90
+ date_from = datetime.now() - timedelta(days=3)
91
+ date_from = date_from.date().strftime('%Y%m%d')
92
+ date_to = (datetime.strptime(date_from, '%Y%m%d') + timedelta(days=4)).strftime('%Y%m%d')
93
+
94
+ return date_from, date_to
95
+
96
+ def get_entsoe_data(date_from, date_to):
97
+ # Client
98
+ client = EntsoePandasClient(api_key="cb3a29b2-3276-4a4c-aba3-6507120d99be")
99
+
100
+ # Date and country
101
+ start = pd.Timestamp(date_from, tz='Europe/Stockholm')
102
+ end = pd.Timestamp(date_to, tz='Europe/Stockholm')
103
+ country_code = 'SE_3'
104
+
105
+ df_day_price = client.query_day_ahead_prices(country_code, start=start,end=end)
106
+ df_generation_per_prod = client.query_generation(country_code, start=start,end=end, psr_type=None)
107
+ df_load = client.query_load(country_code, start=start,end=end)
108
+
109
+ df_entsoe = df_generation_per_prod.join(df_day_price.rename("day_ahead_price"))
110
+ df_entsoe = df_entsoe.join(df_load)
111
+
112
+ df_entsoe_clean = df_entsoe.reset_index()
113
+ df_entsoe_clean = df_entsoe_clean.rename(columns = {'index':'DateTime'})
114
+ df_entsoe_clean['DateTime'] = df_entsoe_clean.DateTime.values.astype('int64') // 10 ** 6
115
+
116
+ col_list = ["Hydro Water Reservoir", "Nuclear", "Other", "Solar", "Wind Onshore"]
117
+ df_entsoe_clean['total_generation'] = df_entsoe_clean[list(col_list)].sum(axis=1)
118
+
119
+ df_entsoe_clean.drop(col_list + ["Fossil Gas"], axis=1, inplace=True)
120
+ df_entsoe_clean.rename(columns={"Actual Load": "total_load", "DateTime":"datetime"}, inplace=True)
121
+
122
+ return df_entsoe_clean.tail(48)
123
+
124
+ def get_temp(timeseries_from, timeseries_to):
125
+
126
+ url = "https://opendata-download-metobs.smhi.se/api/version/latest/parameter/1/station/71420/period/latest-months/data.json"
127
+ response = urlopen(url)
128
+
129
+ # convert response to json, to dataframe
130
+ data_json = json.loads(response.read())
131
+ df_smhi_data = json_normalize(data_json['value'])
132
+
133
+ # extract only the temperature in the time stamp interval
134
+ df_smhi_data = df_smhi_data.loc[(df_smhi_data['date'] >= timeseries_from) & (df_smhi_data['date'] <= timeseries_to)]
135
+ df_smhi_data = df_smhi_data.reset_index().rename(columns = {'date':'datetime'})
136
+
137
+ df_smhi_data.drop(["index", "quality"], axis=1, inplace=True)
138
+ df_smhi_data["value"] = df_smhi_data["value"].astype(float)
139
+ df_smhi_data.rename(columns={"value": "temperature"}, inplace=True)
140
+
141
+ return df_smhi_data
142
+
143
+ demo = gr.Interface(
144
+ fn = get_price_forecast,
145
+ title = "SE3 Electricity Day-Ahead Price Prediction",
146
+ description ="SE3 Electricity Day-Ahead Price Prediction, based on electricity production, generation and temperature",
147
+ allow_flagging = "never",
148
+ inputs = [],
149
+ outputs = [
150
+ gr.DataFrame(x="datetime", y="Price prediction [EUR/MWh]")
151
+ # gr.Textbox(label="Date"),
152
+ # gr.Textbox(label="Temperature Forecast [℃]"),
153
+ # gr.Textbox(label="Total Load Forecast [MWh]"),
154
+ # gr.Textbox(label="Total Generation Forecast [MWh]"),
155
+ # gr.Textbox(label="Predicted Day-Ahead Price [EUR/MWh]"),
156
+ ]
157
+ )
158
+
159
+ demo.launch()
160
+
161
+
162
+ # TODO: we have only the demand predictions for two days ago, so we have two options
163
+ # - skip EIA demand forecast (no comparison)
164
+ # - show prediction for two days ago
165
+ # TODO: allow custom date/temp input (default to today)?
166
+ # TODO: have done some versioning mess (see reqs file)
167
+
requirements.txt ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ hopsworks
2
+ joblib
3
+ tensorflow
4
+ pandas
5
+ keras
6
+ entsoe
7
+ scikit-learn==1.0.2