Upload 2 files

app.py

@@ -0,0 +1,71 @@
+import streamlit as st
+import hopsworks
+import joblib
+import pandas as pd
+import numpy as np
+from datetime import timedelta, datetime
+
+from functions import *
+
+
+def fancy_header(text, font_size=24):
+    res = f'<span style="color:#ff5f27; font-size: {font_size}px;">{text}</span>'
+    st.markdown(res, unsafe_allow_html=True )
+
+
+st.title('Air Quality Prediction Project🌩')
+
+progress_bar = st.sidebar.header('Working Progress')
+progress_bar = st.sidebar.progress(0)
+st.write(36 * "-")
+fancy_header('\n Connecting to Hopsworks Feature Store...')
+
+project = hopsworks.login()
+fs = project.get_feature_store()
+feature_view = fs.get_feature_view(
+    name = 'air_quality_fv',
+    version = 1
+)
+
+st.write("Successfully connected!✔️")
+progress_bar.progress(20)
+
+st.write(36 * "-")
+fancy_header('\n Getting data from Feature Store...')
+
+today = datetime.date.today()
+city = "vienna"
+weekly_data = get_weather_data_weekly(city, today)
+
+progress_bar.progress(50)
+
+#latest_date_unix = str(X.date.values[0])[:10]
+#latest_date = time.ctime(int(latest_date_unix))
+
+#st.write(f"Data for {latest_date}")
+
+#data_to_display = decode_features(X, feature_view=feature_view)
+
+progress_bar.progress(60)
+
+st.write(36 * "-")
+
+mr = project.get_model_registry()
+model = mr.get_best_model("aqi_model", "rmse", "min")
+model_dir = model.download()
+model = joblib.load(model_dir + "/aqi_model.pkl")
+
+progress_bar.progress(80)
+st.sidebar.write("-" * 36)
+
+
+preds = model.predict(data_encoder(weekly_data)).astype(int)
+poll_level = get_aplevel(preds.T.reshape(-1, 1))
+
+next_week = [(datetime.today() + timedelta(days=d)).strftime('%A') for d in range(1, 7)]
+
+df = pd.DataFrame(data=preds, index=["eg"], columns=[f"AQI Predictions for {next_day}" for next_day in next_week], dtype=int)
+
+st.sidebar.write(df)
+progress_bar.progress(100)
+st.button("Re-run")

functions.py

@@ -0,0 +1,207 @@
+import requests
+import os
+import joblib
+import pandas as pd
+import datetime
+import numpy as np
+from sklearn.preprocessing import OrdinalEncoder
+from dotenv import load_dotenv
+load_dotenv(override=True)
+
+
+def decode_features(df, feature_view):
+    """Decodes features in the input DataFrame using corresponding Hopsworks Feature Store transformation functions"""
+    df_res = df.copy()
+
+    import inspect
+
+
+    td_transformation_functions = feature_view._batch_scoring_server._transformation_functions
+
+    res = {}
+    for feature_name in td_transformation_functions:
+        if feature_name in df_res.columns:
+            td_transformation_function = td_transformation_functions[feature_name]
+            sig, foobar_locals = inspect.signature(td_transformation_function.transformation_fn), locals()
+            param_dict = dict([(param.name, param.default) for param in sig.parameters.values() if param.default != inspect._empty])
+            if td_transformation_function.name == "min_max_scaler":
+                df_res[feature_name] = df_res[feature_name].map(
+                    lambda x: x * (param_dict["max_value"] - param_dict["min_value"]) + param_dict["min_value"])
+
+            elif td_transformation_function.name == "standard_scaler":
+                df_res[feature_name] = df_res[feature_name].map(
+                    lambda x: x * param_dict['std_dev'] + param_dict["mean"])
+            elif td_transformation_function.name == "label_encoder":
+                dictionary = param_dict['value_to_index']
+                dictionary_ = {v: k for k, v in dictionary.items()}
+                df_res[feature_name] = df_res[feature_name].map(
+                    lambda x: dictionary_[x])
+    return df_res
+
+
+def get_model(project, model_name, evaluation_metric, sort_metrics_by):
+    """Retrieve desired model or download it from the Hopsworks Model Registry.
+    In second case, it will be physically downloaded to this directory"""
+    TARGET_FILE = "model.pkl"
+    list_of_files = [os.path.join(dirpath,filename) for dirpath, _, filenames \
+                     in os.walk('.') for filename in filenames if filename == TARGET_FILE]
+
+    if list_of_files:
+        model_path = list_of_files[0]
+        model = joblib.load(model_path)
+    else:
+        if not os.path.exists(TARGET_FILE):
+            mr = project.get_model_registry()
+            # get best model based on custom metrics
+            model = mr.get_best_model(model_name,
+                                      evaluation_metric,
+                                      sort_metrics_by)
+            model_dir = model.download()
+            model = joblib.load(model_dir + "/model.pkl")
+
+    return model
+
+
+def get_air_quality_data(station_name):
+    AIR_QUALITY_API_KEY = os.getenv('AIR_QUALITY_API_KEY')
+    request_value = f'https://api.waqi.info/feed/{station_name}/?token={AIR_QUALITY_API_KEY}'
+    answer = requests.get(request_value).json()["data"]
+    forecast = answer['forecast']['daily']
+    return [
+        answer["time"]["s"][:10],      # Date
+        int(forecast['pm25'][0]['avg']),  # avg predicted pm25
+        int(forecast['pm10'][0]['avg']),  # avg predicted pm10
+        max(int(forecast['pm25'][0]['avg']), int(forecast['pm10'][0]['avg'])) # avg predicted aqi
+    ]
+
+def get_air_quality_df(data):
+    col_names = [
+        'date',
+        'pm25',
+        'pm10',
+        'aqi'
+    ]
+
+    new_data = pd.DataFrame(
+        data
+    ).T
+    new_data.columns = col_names
+    new_data['pm25'] = pd.to_numeric(new_data['pm25'])
+    new_data['pm10'] = pd.to_numeric(new_data['pm10'])
+    new_data['aqi'] = pd.to_numeric(new_data['aqi'])
+
+    print(new_data)
+    return new_data
+
+
+def get_weather_data_daily(city):
+    WEATHER_API_KEY = os.getenv('WEATHER_API_KEY')
+    answer = requests.get(f'https://weather.visualcrossing.com/VisualCrossingWebServices/rest/services/timeline/{city}/today?unitGroup=metric&include=days&key={WEATHER_API_KEY}&contentType=json').json()
+    data = answer['days'][0]
+    return [
+        answer['address'].lower(),
+        data['datetime'],
+        data['tempmax'],
+        data['tempmin'],
+        data['temp'],
+        data['feelslikemax'],
+        data['feelslikemin'],
+        data['feelslike'],
+        data['dew'],
+        data['humidity'],
+        data['precip'],
+        data['precipprob'],
+        data['precipcover'],
+        data['snow'],
+        data['snowdepth'],
+        data['windgust'],
+        data['windspeed'],
+        data['winddir'],
+        data['pressure'],
+        data['cloudcover'],
+        data['visibility'],
+        data['solarradiation'],
+        data['solarenergy'],
+        data['uvindex'],
+        data['conditions']
+    ]
+def get_weather_data_weekly(city: str, start_date: datetime) -> pd.DataFrame:
+    WEATHER_API_KEY = os.getenv('WEATHER_API_KEY')
+    end_date = f"{start_date + datetime.timedelta(days=6):%Y-%m-%d}"
+    answer = requests.get(f'https://weather.visualcrossing.com/VisualCrossingWebServices/rest/services/timeline/{city}/{start_date}/{end_date}?unitGroup=metric&include=days&key={WEATHER_API_KEY}&contentType=json').json()
+    weather_data = answer['days']
+    final_df = pd.DataFrame()
+
+    for i in range(7):
+        data = weather_data[i]
+        list_of_data = [
+        answer['address'].lower(), data['datetime'], data['tempmax'], data['tempmin'], data['temp'], data['feelslikemax'],
+        data['feelslikemin'], data['feelslike'], data['dew'], data['humidity'], data['precip'], data['precipprob'], data['precipcover'],
+        data['snow'], data['snowdepth'], data['windgust'], data['windspeed'], data['winddir'], data['pressure'], data['cloudcover'],
+        data['visibility'], data['solarradiation'], data['solarenergy'], data['uvindex'], data['conditions']
+    ]
+        weather_df = get_weather_df(list_of_data)
+        final_df = pd.concat([final_df, weather_df])
+    return final_df
+
+def get_weather_df(data):
+    col_names = [
+        'name',
+        'date',
+        'tempmax',
+        'tempmin',
+        'temp',
+        'feelslikemax',
+        'feelslikemin',
+        'feelslike',
+        'dew',
+        'humidity',
+        'precip',
+        'precipprob',
+        'precipcover',
+        'snow',
+        'snowdepth',
+        'windgust',
+        'windspeed',
+        'winddir',
+        'pressure',
+        'cloudcover',
+        'visibility',
+        'solarradiation',
+        'solarenergy',
+        'uvindex',
+        'conditions'
+    ]
+
+    new_data = pd.DataFrame(
+        data
+    ).T
+    new_data.columns = col_names
+    for col in col_names:
+        if col not in ['name', 'date', 'conditions']:
+            new_data[col] = pd.to_numeric(new_data[col])
+
+    return new_data
+
+def data_encoder(X):
+    X.drop(columns=['date', 'name'], inplace=True)
+    X['conditions'] = OrdinalEncoder().fit_transform(X[['conditions']])
+    return X
+
+def transform(df):
+    df.loc[df["windgust"].isna(),'windgust'] = df['windspeed']
+    df['snow'].fillna(0,inplace=True)
+    df['snowdepth'].fillna(0, inplace=True)
+    df['pressure'].fillna(df['pressure'].mean(), inplace=True)
+    return df
+
+
+def get_aplevel(temps:np.ndarray) -> list:
+    boundary_list = np.array([0, 50, 100, 150, 200, 300]) # assert temps.shape == [x, 1]
+    redf = np.logical_not(temps<=boundary_list) # temps.shape[0] x boundary_list.shape[0] ndarray
+    hift = np.concatenate((np.roll(redf, -1)[:, :-1], np.full((temps.shape[0], 1), False)), axis = 1)
+    cat = np.nonzero(np.not_equal(redf,hift))
+
+    air_pollution_level = ['Good', 'Moderate', 'Unhealthy for sensitive Groups','Unhealthy' ,'Very Unhealthy', 'Hazardous']
+    level = [air_pollution_level[el] for el in cat[1]]
+    return level