Added more features.

app.py

@@ -2,23 +2,30 @@ import streamlit as st
 import pandas as pd
 from modules.data_preparation import prepare_df, plot_3dgraph
 import numpy as np
-import datetime
+from datetime import datetime
 
 st.title('Sentiment Analysis for Price Trend Prediction')
 
 st.header(f'Data based on Platts News and Insights Data')
-st.subheader(f'{datetime.datetime.now()}')
-
-# news_category = st.selectbox("Select Market Movers Category", ("Crude Oil", "Light Ends", "Middle Distillates", "Heavy Distillates"))
+st.subheader(f'{datetime.now()}')
 
 news_categories = st.multiselect("Select desired Market Movers categories", 
                                  ["Macroeconomic & Geopolitics", "Crude Oil", "Light Ends", "Middle Distillates", "Heavy Distillates", "Other"],
                                  ["Macroeconomic & Geopolitics", "Crude Oil"])
 
-latest_news = prepare_df(pd.read_csv('data/results_platts_09082024_clean.csv'), news_categories)
-top_news = prepare_df(pd.read_csv('data/topresults_platts_09082024_clean.csv'), news_categories)
+# date_filter = st.slider(
+#     "Date Filter",
+#     value=(datetime(2024, 8, 4), datetime(2024,8,9)),
+#     format="MM/DD/YY",
+# )
+
+#latest_news = prepare_df(pd.read_csv('data/results_platts_09082024_clean.csv'), news_categories)
+#top_news = prepare_df(pd.read_csv('data/topresults_platts_09082024_clean.csv'), news_categories)
+
+#df_news = pd.concat([latest_news, top_news], ignore_index=True).drop_duplicates(['headline'])
 
-df_news = pd.concat([latest_news, top_news], ignore_index=True).drop_duplicates(['headline'])
+latest_news = prepare_df(pd.read_excel('evaluation.xlsx'), news_categories, date_filter)
+df_news = pd.concat([latest_news], ignore_index=True).drop_duplicates(['headline'])
 
 df_mean = pd.DataFrame({
     'headline' : ['MEAN OF SELECTED NEWS'],
@@ -32,7 +39,7 @@ df_news_final = pd.concat([df_news, df_mean])
 
 df_news_final.index = np.arange(1, len(df_news_final) + 1)
 
-df_news_final
+#df_news_final
 
 st.markdown('---')
 

convert_first.csv

@@ -0,0 +1,150 @@
+,headline,topic_verification
+0,SPAIN DATA: H1 crude imports rise 11% to 1.4 million b/d,Crude Oil
+1,REFINERY NEWS: Host of Chinese units back from works; Jinling maintenance in Nov-Dec,Macroeconomic & Geopolitics
+2,REFINERY NEWS ROUNDUP: Mixed runs in Asia-Pacific,Macroeconomic & Geopolitics
+3,Physical 1%S fuel oil Med-North spread hits record high on competitive bidding in Platts MOC,Middle Distillates
+4,"Indian ports see Jan-July bunker, STS calls up 64% on year, monsoon hits July demand",Macroeconomic & Geopolitics
+5,LNG bunker prices in Europe hit 8-month high amid rising demand,Light Ends
+6,REFINERY NEWS: Wellbred Trading acquires La Nivernaise de Raffinage in France,Heavy Distillates
+7,Wellbred Trading buys French diesel refinery that runs on used cooking oil,Macroeconomic & Geopolitics
+8,EU's climate monitor says 2024 'increasingly likely' to be warmest year on record,Macroeconomic & Geopolitics
+9,"European LPG discount to naphtha narrows, shifting petchem feedstock appetite",Light Ends
+10,REFINERY NEWS: Thai Oil’s Q2 utilization drops on planned CDU shutdown,Crude Oil
+11,South Korea’s top oil refiner SK Innovation joins carbon storage project in Australia,Middle Distillates
+12,CRUDE MOC: Middle East sour crude cash differentials hit month-to-date highs,Crude Oil
+13,CNOOC approves 100 Bcm of proven reserves at South China Sea gas field,Crude Oil
+14,"Singapore to work with Shell’s refinery, petrochemicals asset buyers to decarbonize: minister",Light Ends
+15,BLM federal Montana-Dakotas oil and gas lease sale nets nearly $24 mil: Energynet.com,Middle Distillates
+16,REFINERY NEWS: Oman's Sohar undergoes unplanned shutdown: sources,Crude Oil
+17,"OIL FUTURES: Crude prices higher as US stockpiles extend decline, demand concerns cap gains",Crude Oil
+18,Qatar announces acceptance of Sep LPG cargoes with no cuts or delays heard,Light Ends
+19,"South Korea aims for full GCC FTA execution by year-end, refiners hopeful for cheaper sour crude",Crude Oil
+20,"Indonesia sets Minas crude price at $84.95/b for July, rising $3.35/b from June",Crude Oil
+21,Cathay Pacific H1 2024 passenger traffic rises 36% on year; Hong Kong’s jet fuel demand bolstered,Middle Distillates
+22,US DATA: Total ULSD stocks near a six-month high as demand continues to fall,Middle Distillates
+23,US DATA: Product supplied of propane and propylene reach three-month high,Light Ends
+24,Internatonal Seaways focused on replacing aging fleet during second quarter: CEO,Crude Oil
+25,"Devon Energy's oil output hits all-time record high from Delaware, Eagle Ford operations",Crude Oil
+26,"Brazil's Prio still waiting on IBAMA license approvals to boost oil, gas output",Crude Oil
+27,REFINERY NEWS: Delek US sees Q3 refinery utilization dip from record Q2 highs,Light Ends
+28,"OIL FUTURES: Crude rallies as traders eye tighter US supply, global financial market stabilization",Crude Oil
+29,"Prompt DFL, CFD contracts rally",Crude Oil
+30,"REFINERY NEWS: Petroperú sees 2Q refined fuel sales drop 4.4% on year to 93,700 b/d",Middle Distillates
+31,W&T Offshore nears close of new US Gulf of Mexico drilling joint venture,Crude Oil
+32,"Imrproved efficiencies, continued M&A activity to drive growth for Permian Resources",Heavy Distillates
+33,Mexico's Pemex to explore deposit adjacent to major onshore gas field Quesqui,Middle Distillates
+34,REFINERY NEWS: Par Pacific reports softer south Rockies results as Midwest barrels spill into region,Middle Distillates
+35,"Suncor sees improved H2 oil and gas output, completes major Q2 turnarounds",Middle Distillates
+36,"Brazil's Petrobras, Espirito Santo state to study potential CCUS, hydrogen hubs",Middle Distillates
+37,"Argentina raises biodiesel, ethanol prices for blending by 1.5% in August",Middle Distillates
+38,Bolivia offers tax breaks to import equipment for biodiesel plants following fuel shortages,Light Ends
+39,"US DATA: West Coast fuel oil stocks hit a six-week low, EIA says",Middle Distillates
+40,Iraq’s SOMO cuts official selling prices for September-loading crude oil for Europe,Crude Oil
+41,Nigeria's Dangote refinery plans to divest 12.75% stake: ratings agency,Middle Distillates
+42,REFINERY NEWS: Kazakhstan's Atyrau processes 2.9 mil mt crude in H1,Middle Distillates
+43,REFINERY NEWS: Thailand's IRPC reports Q2 utilization of 94%,Light Ends
+44,DNO reports higher Q2 crude production in Iraq's Kurdish region,Crude Oil
+45,"ADNOC L&S expects ‘strong rates’ in tankers, dry-bulk, containers in 2024",Crude Oil
+46,WAF crude tanker rates hit 10-month lows amid sluggish inquiry levels,Crude Oil
+47,Senegal's inaugural crude stream Sangomar to load 3.8 mil barrels in September,Crude Oil
+48,China's July vegetable oil imports rise 3% on month as buyers replenish domestic stocks,Macroeconomic & Geopolitics
+49,CRUDE MOC: Middle East sour crude cash differentials rebound,Crude Oil
+50,OIL FUTURES: Crude oil recovers as financial markets improve,Crude Oil
+51,"Tullow sees rise in crude output, profits on-year in H1 2024",Crude Oil
+52,Russia's Taman port June-July oil products throughput up 26% on year,Heavy Distillates
+53,JAPAN DATA: Oil product exports rise 4.5% on week to 2.42 mil barrels,Crude Oil
+54,REFINERY NEWS: Petro Rabigh to be upgraded after Aramco takes control,Crude Oil
+55,Canada's ShaMaran closes acquisition of Atrush oil field,Crude Oil
+56,CHINA DATA: July natural gas imports rise 5% on year to 10.9 mil mt,Light Ends
+57,"OIL FUTURES: Crude stabilizes on technical bounce, supply uncertainty",Crude Oil
+58,JAPAN DATA: Oil product stocks rise 0.8% on week to 55.32 mil barrels,Crude Oil
+59,Japan cuts Aug 8-14 fuel subsidy by 21% as crude prices drop,Middle Distillates
+60,JAPAN DATA: Refinery runs rise to 67% over July 28-Aug 3 on higher crude throughput,Light Ends
+61,Asian reforming spread hits over two-year low as gasoline prices lag naphtha,Light Ends
+62,Asia medium sulfur gasoil differential weakens as Indonesia demand tapers,Middle Distillates
+63,"QatarEnergy raises Sep Land, Marine crude OSPs by 45-75 cents/b from Aug",Heavy Distillates
+64,ADNOC sets Murban Sep OSP $1.28/b higher on month at $83.80/b,Heavy Distillates
+65,"Diamondback Energy keeps pushing well drilling, completion efficiencies in Q2",Middle Distillates
+66,"Genel Energy’s oil production from Tawke field increases to 19,510 b/d in 1H 2024",Middle Distillates
+67,Longer laterals and higher well performance drive Rocky Mountain production: Oneok,Light Ends
+68,US DOE seeks to buy 3.5 million barrels of crude for delivery to SPR in January 2025,Crude Oil
+69,"FPSO Maria Quiteria arrives offshore Brazil, to reduce emissions: Petrobras",Middle Distillates
+70,OIL FUTURES: Crude edges higher as market stabilizes amid Middle Eastern supply concerns,Crude Oil
+71,"US EIA lowers 2024 oil price outlook by $2/b, but still predicts increases",Crude Oil
+72,"Shell, BP to fund South Africa's Sapref refinery operations in government takeover",Light Ends
+73,"Indian Oil cancels tender to build a 10,000 mt/yr renewable hydrogen plant",Light Ends
+74,"Brazil's Prio July oil equivalent output falls 31.7% on maintenance, shuttered wells",Crude Oil
+75,Eni follows Ivory Coast discoveries with four new licenses,Crude Oil
+76,EU DATA: MY 2024-25 soybean meal imports rise 8% on year as of Aug 4,Macroeconomic & Geopolitics
+77,"Greek PPC to buy a 600 MW Romanian wind farm, portfolio from Macquarie-owned developer",Macroeconomic & Geopolitics
+78,Vitol to take Italian refiner Saras private after acquiring 51% stake,Macroeconomic & Geopolitics
+79,Mediterranean sweet crude market shows muted response to Sharara shutdown,Macroeconomic & Geopolitics
+80,REFINERY NEWS: Vitol acquires 51% in Italian refiner Saras,Macroeconomic & Geopolitics
+81,Rotterdam LNG bunkers spread with VLSFO narrows to 2024 low,Light Ends
+82,Argentina’s YPF finds buyers for 15 maturing conventional blocks as it focuses on Vaca Muerta,Heavy Distillates
+83,REFINERY NEWS ROUNDUP: Nigerian plants in focus,Macroeconomic & Geopolitics
+84,"REFINERY NEWS: Valero shuts CDU, FCCU at McKee refinery for planned work",Macroeconomic & Geopolitics
+85,Kazakhstan extends ban on oil products exports by truck for six months,Macroeconomic & Geopolitics
+86,Physical Hi-Lo spread hits 3 month high amid prompt LSFO demand,Heavy Distillates
+87,CRUDE MOC: Middle East sour crude cash differentials slip to fresh lows,Crude Oil
+88,"Nigeria launches new Utapate crude grade, first cargo heads to Spain",Crude Oil
+89,REFINERY NEWS: Turkish Tupras Q2 output rises 15% on the quarter and year,Middle Distillates
+90,"CHINA DATA: Independent refineries’ Iranian crude imports fall in July, ESPO inflows rebound",Crude Oil
+91,Gunvor acquires TotalEnergies' 50% stake in Pakistan retail fuel business,Middle Distillates
+92,INTERVIEW: Coal to remain a dominant power source in India: Menar MD,Macroeconomic & Geopolitics
+93,OIL FUTURES: Crude price holds steady as demand expectations cap gains,Crude Oil
+94,Fujairah’s HSFO August HSFO ex-wharf premiums slip; stocks adequate,Heavy Distillates
+95,JAPAN DATA: US crude imports more than double in March as Middle East dependency eases,Crude Oil
+96,Dubai crude futures traded volume on TOCOM rebounds in July from record low,Crude Oil
+97,Japan's spot electricity price retreats 8% as temperatures ease,Macroeconomic & Geopolitics
+98,"HONG KONG DATA: June oil product imports surge 32% on month to 226,475 barrels",Crude Oil
+99,NextDecade signs contract with Bechtel to build Rio Grande LNG expansion,Light Ends
+100,"Kosmos sees 2024 total output of 90,000 boe/d, despite Q2 operations thorns: CEO",Crude Oil
+101,"Dated Brent reaches two-month low Aug. 5 as physical, derivatives prices slide on day",Middle Distillates
+102,"Alaska North Slope crude output up in July, but long-term decline continues",Crude Oil
+103,Balance-month DFL contract slips to seven-week low in bearish sign for physical crude fundamentals,Crude Oil
+104,Iraqi Kurdistan officials order crackdown on illegal refineries over pollution,Macroeconomic & Geopolitics
+105,Rhine barge cargo navigation limits set to kick in amid dryer weather,Middle Distillates
+106,Bolivia returns diesel supplies to normal following shortages,Middle Distillates
+107,OCI optimistic about methanol demand driven by decarbonization efforts,Light Ends
+108,Mitsubishi to supply turbine for 30% hydrogen co-firing in Malaysia power plant,Middle Distillates
+109,ATLANTIC LNG: Key market indicators for Aug. 5-9,Light Ends
+110,"Eurobob swap, gas-nap spread falls below 6-month low amid crude selloff",Light Ends
+111,EMEA PETROCHEMICALS: Key market indicators for Aug 5-9,Light Ends
+112,EMEA LIGHT ENDS: Key market indicators for Aug 5 – 9,Light Ends
+113,EUROPE AND AFRICA RESIDUAL AND MARINE FUEL: Key market indicators Aug 5-9,Heavy Distillates
+114,TURKEY DATA: June crude flows via BTC pipeline up 8.1% on month,Crude Oil
+115,EMEA AGRICULTURE: Key market indicators for Aug 5–9,Macroeconomic & Geopolitics
+116,OIL FUTURES: Crude oil faces downward pressure amid wider weakness in financial markets,Crude Oil
+117,Woodside to acquire OCI’s low carbon ammonia project with CO2 capture in US,Middle Distillates
+118,Maire secures feasibility study for sustainable aviation fuel project in Indonesia,Middle Distillates
+119,CRUDE MOC: Middle East sour crude cash differentials plunge on risk-off sentiment,Middle Distillates
+120,"Zhoushan LSFO storage availability rises for 3rd month in Aug, hits record high",Middle Distillates
+121,Oil storage in Russia's Rostov region hit by drone strike,Macroeconomic & Geopolitics
+122,WAF TRACKING: Nigerian crude exports to Netherlands top 5-year high in July,Crude Oil
+123,"Vietnam’s Hai Linh receives license to import, export LNG",Light Ends
+124,Japan's Idemitsu could restart Tokuyama steam cracker on Aug 11,Light Ends
+125,Indonesia's biodiesel output up 12% in H1 on increased domestic mandates: APROBI,Middle Distillates
+126,CHINA DATA: Independent refiners' July feedstocks imports hit 3-month low at 3.65 mil b/d,Light Ends
+127,"Singapore’s Aug ex-wharf term LSFO premiums rise, demand moderate",Heavy Distillates
+128,"OIL FUTURES: Crude slumps as market volatility rages on recession, Middle East risks",Crude Oil
+129,Pakistan's HSFO exports nearly triple as focus shifts to cheaper power sources,Heavy Distillates
+130,"TAIWAN DATA: June oil products demand falls 3% on month to 758,139 b/d",Light Ends
+131,REFINERY NEWS: Japan's Cosmo restarts No. 1 Chiba CDU after glitches,Crude Oil
+132,ASIA PETROCHEMICALS: Key market indicators for Aug 5-9,Light Ends
+133,DME Oman crude futures traded volume rises for seventh straight month in July,Crude Oil
+134,ICE front-month Singapore gasoline swaps open interest rises 14.6% on month in July,Light Ends
+135,ASIA OCTANE: Key market indicators for Aug 5-9,Light Ends
+136,ICE Dubai crude futures July total traded volume rises 11.4% on month,Crude Oil
+137,"Lower-than-expected Aramco Sep OSPs a nod to weak Asian market, OPEC+ cut unwind",Crude Oil
+138,ASIA CRUDE OIL: Key market indicators for Aug 5-8,Crude Oil
+139,ASIA LIGHT ENDS: Key market indicators for Aug 5-8,Light Ends
+140,China fuel oil quotas decline seen supporting Q3 LSFO premiums in Zhoushan,Middle Distillates
+141,South Korea's short-term diesel demand under pressure on e-commerce firms' bankruptcy,Middle Distillates
+142,ICE front-month Singapore 10 ppm gasoil swap open interest rebounds 2% on month in July,Middle Distillates
+143,Saudi Aramco maintains or raises Asia-bound Sep crude OSPs by 10-20 cents/b,Crude Oil
+144,ASIA MIDDLE DISTILLATES: Key market indicators for Aug 5-8,Middle Distillates
+145,ICE front-month Singapore HSFO open interest rises 19.6% on month in July,Heavy Distillates
+146,REFINERY NEWS: Fort Energy at Fujairah ‘remains operational’,Macroeconomic & Geopolitics
+147,Container ship Groton attacked near Yemen amid growing Middle East security risks,Macroeconomic & Geopolitics
+148,Oil depot in Russia’s Belgorod region hit by drone strike,Macroeconomic & Geopolitics

modules/data_preparation.py

@@ -1,5 +1,4 @@
 import re
-import datetime
 import plotly.express as px
 import datetime
 import plotly.graph_objects as go
@@ -11,13 +10,13 @@ def clean_text(text):
         new_text = re.sub(rgx_match, '', new_text)
     return new_text
 
-def prepare_df(df, categories):
+def prepare_df(df, categories, date_filter):
     try:
         df.drop(columns=['Unnamed: 0'], inplace=True)
     except:
         pass
 
-    df['topic_verification'][(df.headline.str.contains('crude', case=False)) | df.body.str.contains('crude', case=False)] = 'Crude Oil'
+    #df['topic_verification'][(df.headline.str.contains('crude', case=False)) | df.body.str.contains('crude', case=False)] = 'Crude Oil'
 
     try:
         news_data = df[df['topic_verification'].isin(categories)]
@@ -33,35 +32,16 @@ def prepare_df(df, categories):
         
         # insert column using insert(position,column_name,first_column) function 
         news_data.insert(0, 'headline', first_column) 
-    
+        
+        news_data['updatedDate'] = news_data['updatedDate'].apply(lambda x: datetime.datetime.strptime(x, '%y/%m/%d %H:%M:%S'))
+
+        news_data = news_data[(news_data['updatedDate'] >= date_filter[0]) & (news_data['updatedDate'] <= date_filter[1])]
+
     except Exception as E:
         print(E)
 
     return news_data
 
-# def prepare_df(df, category):
-#     try:
-#         df.drop(columns=['Unnamed: 0'], inplace=True)
-#     except:
-#         pass
-
-#     if category == 'Crude Oil':
-#         news_data = df[(df['topic_verification'] == 'Crude Oil') | (df['topic_verification'] == 'Macroeconomic & Geopolitics')]
-#     if category == 'Light Ends':
-#         news_data = df[(df['topic_verification'] == 'Light Ends')]
-#     if category == 'Middle Distillates':
-#         news_data = df[(df['topic_verification'] == 'Middle Distillates')]
-#     if category == 'Heavy Distillates':
-#         news_data = df[(df['topic_verification'] == 'Heavy Distillates')]
-
-#     actual_day = datetime.date.today() - datetime.timedelta(days=1)
-#     pattern_del = actual_day.strftime('%b').upper()
-
-#     filter = news_data['headline'].str.contains(pattern_del)
-#     news_data = news_data[~filter]
-
-#     return news_data
-
 def plot_3dgraph(news_data):
     fig = px.scatter_3d(news_data, 
                         x='neutral_score', 
@@ -77,7 +57,9 @@ def plot_3dgraph(news_data):
 
     fig.update_layout(
         height=600,
-        title=dict(text=f"Platts News Sentiments ({datetime.datetime.now().strftime('%d/%m/%y')})<br><sup>Hover cursor on a datapoint to show news title</sup>", font=dict(size=35), automargin=False)
+        title=dict(text=f"Platts News Sentiments ({datetime.datetime.now().strftime('%d/%m/%y')})<br><sup>Hover cursor on a datapoint to show news title</sup>", 
+                   font=dict(size=35), 
+                   automargin=False)
     )
 
     fig.update_traces(textfont_size=8)

test.ipynb

@@ -0,0 +1,903 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Unnamed: 0</th>\n",
+       "      <th>body</th>\n",
+       "      <th>headline</th>\n",
+       "      <th>updatedDate</th>\n",
+       "      <th>topic_prediction</th>\n",
+       "      <th>topic_verification</th>\n",
+       "      <th>negative_score</th>\n",
+       "      <th>neutral_score</th>\n",
+       "      <th>positive_score</th>\n",
+       "      <th>trend_prediction</th>\n",
+       "      <th>trend_verification</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0</td>\n",
+       "      <td>Spanish crude import volumes increased 11% ye...</td>\n",
+       "      <td>SPAIN DATA: H1 crude imports rise 11% to 1.4 m...</td>\n",
+       "      <td>2024-08-08 12:11:55+00:00</td>\n",
+       "      <td>Crude Oil</td>\n",
+       "      <td>Crude Oil</td>\n",
+       "      <td>0.991473</td>\n",
+       "      <td>0.005524</td>\n",
+       "      <td>0.519264</td>\n",
+       "      <td>Bearish</td>\n",
+       "      <td>Bearish</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>1</td>\n",
+       "      <td>A number of refineries in China have resumed ...</td>\n",
+       "      <td>REFINERY NEWS: Host of Chinese units back from...</td>\n",
+       "      <td>2024-08-08 11:51:12+00:00</td>\n",
+       "      <td>Macroeconomic &amp; Geopolitics</td>\n",
+       "      <td>Macroeconomic &amp; Geopolitics</td>\n",
+       "      <td>0.417054</td>\n",
+       "      <td>0.845595</td>\n",
+       "      <td>0.180685</td>\n",
+       "      <td>Neutral</td>\n",
+       "      <td>Neutral</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>2</td>\n",
+       "      <td>Some refineries in the Asia-Pacific region in...</td>\n",
+       "      <td>REFINERY NEWS ROUNDUP: Mixed runs in Asia-Pacific</td>\n",
+       "      <td>2024-08-08 11:50:48+00:00</td>\n",
+       "      <td>Macroeconomic &amp; Geopolitics</td>\n",
+       "      <td>Macroeconomic &amp; Geopolitics</td>\n",
+       "      <td>0.268708</td>\n",
+       "      <td>0.044504</td>\n",
+       "      <td>0.992063</td>\n",
+       "      <td>Bullish</td>\n",
+       "      <td>Bullish</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>3</td>\n",
+       "      <td>The physical low sulfur (1%S) fuel oil Med-No...</td>\n",
+       "      <td>Physical 1%S fuel oil Med-North spread hits re...</td>\n",
+       "      <td>2024-08-08 11:28:20+00:00</td>\n",
+       "      <td>Middle Distillates</td>\n",
+       "      <td>Middle Distillates</td>\n",
+       "      <td>0.951985</td>\n",
+       "      <td>0.009613</td>\n",
+       "      <td>0.822905</td>\n",
+       "      <td>Bearish</td>\n",
+       "      <td>Bearish</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>4</td>\n",
+       "      <td>Bunkering activity in India has experienced s...</td>\n",
+       "      <td>Indian ports see Jan-July bunker, STS calls up...</td>\n",
+       "      <td>2024-08-08 11:27:15+00:00</td>\n",
+       "      <td>Heavy Distillates</td>\n",
+       "      <td>Heavy Distillates</td>\n",
+       "      <td>0.098844</td>\n",
+       "      <td>0.059348</td>\n",
+       "      <td>0.997325</td>\n",
+       "      <td>Bullish</td>\n",
+       "      <td>Bullish</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>153</th>\n",
+       "      <td>367</td>\n",
+       "      <td>The Asian middle distillates complex could be...</td>\n",
+       "      <td>ASIA MIDDLE DISTILLATES: Key market indicators...</td>\n",
+       "      <td>2024-08-05 01:50:17+00:00</td>\n",
+       "      <td>Middle Distillates</td>\n",
+       "      <td>Middle Distillates</td>\n",
+       "      <td>0.034932</td>\n",
+       "      <td>0.310855</td>\n",
+       "      <td>0.993474</td>\n",
+       "      <td>Bullish</td>\n",
+       "      <td>Bullish</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>154</th>\n",
+       "      <td>368</td>\n",
+       "      <td>The combined open interest for front-month Si...</td>\n",
+       "      <td>ICE front-month Singapore HSFO open interest r...</td>\n",
+       "      <td>2024-08-05 01:26:07+00:00</td>\n",
+       "      <td>Heavy Distillates</td>\n",
+       "      <td>Heavy Distillates</td>\n",
+       "      <td>0.368089</td>\n",
+       "      <td>0.017483</td>\n",
+       "      <td>0.994805</td>\n",
+       "      <td>Bullish</td>\n",
+       "      <td>Bullish</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>155</th>\n",
+       "      <td>369</td>\n",
+       "      <td>Production will be increasing “in the near fu...</td>\n",
+       "      <td>REFINERY NEWS: Fort Energy at Fujairah ‘remain...</td>\n",
+       "      <td>2024-08-05 00:45:56+00:00</td>\n",
+       "      <td>Macroeconomic &amp; Geopolitics</td>\n",
+       "      <td>Macroeconomic &amp; Geopolitics</td>\n",
+       "      <td>0.775953</td>\n",
+       "      <td>0.520116</td>\n",
+       "      <td>0.177664</td>\n",
+       "      <td>Bearish</td>\n",
+       "      <td>Bearish</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>156</th>\n",
+       "      <td>370</td>\n",
+       "      <td>Container ship Groton was attacked 125 nautic...</td>\n",
+       "      <td>Container ship Groton attacked near Yemen amid...</td>\n",
+       "      <td>2024-08-04 12:25:30+00:00</td>\n",
+       "      <td>Macroeconomic &amp; Geopolitics</td>\n",
+       "      <td>Macroeconomic &amp; Geopolitics</td>\n",
+       "      <td>0.245594</td>\n",
+       "      <td>0.044671</td>\n",
+       "      <td>0.994086</td>\n",
+       "      <td>Bullish</td>\n",
+       "      <td>Bullish</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>157</th>\n",
+       "      <td>371</td>\n",
+       "      <td>A drone strike on an oil depot in Russia’s Be...</td>\n",
+       "      <td>Oil depot in Russia’s Belgorod region hit by d...</td>\n",
+       "      <td>2024-08-04 10:14:50+00:00</td>\n",
+       "      <td>Macroeconomic &amp; Geopolitics</td>\n",
+       "      <td>Macroeconomic &amp; Geopolitics</td>\n",
+       "      <td>0.243901</td>\n",
+       "      <td>0.951199</td>\n",
+       "      <td>0.129076</td>\n",
+       "      <td>Neutral</td>\n",
+       "      <td>Neutral</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>158 rows × 11 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "     Unnamed: 0                                               body  \\\n",
+       "0             0   Spanish crude import volumes increased 11% ye...   \n",
+       "1             1   A number of refineries in China have resumed ...   \n",
+       "2             2   Some refineries in the Asia-Pacific region in...   \n",
+       "3             3   The physical low sulfur (1%S) fuel oil Med-No...   \n",
+       "4             4   Bunkering activity in India has experienced s...   \n",
+       "..          ...                                                ...   \n",
+       "153         367   The Asian middle distillates complex could be...   \n",
+       "154         368   The combined open interest for front-month Si...   \n",
+       "155         369   Production will be increasing “in the near fu...   \n",
+       "156         370   Container ship Groton was attacked 125 nautic...   \n",
+       "157         371   A drone strike on an oil depot in Russia’s Be...   \n",
+       "\n",
+       "                                              headline  \\\n",
+       "0    SPAIN DATA: H1 crude imports rise 11% to 1.4 m...   \n",
+       "1    REFINERY NEWS: Host of Chinese units back from...   \n",
+       "2    REFINERY NEWS ROUNDUP: Mixed runs in Asia-Pacific   \n",
+       "3    Physical 1%S fuel oil Med-North spread hits re...   \n",
+       "4    Indian ports see Jan-July bunker, STS calls up...   \n",
+       "..                                                 ...   \n",
+       "153  ASIA MIDDLE DISTILLATES: Key market indicators...   \n",
+       "154  ICE front-month Singapore HSFO open interest r...   \n",
+       "155  REFINERY NEWS: Fort Energy at Fujairah ‘remain...   \n",
+       "156  Container ship Groton attacked near Yemen amid...   \n",
+       "157  Oil depot in Russia’s Belgorod region hit by d...   \n",
+       "\n",
+       "                   updatedDate             topic_prediction  \\\n",
+       "0    2024-08-08 12:11:55+00:00                    Crude Oil   \n",
+       "1    2024-08-08 11:51:12+00:00  Macroeconomic & Geopolitics   \n",
+       "2    2024-08-08 11:50:48+00:00  Macroeconomic & Geopolitics   \n",
+       "3    2024-08-08 11:28:20+00:00           Middle Distillates   \n",
+       "4    2024-08-08 11:27:15+00:00            Heavy Distillates   \n",
+       "..                         ...                          ...   \n",
+       "153  2024-08-05 01:50:17+00:00           Middle Distillates   \n",
+       "154  2024-08-05 01:26:07+00:00            Heavy Distillates   \n",
+       "155  2024-08-05 00:45:56+00:00  Macroeconomic & Geopolitics   \n",
+       "156  2024-08-04 12:25:30+00:00  Macroeconomic & Geopolitics   \n",
+       "157  2024-08-04 10:14:50+00:00  Macroeconomic & Geopolitics   \n",
+       "\n",
+       "              topic_verification  negative_score  neutral_score  \\\n",
+       "0                      Crude Oil        0.991473       0.005524   \n",
+       "1    Macroeconomic & Geopolitics        0.417054       0.845595   \n",
+       "2    Macroeconomic & Geopolitics        0.268708       0.044504   \n",
+       "3             Middle Distillates        0.951985       0.009613   \n",
+       "4              Heavy Distillates        0.098844       0.059348   \n",
+       "..                           ...             ...            ...   \n",
+       "153           Middle Distillates        0.034932       0.310855   \n",
+       "154            Heavy Distillates        0.368089       0.017483   \n",
+       "155  Macroeconomic & Geopolitics        0.775953       0.520116   \n",
+       "156  Macroeconomic & Geopolitics        0.245594       0.044671   \n",
+       "157  Macroeconomic & Geopolitics        0.243901       0.951199   \n",
+       "\n",
+       "     positive_score trend_prediction trend_verification  \n",
+       "0          0.519264          Bearish            Bearish  \n",
+       "1          0.180685          Neutral            Neutral  \n",
+       "2          0.992063          Bullish            Bullish  \n",
+       "3          0.822905          Bearish            Bearish  \n",
+       "4          0.997325          Bullish            Bullish  \n",
+       "..              ...              ...                ...  \n",
+       "153        0.993474          Bullish            Bullish  \n",
+       "154        0.994805          Bullish            Bullish  \n",
+       "155        0.177664          Bearish            Bearish  \n",
+       "156        0.994086          Bullish            Bullish  \n",
+       "157        0.129076          Neutral            Neutral  \n",
+       "\n",
+       "[158 rows x 11 columns]"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "import os\n",
+    "\n",
+    "test = pd.read_csv(os.path.join('data','all_platts_1week_clean.csv'))\n",
+    "test"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test.to_excel('evaluation.xlsx')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train = test.drop(columns=['Unnamed: 0', 'body', 'updatedDate', 'topic_prediction', 'negative_score', 'neutral_score', 'positive_score', 'trend_prediction', 'trend_verification'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train.to_csv('convert_first.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dated_brent = pd.read_csv('data/dated_brent_allbate.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dated_brent = dated_brent[dated_brent['assessDate'] >= '2024-08-04']\n",
+    "dated_brent = dated_brent[dated_brent['bate']=='c']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Unnamed: 0</th>\n",
+       "      <th>bate</th>\n",
+       "      <th>value</th>\n",
+       "      <th>assessDate</th>\n",
+       "      <th>isCorrected</th>\n",
+       "      <th>modDate</th>\n",
+       "      <th>symbol</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>8019</th>\n",
+       "      <td>8019</td>\n",
+       "      <td>c</td>\n",
+       "      <td>76.700</td>\n",
+       "      <td>2024-08-05</td>\n",
+       "      <td>N</td>\n",
+       "      <td>2024-08-05 17:55:34</td>\n",
+       "      <td>PCAAS00</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8022</th>\n",
+       "      <td>8022</td>\n",
+       "      <td>c</td>\n",
+       "      <td>76.275</td>\n",
+       "      <td>2024-08-06</td>\n",
+       "      <td>N</td>\n",
+       "      <td>2024-08-06 17:48:25</td>\n",
+       "      <td>PCAAS00</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8025</th>\n",
+       "      <td>8025</td>\n",
+       "      <td>c</td>\n",
+       "      <td>79.910</td>\n",
+       "      <td>2024-08-07</td>\n",
+       "      <td>N</td>\n",
+       "      <td>2024-08-07 19:01:31</td>\n",
+       "      <td>PCAAS00</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8028</th>\n",
+       "      <td>8028</td>\n",
+       "      <td>c</td>\n",
+       "      <td>81.100</td>\n",
+       "      <td>2024-08-08</td>\n",
+       "      <td>N</td>\n",
+       "      <td>2024-08-08 18:15:44</td>\n",
+       "      <td>PCAAS00</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8031</th>\n",
+       "      <td>8031</td>\n",
+       "      <td>c</td>\n",
+       "      <td>81.615</td>\n",
+       "      <td>2024-08-09</td>\n",
+       "      <td>N</td>\n",
+       "      <td>2024-08-09 18:00:52</td>\n",
+       "      <td>PCAAS00</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "      Unnamed: 0 bate   value  assessDate isCorrected              modDate  \\\n",
+       "8019        8019    c  76.700  2024-08-05           N  2024-08-05 17:55:34   \n",
+       "8022        8022    c  76.275  2024-08-06           N  2024-08-06 17:48:25   \n",
+       "8025        8025    c  79.910  2024-08-07           N  2024-08-07 19:01:31   \n",
+       "8028        8028    c  81.100  2024-08-08           N  2024-08-08 18:15:44   \n",
+       "8031        8031    c  81.615  2024-08-09           N  2024-08-09 18:00:52   \n",
+       "\n",
+       "       symbol  \n",
+       "8019  PCAAS00  \n",
+       "8022  PCAAS00  \n",
+       "8025  PCAAS00  \n",
+       "8028  PCAAS00  \n",
+       "8031  PCAAS00  "
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dated_brent"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Unnamed: 0</th>\n",
+       "      <th>body</th>\n",
+       "      <th>headline</th>\n",
+       "      <th>updatedDate</th>\n",
+       "      <th>topic_prediction</th>\n",
+       "      <th>topic_verification</th>\n",
+       "      <th>negative_score</th>\n",
+       "      <th>neutral_score</th>\n",
+       "      <th>positive_score</th>\n",
+       "      <th>trend_prediction</th>\n",
+       "      <th>trend_verification</th>\n",
+       "      <th>Unnamed: 11</th>\n",
+       "      <th>Price Data</th>\n",
+       "      <th>Unnamed: 13</th>\n",
+       "      <th>Unnamed: 14</th>\n",
+       "      <th>Bearish Prediction</th>\n",
+       "      <th>Neutral Prediction</th>\n",
+       "      <th>Bullish Prediction</th>\n",
+       "      <th>Prediction (All News)</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0</td>\n",
+       "      <td>Spanish crude import volumes increased 11% ye...</td>\n",
+       "      <td>SPAIN DATA: H1 crude imports rise 11% to 1.4 m...</td>\n",
+       "      <td>2024-08-08 12:11:55+00:00</td>\n",
+       "      <td>Crude Oil</td>\n",
+       "      <td>Crude Oil</td>\n",
+       "      <td>0.991473</td>\n",
+       "      <td>0.005524</td>\n",
+       "      <td>0.519264</td>\n",
+       "      <td>Bearish</td>\n",
+       "      <td>Bearish</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>2024-08-05</td>\n",
+       "      <td>76.7</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.575591</td>\n",
+       "      <td>0.183341</td>\n",
+       "      <td>0.748952</td>\n",
+       "      <td>Bullish</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>1</td>\n",
+       "      <td>A number of refineries in China have resumed ...</td>\n",
+       "      <td>REFINERY NEWS: Host of Chinese units back from...</td>\n",
+       "      <td>2024-08-08 11:51:12+00:00</td>\n",
+       "      <td>Macroeconomic &amp; Geopolitics</td>\n",
+       "      <td>Macroeconomic &amp; Geopolitics</td>\n",
+       "      <td>0.417054</td>\n",
+       "      <td>0.845595</td>\n",
+       "      <td>0.180685</td>\n",
+       "      <td>Neutral</td>\n",
+       "      <td>Neutral</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>2024-08-06</td>\n",
+       "      <td>76.275</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.653413</td>\n",
+       "      <td>0.137436</td>\n",
+       "      <td>0.672831</td>\n",
+       "      <td>Bullish</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>2</td>\n",
+       "      <td>Some refineries in the Asia-Pacific region in...</td>\n",
+       "      <td>REFINERY NEWS ROUNDUP: Mixed runs in Asia-Pacific</td>\n",
+       "      <td>2024-08-08 11:50:48+00:00</td>\n",
+       "      <td>Macroeconomic &amp; Geopolitics</td>\n",
+       "      <td>Macroeconomic &amp; Geopolitics</td>\n",
+       "      <td>0.268708</td>\n",
+       "      <td>0.044504</td>\n",
+       "      <td>0.992063</td>\n",
+       "      <td>Bullish</td>\n",
+       "      <td>Bullish</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>2024-08-07</td>\n",
+       "      <td>79.91</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.653586</td>\n",
+       "      <td>0.096258</td>\n",
+       "      <td>0.727852</td>\n",
+       "      <td>Bullish</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>3</td>\n",
+       "      <td>The physical low sulfur (1%S) fuel oil Med-No...</td>\n",
+       "      <td>Physical 1%S fuel oil Med-North spread hits re...</td>\n",
+       "      <td>2024-08-08 11:28:20+00:00</td>\n",
+       "      <td>Middle Distillates</td>\n",
+       "      <td>Middle Distillates</td>\n",
+       "      <td>0.951985</td>\n",
+       "      <td>0.009613</td>\n",
+       "      <td>0.822905</td>\n",
+       "      <td>Bearish</td>\n",
+       "      <td>Bearish</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>2024-08-08</td>\n",
+       "      <td>81.1</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.499159</td>\n",
+       "      <td>0.209326</td>\n",
+       "      <td>0.750482</td>\n",
+       "      <td>Bullish</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>4</td>\n",
+       "      <td>Bunkering activity in India has experienced s...</td>\n",
+       "      <td>Indian ports see Jan-July bunker, STS calls up...</td>\n",
+       "      <td>2024-08-08 11:27:15+00:00</td>\n",
+       "      <td>Heavy Distillates</td>\n",
+       "      <td>Heavy Distillates</td>\n",
+       "      <td>0.098844</td>\n",
+       "      <td>0.059348</td>\n",
+       "      <td>0.997325</td>\n",
+       "      <td>Bullish</td>\n",
+       "      <td>Bullish</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>2024-08-09</td>\n",
+       "      <td>81.615</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>136</th>\n",
+       "      <td>136</td>\n",
+       "      <td>Saudi Aramco maintained or raised the Asia-bo...</td>\n",
+       "      <td>Saudi Aramco maintains or raises Asia-bound Se...</td>\n",
+       "      <td>2024-08-05 01:59:16+00:00</td>\n",
+       "      <td>Crude Oil</td>\n",
+       "      <td>Crude Oil</td>\n",
+       "      <td>0.154642</td>\n",
+       "      <td>0.032633</td>\n",
+       "      <td>0.997273</td>\n",
+       "      <td>Bullish</td>\n",
+       "      <td>Bullish</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaT</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>137</th>\n",
+       "      <td>137</td>\n",
+       "      <td>The combined open interest for front-month Si...</td>\n",
+       "      <td>ICE front-month Singapore HSFO open interest r...</td>\n",
+       "      <td>2024-08-05 01:26:07+00:00</td>\n",
+       "      <td>Heavy Distillates</td>\n",
+       "      <td>Heavy Distillates</td>\n",
+       "      <td>0.368089</td>\n",
+       "      <td>0.017483</td>\n",
+       "      <td>0.994805</td>\n",
+       "      <td>Bullish</td>\n",
+       "      <td>Bullish</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaT</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>138</th>\n",
+       "      <td>138</td>\n",
+       "      <td>Production will be increasing “in the near fu...</td>\n",
+       "      <td>REFINERY NEWS: Fort Energy at Fujairah ‘remain...</td>\n",
+       "      <td>2024-08-05 00:45:56+00:00</td>\n",
+       "      <td>Macroeconomic &amp; Geopolitics</td>\n",
+       "      <td>Macroeconomic &amp; Geopolitics</td>\n",
+       "      <td>0.775953</td>\n",
+       "      <td>0.520116</td>\n",
+       "      <td>0.177664</td>\n",
+       "      <td>Bearish</td>\n",
+       "      <td>Bearish</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaT</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>139</th>\n",
+       "      <td>139</td>\n",
+       "      <td>Container ship Groton was attacked 125 nautic...</td>\n",
+       "      <td>Container ship Groton attacked near Yemen amid...</td>\n",
+       "      <td>2024-08-04 12:25:30+00:00</td>\n",
+       "      <td>Macroeconomic &amp; Geopolitics</td>\n",
+       "      <td>Macroeconomic &amp; Geopolitics</td>\n",
+       "      <td>0.245594</td>\n",
+       "      <td>0.044671</td>\n",
+       "      <td>0.994086</td>\n",
+       "      <td>Bullish</td>\n",
+       "      <td>Bullish</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaT</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>140</th>\n",
+       "      <td>140</td>\n",
+       "      <td>A drone strike on an oil depot in Russia’s Be...</td>\n",
+       "      <td>Oil depot in Russia’s Belgorod region hit by d...</td>\n",
+       "      <td>2024-08-04 10:14:50+00:00</td>\n",
+       "      <td>Macroeconomic &amp; Geopolitics</td>\n",
+       "      <td>Macroeconomic &amp; Geopolitics</td>\n",
+       "      <td>0.243901</td>\n",
+       "      <td>0.951199</td>\n",
+       "      <td>0.129076</td>\n",
+       "      <td>Neutral</td>\n",
+       "      <td>Neutral</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaT</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>141 rows × 19 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "     Unnamed: 0                                               body  \\\n",
+       "0             0   Spanish crude import volumes increased 11% ye...   \n",
+       "1             1   A number of refineries in China have resumed ...   \n",
+       "2             2   Some refineries in the Asia-Pacific region in...   \n",
+       "3             3   The physical low sulfur (1%S) fuel oil Med-No...   \n",
+       "4             4   Bunkering activity in India has experienced s...   \n",
+       "..          ...                                                ...   \n",
+       "136         136   Saudi Aramco maintained or raised the Asia-bo...   \n",
+       "137         137   The combined open interest for front-month Si...   \n",
+       "138         138   Production will be increasing “in the near fu...   \n",
+       "139         139   Container ship Groton was attacked 125 nautic...   \n",
+       "140         140   A drone strike on an oil depot in Russia’s Be...   \n",
+       "\n",
+       "                                              headline  \\\n",
+       "0    SPAIN DATA: H1 crude imports rise 11% to 1.4 m...   \n",
+       "1    REFINERY NEWS: Host of Chinese units back from...   \n",
+       "2    REFINERY NEWS ROUNDUP: Mixed runs in Asia-Pacific   \n",
+       "3    Physical 1%S fuel oil Med-North spread hits re...   \n",
+       "4    Indian ports see Jan-July bunker, STS calls up...   \n",
+       "..                                                 ...   \n",
+       "136  Saudi Aramco maintains or raises Asia-bound Se...   \n",
+       "137  ICE front-month Singapore HSFO open interest r...   \n",
+       "138  REFINERY NEWS: Fort Energy at Fujairah ‘remain...   \n",
+       "139  Container ship Groton attacked near Yemen amid...   \n",
+       "140  Oil depot in Russia’s Belgorod region hit by d...   \n",
+       "\n",
+       "                   updatedDate             topic_prediction  \\\n",
+       "0    2024-08-08 12:11:55+00:00                    Crude Oil   \n",
+       "1    2024-08-08 11:51:12+00:00  Macroeconomic & Geopolitics   \n",
+       "2    2024-08-08 11:50:48+00:00  Macroeconomic & Geopolitics   \n",
+       "3    2024-08-08 11:28:20+00:00           Middle Distillates   \n",
+       "4    2024-08-08 11:27:15+00:00            Heavy Distillates   \n",
+       "..                         ...                          ...   \n",
+       "136  2024-08-05 01:59:16+00:00                    Crude Oil   \n",
+       "137  2024-08-05 01:26:07+00:00            Heavy Distillates   \n",
+       "138  2024-08-05 00:45:56+00:00  Macroeconomic & Geopolitics   \n",
+       "139  2024-08-04 12:25:30+00:00  Macroeconomic & Geopolitics   \n",
+       "140  2024-08-04 10:14:50+00:00  Macroeconomic & Geopolitics   \n",
+       "\n",
+       "              topic_verification  negative_score  neutral_score  \\\n",
+       "0                      Crude Oil        0.991473       0.005524   \n",
+       "1    Macroeconomic & Geopolitics        0.417054       0.845595   \n",
+       "2    Macroeconomic & Geopolitics        0.268708       0.044504   \n",
+       "3             Middle Distillates        0.951985       0.009613   \n",
+       "4              Heavy Distillates        0.098844       0.059348   \n",
+       "..                           ...             ...            ...   \n",
+       "136                    Crude Oil        0.154642       0.032633   \n",
+       "137            Heavy Distillates        0.368089       0.017483   \n",
+       "138  Macroeconomic & Geopolitics        0.775953       0.520116   \n",
+       "139  Macroeconomic & Geopolitics        0.245594       0.044671   \n",
+       "140  Macroeconomic & Geopolitics        0.243901       0.951199   \n",
+       "\n",
+       "     positive_score trend_prediction trend_verification  Unnamed: 11  \\\n",
+       "0          0.519264          Bearish            Bearish          NaN   \n",
+       "1          0.180685          Neutral            Neutral          NaN   \n",
+       "2          0.992063          Bullish            Bullish          NaN   \n",
+       "3          0.822905          Bearish            Bearish          NaN   \n",
+       "4          0.997325          Bullish            Bullish          NaN   \n",
+       "..              ...              ...                ...          ...   \n",
+       "136        0.997273          Bullish            Bullish          NaN   \n",
+       "137        0.994805          Bullish            Bullish          NaN   \n",
+       "138        0.177664          Bearish            Bearish          NaN   \n",
+       "139        0.994086          Bullish            Bullish          NaN   \n",
+       "140        0.129076          Neutral            Neutral          NaN   \n",
+       "\n",
+       "    Price Data Unnamed: 13  Unnamed: 14  Bearish Prediction  \\\n",
+       "0   2024-08-05        76.7          NaN            0.575591   \n",
+       "1   2024-08-06      76.275          NaN            0.653413   \n",
+       "2   2024-08-07       79.91          NaN            0.653586   \n",
+       "3   2024-08-08        81.1          NaN            0.499159   \n",
+       "4   2024-08-09      81.615          NaN                 NaN   \n",
+       "..         ...         ...          ...                 ...   \n",
+       "136        NaT         NaN          NaN                 NaN   \n",
+       "137        NaT         NaN          NaN                 NaN   \n",
+       "138        NaT         NaN          NaN                 NaN   \n",
+       "139        NaT         NaN          NaN                 NaN   \n",
+       "140        NaT         NaN          NaN                 NaN   \n",
+       "\n",
+       "     Neutral Prediction  Bullish Prediction Prediction (All News)  \n",
+       "0              0.183341            0.748952               Bullish  \n",
+       "1              0.137436            0.672831               Bullish  \n",
+       "2              0.096258            0.727852               Bullish  \n",
+       "3              0.209326            0.750482               Bullish  \n",
+       "4                   NaN                 NaN                   NaN  \n",
+       "..                  ...                 ...                   ...  \n",
+       "136                 NaN                 NaN                   NaN  \n",
+       "137                 NaN                 NaN                   NaN  \n",
+       "138                 NaN                 NaN                   NaN  \n",
+       "139                 NaN                 NaN                   NaN  \n",
+       "140                 NaN                 NaN                   NaN  \n",
+       "\n",
+       "[141 rows x 19 columns]"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "eval = pd.read_excel('evaluation.xlsx')\n",
+    "eval"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Bearish prediction 0.5093043801287273\n",
+      " Neutral prediction 0.24558571118013772\n",
+      " Bullish prediction 0.7223830104380737\n"
+     ]
+    }
+   ],
+   "source": [
+    "test = eval[(eval['topic_prediction']=='Macroeconomic & Geopolitics') | (eval['topic_prediction']=='Crude Oil')]\n",
+    "test = test[test['updatedDate'].str.contains('2024-08-08')]\n",
+    "print(f'Bearish prediction {test.negative_score.mean()}\\n Neutral prediction {test.neutral_score.mean()}\\n Bullish prediction {test.positive_score.mean()}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "rl",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.18"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}