Update app.py

app.py

@@ -3,22 +3,14 @@ import pickle
 import pandas as pd
 from sentence_transformers import SentenceTransformer, util
 import numpy as np
-from pymongo import MongoClient
 import urllib.parse
 import requests
-from bertopic import BERTopic
-from sklearn.feature_extraction.text import TfidfVectorizer
-from sklearn.metrics.pairwise import cosine_similarity
-import string
-import deepcut
-import unicodedata
-from pythainlp.util import normalize
-import torch
-import csv
 
 #initial state
-if 'state' not in st.session_state:
-    st.session_state.state = 0
+if 'state_p1' not in st.session_state:
+    st.session_state.state_p1 = 0
+if 'state_p2' not in st.session_state:
+    st.session_state.state_p2 = 0
 if 'age' not in st.session_state:
     st.session_state.age = 0
 if 'weight' not in st.session_state:
@@ -33,26 +25,25 @@ if 'drug_allergy' not in st.session_state:
     st.session_state.drug_allergy = 0
 if 'congentital_disease' not in st.session_state:
     st.session_state.congentital_disease = 0
-if 'optional_keyword' not in st.session_state:
-    st.session_state.optional_keyword = 0
-if 'all_recommend' not in st.session_state:
-    st.session_state.all_recommend = None
-if 'true_check' not in st.session_state:
-    st.session_state.true_check = None
 if 'queries' not in st.session_state:
     st.session_state.queries = None
-if 'string_contain' not in st.session_state:
-    st.session_state.string_contain = False
 if 'sbert_searched_df' not in st.session_state:
     st.session_state.sbert_searched_df = None
-if 'string_contain_df' not in st.session_state:
-    st.session_state.string_contain_df = None
+if 'queries_p2' not in st.session_state:
+    st.session_state.queries_p2 = None
+if 'sbert_searched_df_p2' not in st.session_state:
+    st.session_state.sbert_searched_df_p2 = None
 for i in range(10):
     if 'score_'+str(i+1) not in st.session_state:
         st.session_state['score_'+str(i+1)] = 'NA'
+if 'current_page' not in st.session_state:
+    st.session_state.current_page = 1
 
-def set_state(state):
-    st.session_state.state = state
+def set_state_p1(state):
+    st.session_state.state_p1 = state
+
+def set_state_p2(state):
+    st.session_state.state_p2 = state
 
 def split_text(text):
     return text.split(',')
@@ -60,39 +51,12 @@ def split_text(text):
 #import data
 sbert_model = SentenceTransformer('paraphrase-multilingual-mpnet-base-v2')
 
-with open('corpus_embeddings.pickle', 'rb') as file:
-    corpus_embeddings = pickle.load(file)
-corpus_embeddings = pd.DataFrame(corpus_embeddings)
-
-topic_model = BERTopic.load("Jiranuwat/topic_model",embedding_model=sbert_model)
 data = pd.read_csv('articles_data.csv')
-data['child_topic'] = topic_model.topics_[:]
 
-with open('sensitive_words.txt', 'r',encoding='utf-8') as file:
-    sensitive_words = file.read()
-sensitive_words = sensitive_words.lower().replace('\n','').split(' ')
-sensitive_words = list(set(sensitive_words))
+with open('corpus_embeddings.pickle', 'rb') as file:
+    corpus_embeddings = pickle.load(file)
 
 #local function
-def save_session_state_data(session_state_data, filename):
-    with open(filename, 'a', newline='', encoding='utf-8') as file:
-        writer = csv.DictWriter(file, fieldnames=session_state_data.keys())
-        if file.tell() == 0:
-            writer.writeheader()
-        writer.writerow(session_state_data)
-
-def deepcut_tokenizer(text,sensitive_words=sensitive_words):
-    cleanedText = "".join([i for i in text if i not in string.punctuation]).lower()
-    cleanedText = normalize(unicodedata.normalize('NFKD', cleanedText).replace('\n','').replace('\r','').replace('\t','').replace('“','').replace('”','').replace('.','').replace('–','').replace('‘','').replace('’','').replace('ํา','ำ').replace('...','').replace(',','').replace( 'ี','ี'))
-    #cleanedText = re.sub(r'\d+', '', cleanedText)
-    cleanedText = deepcut.tokenize(cleanedText,custom_dict=sensitive_words)
-    #stopwords = list(thai_stopwords())+'EMagazine GJ international bangkok hospital'.lower().split(' ')
-    stopwords = 'EMagazine GJ international bangkok hospital'.lower().split(' ')
-    cleanedText = [i for i in cleanedText if i not in stopwords]
-    cleanedText = [i.replace(' ','') for i in cleanedText if len(i) != 1 and len(i) !=0]
-    cleanedText = ','.join(cleanedText)
-    return cleanedText
-
 def personal_check(age,weight,height,gender):
 
     #age check
@@ -129,7 +93,7 @@ def sbert_search(queries,data,embeddiing,sbert_model=sbert_model):
     score_lst = []
 
     query_embedding = sbert_model.encode(queries, convert_to_tensor=True)
-    hits = util.semantic_search(query_embedding, embeddiing, top_k=15)
+    hits = util.semantic_search(query_embedding, embeddiing, top_k=10)
     hits = hits[0]
     for hit in hits:
         index_lst.append(hit['corpus_id'])
@@ -140,53 +104,9 @@ def sbert_search(queries,data,embeddiing,sbert_model=sbert_model):
 
     return sbert_searched
 
-def sbert_tfidf_search(queries,head,topic_model=topic_model,data=data,corpus_embeddings=corpus_embeddings):
-    
-    similar_df = None
-    text_to_predict_token = deepcut_tokenizer(queries)
-
-    # Find topics
-    try:
-        similar_topics, similarity = topic_model.find_topics(text_to_predict_token, top_n=1)
-    except:
-        similar_topics, similarity = topic_model.find_topics(queries, top_n=1)
-
-    # Example DataFrame
-    similar_df = data[data['child_topic'] == similar_topics[0]]
-
-    # TF-IDF vectorizer
-    vectorizer = TfidfVectorizer(tokenizer=lambda x: x, lowercase=False)
-    tfidf_matrix = vectorizer.fit_transform(similar_df['text_token'])
-
-    # TF-IDF vector for input text
-    text_tfidf = vectorizer.transform([text_to_predict_token])
-
-    # Compute cosine similarity
-    similarity_scores = cosine_similarity(tfidf_matrix, text_tfidf)
-
-    # Add similarity scores to DataFrame
-    similar_df['score'] = similarity_scores
-
-    similar_df = similar_df.sort_values('score', ascending=False).head(15)
-
-    select_corpus = corpus_embeddings.iloc[similar_df.index.sort_values()]
-    similar_embedding = torch.tensor(select_corpus.values)
-    similar_searched = sbert_search(queries,similar_df,similar_embedding)
-    sbert_searched = sbert_search(queries,data,torch.tensor(corpus_embeddings.values))
-    combined_searched = pd.concat([similar_searched,sbert_searched])
-    output = combined_searched.sort_values('score', ascending=False).head(head)
-
-    return output
-
-def string_contain_search(queries,sample,data=data):
-    data['all_content'] = data['title']+data['content']
-    return data[data['all_content'].str.contains(queries,na=False)].sample(sample)
-
-#main
-def main():
+def page1_recommendation():
     #header
     st.markdown("<h1 style='text-align: center; color: black;'>---ระบบแนะนำบทความสุขภาพ---</h1>", unsafe_allow_html=True)
-    st.subheader("ให้คะแนนบทความหน่อยนะครับ😄")
 
     with st.form('user_info'):
 
@@ -207,11 +127,10 @@ def main():
         with col5:
             drug_allergy = st.selectbox('แพ้ยา?',('ไม่แพ้', 'แพ้ยา'))
         congentital_disease = st.text_input('โรคประจำตัวของคุณ (ถ้าหากไม่มี ไม่ต้องกรอก หรือใส่ "ไม่มี")')
-        optional_keyword = st.text_input('คำค้นหาเพิ่มเติม (ถ้ามี)')
 
-        st.form_submit_button(on_click=set_state,args=(1,))
+        st.form_submit_button(on_click=set_state_p1,args=(1,))
 
-    if st.session_state.state == 1:
+    if st.session_state.state_p1 == 1:
 
         #asign state
         st.session_state.age = age
@@ -221,7 +140,6 @@ def main():
         st.session_state.food_allergy = food_allergy
         st.session_state.drug_allergy = drug_allergy
         st.session_state.congentital_disease = congentital_disease
-        st.session_state.optional_keyword = optional_keyword
 
         #algorithm
         age,gender,bmi = personal_check(age,weight,height,gender)
@@ -232,115 +150,88 @@ def main():
             drug_allergy = ''
         if congentital_disease == 'ไม่มี':
             congentital_disease = ''
-        if congentital_disease != '' or optional_keyword != '':
-            queries = optional_keyword+congentital_disease
+
+        if congentital_disease == '':
+            queries = gender+age+bmi+food_allergy+drug_allergy
         else:
-            queries = gender+age+bmi+food_allergy+drug_allergy+congentital_disease+optional_keyword
+            queries = congentital_disease
         
         #Bertopic search
-        try:
-            sbert_searched = sbert_tfidf_search(queries,5)
-            string_contain = string_contain_search(queries,5)
-            all_recommend = pd.concat([sbert_searched,string_contain])
-            all_recommend = all_recommend.drop_duplicates(subset=['url'])
-
-            if len(all_recommend) != 10:
-                for i in range(3):
-                    if len(all_recommend) < 10:
-                        all_recommend = None
-                        sbert_searched = sbert_tfidf_search(queries,5+i+1)
-                        sbert_searched = sbert_searched.head(5)
-                        string_contain = string_contain_search(queries,5+i+1)
-                        string_contain = string_contain.head(5)
-                        all_recommend = pd.concat([sbert_searched,string_contain])
-                        all_recommend = all_recommend.drop_duplicates(subset=['url'])
-
-            st.session_state.sbert_searched_df = sbert_searched
-            st.session_state.string_contain_df = string_contain
-            st.session_state.string_contain = True
-
-        except:
-            sbert_searched = sbert_tfidf_search(queries,10)
-            st.session_state.sbert_searched_df = sbert_searched
-            all_recommend = sbert_searched
-
-        st.session_state.all_recommend = all_recommend
+        sbert_searched = sbert_search(queries,data,corpus_embeddings)
+
+        st.session_state.sbert_searched_df = sbert_searched
         st.session_state.queries = queries
-        st.session_state.state = 2
+        st.session_state.state_p1 = 2
+
+    if st.session_state.state_p1 == 2:
+
+        with st.form('recommendations'):
+            st.markdown("<h1 style='text-align: center; color: black;'>📰บทความสำหรับคุณ😆</h1>", unsafe_allow_html=True)
+            st.write("---------------------------------------------------------------------------------------")
+
+            for i in range(len(st.session_state.sbert_searched_df)):
+                st.header(str(i+1)+'. '+st.session_state.sbert_searched_df.iloc[i]['title'])
+                st.markdown(f"[Page source (Click here.)]({st.session_state.sbert_searched_df.iloc[i].url})")
+
+                try:
+                    banner_url = urllib.parse.quote(st.session_state.sbert_searched_df.iloc[i]['banner'], safe=':/')
+                    response = requests.get(banner_url,timeout=5)
+                    st.image(response.content)
+                except:
+                    st.image('https://icon-library.com/images/no-photo-icon/no-photo-icon-1.jpg')
+                finally:
+                    st.write("---------------------------------------------------------------------------------------")
+            
+            st.form_submit_button('Submit',on_click=set_state_p1,args=(0,))
+
+def page2_search_engine():
+    st.title("Search engine")
+
+    with st.form('queries'):
+        queries = st.text_input('คำหรือหัวข้อที่ต้องการค้นหา')
+        st.form_submit_button(on_click=set_state_p2,args=(1,))
+    
+    if st.session_state.state_p2 == 1:
+        sbert_searched = sbert_search(queries,data,corpus_embeddings)
 
-    if st.session_state.state == 2:
-        placeholder = st.empty()
+        st.session_state.sbert_searched_df_p2 = sbert_searched
+        st.session_state.queries_p2 = queries
+        st.session_state.state_p2 = 2
 
-        #satisfaction
-        with placeholder.form('Satisfaction Survey'):
+    if st.session_state.state_p2 == 2:
+        with st.form('recommendations'):
             st.markdown("<h1 style='text-align: center; color: black;'>📰บทความสำหรับคุณ😆</h1>", unsafe_allow_html=True)
-            st.header("ระดับความเกี่ยวข้อง")
-            st.write("😞 หมายถึง ไม่เกี่ยวข้องเลย")
-            st.write("🙁 หมายถึง เกี่ยวข้องเล็กน้อย")
-            st.write("😐 หมายถึง เฉยๆ")
-            st.write("🙂 หมายถึง ค่อนข้างเกี่ยวข้อง")
-            st.write("😀 หมายถึง เกี่ยวข้องมากที่สุด")
             st.write("---------------------------------------------------------------------------------------")
 
-            for i in range(len(st.session_state.all_recommend)):
-                st.header(str(i+1)+'. '+st.session_state.all_recommend.iloc[i]['title'])
-                st.markdown(f"[Page source (Click here.)]({st.session_state.all_recommend.iloc[i].url})")
+            for i in range(len(st.session_state.sbert_searched_df_p2)):
+                st.header(str(i+1)+'. '+st.session_state.sbert_searched_df_p2.iloc[i]['title'])
+                st.markdown(f"[Page source (Click here.)]({st.session_state.sbert_searched_df_p2.iloc[i].url})")
 
                 try:
-                    banner_url = urllib.parse.quote(st.session_state.all_recommend.iloc[i]['banner'], safe=':/')
-                    response = requests.get(banner_url)
+                    banner_url = urllib.parse.quote(st.session_state.sbert_searched_df_p2.iloc[i]['banner'], safe=':/')
+                    response = requests.get(banner_url,timeout=5)
                     st.image(response.content)
                 except:
                     st.image('https://icon-library.com/images/no-photo-icon/no-photo-icon-1.jpg')
+                finally:
+                    st.write("---------------------------------------------------------------------------------------")
+            
+            st.form_submit_button('Submit',on_click=set_state_p2,args=(0,))
 
-                #satisfaction survey
-                st.subheader("Satisfaction Survey")
-                st.write("บทความที่แนะนำเกี่ยวข้องกับคุณมากเพียงใด")
-                st.radio('ระดับความพึงพอใจ',['NA','😞','🙁','😐','🙂','😀'],horizontal=True,key='score_'+str(i+1))
-                st.write("---------------------------------------------------------------------------------------")
-
-            if st.form_submit_button("ยืนยันการส่งคำตอบ"):
-                # Check if all articles have satisfaction levels selected
-                st.session_state.true_check = []
-                for satis_val in [st.session_state[i] for i in ['score_' + str(i+1) for i in range(10)]]:
-                    if satis_val != 'NA': 
-                        st.session_state.true_check.append(True)
-                    else:
-                        st.session_state.true_check.append(False)
-
-                if np.all(st.session_state.true_check):
-                    st.session_state.state = 3
-                    placeholder.empty()
-                
-                else:
-                    idx = []
-                    for i in range(len(st.session_state.true_check)):
-                        if st.session_state.true_check[i] == False:
-                            idx.append(i+1)
-                    article_indexes = ', '.join(map(str, idx))
-                    st.warning(f":red[กรุณาให้คะแนนบทความที่ {article_indexes} ด้วยครับ]")
-
-    if st.session_state.state == 3:
-        st.success('บันทึกคำตอบแล้ว')
-
-        st.session_state.all_recommend = st.session_state.all_recommend.to_dict(orient='records')
-        if st.session_state.sbert_searched_df is not None:
-            st.session_state.sbert_searched_df = st.session_state.sbert_searched_df.to_dict(orient='records')
-        if st.session_state.string_contain_df is not None:
-            st.session_state.string_contain_df = st.session_state.string_contain_df.to_dict(orient='records')
-
-        try:
-            save_session_state_data(st.session_state.to_dict(), 'satisfaction.csv')
-
-        except:
-            #database insertion
-            client = MongoClient('mongodb://192.168.1.103:27017/')
-            database = client['test']
-            collection = database['satisfy_articles']
-            collection.insert_one(st.session_state.to_dict())
-
-        finally:
-            st.session_state.state = 0
+#main
+def main():
+    st.sidebar.title("Navigation")
+    page = st.sidebar.selectbox("Select a page:", ("Recommendation System", "Search Engine"))
+    
+    if page == "Recommendation System":
+        st.session_state.current_page = 1
+    else:
+        st.session_state.current_page = 2
 
+    if page == "Recommendation System" and st.session_state.current_page == 1:
+        page1_recommendation()
+    elif page == "Search Engine" and st.session_state.current_page == 2:
+        page2_search_engine()
+    
 if __name__ == "__main__":
     main()