f

pages/mayakovsky.py

@@ -0,0 +1,64 @@
+import streamlit as st
+import textwrap
+import torch
+from transformers import GPT2LMHeadModel, GPT2Tokenizer
+
+DEVICE = torch.device("cpu")
+# Load GPT-2 model and tokenizer
+tokenizer = GPT2Tokenizer.from_pretrained('sberbank-ai/rugpt3small_based_on_gpt2')
+model_finetuned = GPT2LMHeadModel.from_pretrained(
+    'sberbank-ai/rugpt3small_based_on_gpt2',
+    output_attentions = False,
+    output_hidden_states = False,
+)
+if torch.cuda.is_available():
+    model_finetuned.load_state_dict(torch.load('models/mayakovsky.pt'))
+else:
+    model_finetuned.load_state_dict(torch.load('models/mayakovsky.pt', map_location=torch.device('cpu')))
+model_finetuned.eval()
+
+# Function to generate text
+def generate_text(prompt, temperature, top_p, max_length, top_k):
+    input_ids = tokenizer.encode(prompt, return_tensors="pt")
+    
+    with torch.no_grad():
+        out = model_finetuned.generate(
+            input_ids,
+            do_sample=True,
+            num_beams=5,
+            temperature=temperature,
+            top_p=top_p,
+            max_length=max_length,
+            top_k=top_k,
+            no_repeat_ngram_size=3,
+            num_return_sequences=1,
+        )
+        
+    generated_text = list(map(tokenizer.decode, out))
+    return generated_text
+
+# Streamlit app
+def main():
+    st.title("Генерация текста GPT-моделью в стиле В.В. Маяковского")
+    
+    # User inputs
+    prompt = st.text_area("Введите начало текста")
+    temperature = st.slider("Temperature", min_value=0.2, max_value=2.5, value=1.8, step=0.1)
+    top_p = st.slider("Top-p", min_value=0.1, max_value=1.0, value=0.9, step=0.1)
+    max_length = st.slider("Max Length", min_value=10, max_value=300, value=100, step=10)
+    top_k = st.slider("Top-k", min_value=1, max_value=500, value=500, step=10)
+    num_return_sequences = st.slider("Number of Sequences", min_value=1, max_value=5, value=1, step=1)
+
+    if st.button("Generate Text"):
+        st.subheader("Generated Text:")
+        for i in range(num_return_sequences):
+            generated_text = generate_text(prompt, temperature, top_p, max_length, top_k)
+            st.write(f"Generated Text {i + 1}:")
+            wrapped_text = textwrap.fill(generated_text[0], width=80)
+            st.write(wrapped_text)
+            st.write("------------------")
+
+st.sidebar.image('images/mayakovsky.jpeg', use_column_width=True) 
+
+if __name__ == "__main__":
+    main()

pages/polyclinics.py

@@ -0,0 +1,115 @@
+import streamlit as st
+import numpy as np
+import time
+import pickle  
+import torch   
+import pandas as pd                                       
+from gensim.models import KeyedVectors
+from transformers import BertTokenizer, BertModel
+from nltk.corpus import stopwords
+from nltk.stem import SnowballStemmer
+from function.lstm_preprocessing import (                         
+                                clean,         
+                                tokin,
+                                predict_ml_class,
+                                predict_sentence,
+                                predict_single_string,
+                                LSTMClassifier
+                                )
+
+DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
+
+stemmer = SnowballStemmer('russian') 
+sw = stopwords.words('russian')  
+
+EMBEDDING_DIM = 32
+HIDDEN_DIM = 32
+SEQ_LEN = 200
+VOCAB_SIZE = 196906
+EMBEDDING_DIM = 32
+wv = KeyedVectors.load("file/wv.wordvectors", mmap='r')
+
+with open('file/vocab_to_int.txt', 'rb') as f:
+    vocab_to_int = pickle.load(f)
+
+embedding_matrix = np.zeros((VOCAB_SIZE, EMBEDDING_DIM))
+
+for word, i in vocab_to_int.items():
+    try:
+        embedding_vector = wv[word]
+        embedding_matrix[i] = embedding_vector
+    except KeyError as e:
+        pass
+
+embedding_layer = torch.nn.Embedding.from_pretrained(torch.FloatTensor(embedding_matrix))
+
+model = LSTMClassifier(embedding_dim=EMBEDDING_DIM, hidden_size=HIDDEN_DIM, embedding=embedding_layer).to(DEVICE)
+model.load_state_dict(torch.load('models/LTSM_model_epoch_7.pt', map_location='cpu'))
+
+tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased')
+model_BERT = BertModel.from_pretrained("bert-base-multilingual-cased")
+
+loaded_model = pickle.load(open('models/LogReg.pickle', "rb"))
+
+loaded_classifier = pickle.load(open('models/trained_model.pkl', "rb"))
+loaded_vectorizer = pickle.load(open('models/vectorizer.pkl', "rb"))
+
+def main():
+    st.title("Классификация отзыва на поликлиники")
+    user_input = st.text_area("Введите ваш отзыв:", "")
+    return user_input
+
+user_input = main()
+
+def predict_lstm(user_input):
+    start_time = time.time()
+    prediction = predict_sentence(user_input, model, SEQ_LEN, vocab_to_int)
+    end_time = time.time()
+    return prediction, round((end_time - start_time), 4)
+
+def predict_bert(user_input):
+    start_time = time.time()
+    prediction = predict_single_string(user_input, model_BERT, loaded_model)
+    end_time = time.time()
+    return prediction, round((end_time - start_time), 4)
+
+def predict_ML(user_input):
+    start_time = time.time()
+    prediction = predict_ml_class(user_input, loaded_vectorizer, loaded_classifier)
+    end_time = time.time()
+    return prediction, round((end_time - start_time), 4)
+
+if user_input:
+    prediction_rnn, time_taken_rnn = predict_ML(user_input)
+    st.write("### Bag-of-Words + LogReg")
+    st.write("Предсказанный класс:", prediction_rnn)
+    st.write("Время предсказания:", time_taken_rnn, "сек.")
+    prediction_rnn, time_taken_rnn = predict_lstm(user_input)
+    st.write("### LSTM модель")
+    st.write("Предсказанный класс:", prediction_rnn)
+    st.write("Время предсказания:", time_taken_rnn, "сек.")
+    prediction_rnn, time_taken_rnn = predict_bert(user_input)
+    st.write("### BERT модель + LogReg")
+    st.write("Предсказанный класс:", prediction_rnn)
+    st.write("Время предсказания:", time_taken_rnn, "сек.")
+
+
+st.sidebar.image('images/polyclinic.jpeg', use_column_width=True) 
+f1_score_classic_ml = 0.87
+f1_score_rnn = 0.88
+f1_score_bert = 0.83
+f1_score_classic_ml_valid = 0.89
+f1_score_rnn_valid = 0.92
+f1_score_bert_valid = 0.82
+# Создание DataFrame для сравнения результатов
+
+
+
+st.sidebar.write("### Сравнительная таблица по метрике f1-macro")
+results = {
+"Модель": ["Классический ML", "LSTM", "BERT-based"],
+"train": [f1_score_classic_ml, f1_score_rnn, f1_score_bert],
+"valid": [f1_score_classic_ml_valid, f1_score_rnn_valid, f1_score_bert_valid]
+}
+results_df = pd.DataFrame(results)
+st.sidebar.dataframe(results_df)

pages/pushkin.py

@@ -0,0 +1,64 @@
+import streamlit as st
+import textwrap
+import torch
+from transformers import GPT2LMHeadModel, GPT2Tokenizer
+
+DEVICE = torch.device("cpu")
+# Load GPT-2 model and tokenizer
+tokenizer = GPT2Tokenizer.from_pretrained('sberbank-ai/rugpt3small_based_on_gpt2')
+model_finetuned = GPT2LMHeadModel.from_pretrained(
+    'sberbank-ai/rugpt3small_based_on_gpt2',
+    output_attentions = False,
+    output_hidden_states = False,
+)
+if torch.cuda.is_available():
+    model_finetuned.load_state_dict(torch.load('models/model_pushkin.pt'))
+else:
+    model_finetuned.load_state_dict(torch.load('models/model_pushkin.pt', map_location=torch.device('cpu')))
+model_finetuned.eval()
+
+# Function to generate text
+def generate_text(prompt, temperature, top_p, max_length, top_k):
+    input_ids = tokenizer.encode(prompt, return_tensors="pt")
+    
+    with torch.no_grad():
+        out = model_finetuned.generate(
+            input_ids,
+            do_sample=True,
+            num_beams=5,
+            temperature=temperature,
+            top_p=top_p,
+            max_length=max_length,
+            top_k=top_k,
+            no_repeat_ngram_size=3,
+            num_return_sequences=1,
+        )
+        
+    generated_text = list(map(tokenizer.decode, out))
+    return generated_text
+
+# Streamlit app
+def main():
+    st.title("Генерация текста GPT-моделью в стиле А.С. Пушкина")
+    
+    # User inputs
+    prompt = st.text_area("Введите начало текста")
+    temperature = st.slider("Temperature", min_value=0.2, max_value=2.5, value=1.8, step=0.1)
+    top_p = st.slider("Top-p", min_value=0.1, max_value=1.0, value=0.9, step=0.1)
+    max_length = st.slider("Max Length", min_value=10, max_value=300, value=100, step=10)
+    top_k = st.slider("Top-k", min_value=1, max_value=500, value=500, step=10)
+    num_return_sequences = st.slider("Number of Sequences", min_value=1, max_value=5, value=1, step=1)
+
+    if st.button("Generate Text"):
+        st.subheader("Generated Text:")
+        for i in range(num_return_sequences):
+            generated_text = generate_text(prompt, temperature, top_p, max_length, top_k)
+            st.write(f"Generated Text {i + 1}:")
+            wrapped_text = textwrap.fill(generated_text[0], width=80)
+            st.write(wrapped_text)
+            st.write("------------------")
+
+st.sidebar.image('images/pushkin.jpeg', use_column_width=True) 
+
+if __name__ == "__main__":
+    main()

pages/toxic.py

@@ -0,0 +1,36 @@
+import streamlit as st
+import torch
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+
+def main():
+    st.title("Оценка токсичности сообщений")
+
+    # Загрузка модели
+    model_checkpoint = 'cointegrated/rubert-tiny-toxicity'
+    tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
+    model = AutoModelForSequenceClassification.from_pretrained(model_checkpoint)
+    if torch.cuda.is_available():
+        model.cuda()
+        
+    def text2toxicity(text, aggregate=True):
+        """ Calculate toxicity of a text (if aggregate=True) or a vector of toxicity aspects (if aggregate=False)"""
+        with torch.no_grad():
+            inputs = tokenizer(text, return_tensors='pt', truncation=True, padding=True).to(model.device)
+            proba = torch.sigmoid(model(**inputs).logits).cpu().numpy()
+        if isinstance(text, str):
+            proba = proba[0]
+        if aggregate:
+            return 1 - proba.T[0] * (1 - proba.T[-1])
+        return proba
+
+    message = st.text_area("Введите сообщение для оценки:")
+    if st.button("Оценить"):
+        if message:
+            toxicity_score = text2toxicity(message)
+            st.write(f"Степень токсичности: {toxicity_score:.4f}")
+
+st.sidebar.image('images/toxic.jpeg', use_column_width=True) 
+
+
+if __name__ == "__main__":
+    main()