File size: 3,634 Bytes
1ec57a1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 |
import numpy as np
import pandas as pd
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.feature_extraction.text import CountVectorizer
from datasets import load_metric
import nltk
from nltk.tokenize import wordpunct_tokenize
from nltk.corpus import stopwords
import nltk.translate.bleu_score as bleu
from nltk.translate.bleu_score import SmoothingFunction
import nltk.translate.gleu_score as gleu
import nltk.translate.meteor_score as meteor
from jiwer import wer, mer
import re
import math
from collections import Counter
import string
from tqdm import tqdm
nltk.download('stopwords')
stopwords = stopwords.words("english")
df = pd.read_csv("./test_generated.csv", sep="\t")
true_recipes = df["true_recipe"].values.tolist()
generated_recipes = df["generated_recipe"].values.tolist()
def cleaning(text, rm_sep=True, rm_nl=True, rm_punk_stopwords=True):
if rm_sep:
text = text.replace("--", " ")
if rm_nl:
text = text.replace("\n", " ")
if rm_punk_stopwords:
text = " ".join([word.strip() for word in wordpunct_tokenize(text) if word not in string.punctuation and word not in stopwords and word])
else:
text = " ".join([word.strip() for word in wordpunct_tokenize(text) if word.strip()])
text = text.lower()
return text
X, Y = [], []
for x, y in tqdm(zip(true_recipes, generated_recipes), total=len(df)):
x, y = cleaning(x, True, True, True), cleaning(y, True, True, True)
if len(x) > 16 and len(y) > 16:
X.append(x)
Y.append(y)
print(f"Sample X: {X[0]}")
print(f"Sample Y: {Y[0]}")
def get_cosine(vec1, vec2):
intersection = set(vec1.keys()) & set(vec2.keys())
numerator = sum([vec1[x] * vec2[x] for x in intersection])
sum1 = sum([vec1[x]**2 for x in vec1.keys()])
sum2 = sum([vec2[x]**2 for x in vec2.keys()])
denominator = math.sqrt(sum1) * math.sqrt(sum2)
if not denominator:
return 0.0
else:
return float(numerator) / denominator
def text_to_vector(text):
word = re.compile(r'\w+')
words = word.findall(text)
return Counter(words)
def get_result(content_a, content_b):
text1 = content_a
text2 = content_b
vector1 = text_to_vector(text1)
vector2 = text_to_vector(text2)
cosine_result = get_cosine(vector1, vector2)
return cosine_result
cosim_scores = []
for i in tqdm(range(len(X))):
cosim_scores.append(get_result(X[i], Y[i]))
cosim_score = np.array(cosim_scores).mean()
print(f"Cosine similarity score: {cosim_score}") # 0.714542
X, Y = [], []
for x, y in tqdm(zip(true_recipes, generated_recipes), total=len(df)):
x, y = cleaning(x, True, True, False), cleaning(y, True, True, False)
if len(x) > 16 and len(y) > 16:
X.append(x)
Y.append(y)
wer = load_metric("wer")
wer_score = wer.compute(predictions=Y, references=X)
print(f"WER score: {wer_score}") # 0.70938
rouge = load_metric("rouge")
rouge_score = rouge.compute(predictions=Y, references=X, use_stemmer=True)
rouge_score = {key: value.mid.fmeasure * 100 for key, value in rouge_score.items()}
print(f"Rouge score: {rouge_score}") # {'rouge1': 56.30779082900833, 'rouge2': 29.07704230163075, 'rougeL': 45.812165960365924, 'rougeLsum': 45.813971137090654}
bleu = load_metric("bleu")
def postprocess_text(preds, labels):
preds = [wordpunct_tokenize(pred) for pred in preds]
labels = [[wordpunct_tokenize(label)] for label in labels]
return preds, labels
Y, X = postprocess_text(Y, X)
bleu_score = bleu.compute(predictions=Y, references=X)["bleu"]
print(f"BLEU score: {bleu_score}") # 0.203867 |