Finalize model

README.md

@@ -7,17 +7,13 @@ tags:
 - recipe-generation
 pipeline_tag: text2text-generation
 widget:
-  - text: "gold potatoes, olive oil, kosher salt, truffle salt"
-  - text: "custard, sage, sugar, milk, heavy cream, eggs, egg yolks, salt, vanilla, maple bourbon caramel sauce, maple syrup, bourbon, sugar, water, light corn syrup, cream of tartar"
-  - text: "bulgar wheat, olive oil, cucumber, tomato, lemon, garlic, green onions, fresh mint, salt"
-  - text: "active dry yeast, milk, sugar, unsalted butter, chocolate, espresso powder, egg, vanilla, cocoa, salt, bread flour"
-  - text: "penne pasta, ground beef, pasta sauce, ricotta cheese, mozzarella cheese, parmesan cheese, egg"
-  - text: "fresh strawberries, sugar, vinegar"
-  - text: "soy sauce, lime juice, apricot preserves, water, garlic, ground ginger, carrots, green onions, olive oil, shrimp, sweet red pepper, rice, lettuce leaves"
-  - text: "cucumbers, lemon juice, parsley, water, onion, sour cream, black pepper, salt"
+  - text: "provolone cheese, bacon, bread, ginger"
+  - text: "sugar, crunchy jif peanut butter, cornflakes"
+  - text: "sweet butter, confectioners sugar, flaked coconut, condensed milk, nuts, vanilla, dipping chocolate"
+  - text: "macaroni, butter, salt, bacon, milk, flour, pepper, cream corn"
+  - text: "hamburger, sausage, onion, regular, american cheese, colby cheese"
   - text: "chicken breasts, onion, garlic, great northern beans, black beans, green chilies, broccoli, garlic oil, butter, cajun seasoning, salt, oregano, thyme, black pepper, basil, worcestershire sauce, chicken broth, sour cream, chardonnay wine"
   - text: "serrano peppers, garlic, celery, oregano, canola oil, vinegar, water, kosher salt, salt, black pepper"
-
 ---
 
 ![avatar](chef-transformer.png)
@@ -26,7 +22,6 @@ widget:
 > This is part of the
 [Flax/Jax Community Week](https://discuss.huggingface.co/t/recipe-generation-model/7475), organized by [HuggingFace](https://huggingface.co/) and TPU usage sponsored by Google.
 
-... SOON
 
 ## Team Members
 - Mehrdad Farahani ([m3hrdadfi](https://huggingface.co/m3hrdadfi))
@@ -73,19 +68,151 @@ widget:
 
 ## How To Use
 
-... SOON
+### Installing requirements
 
-## Evaluation
+```bash
+pip install transformers
+```
 
-... SOON
+```python
+from transformers import FlaxAutoModelForSeq2SeqLM
+from transformers import AutoTokenizer
+
+MODEL_NAME_OR_PATH = "flax-community/t5-recipe-generation"
+tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME_OR_PATH, use_fast=True)
+model = FlaxAutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME_OR_PATH)
+
+prefix = "items: "
+# generation_kwargs = {
+#     "max_length": 1024,
+#     "min_length": 128,
+#     "no_repeat_ngram_size": 3,
+#     "do_sample": True,
+#     "top_k": 60,
+#     "top_p": 0.95
+# }
+generation_kwargs = {
+    "max_length": 512,
+    "min_length": 64,
+    "no_repeat_ngram_size": 3,
+    "early_stopping": True,
+    "num_beams": 5,
+    "length_penalty": 1.5,
+}
 
-### Baseline
+special_tokens = tokenizer.all_special_tokens
+tokens_map = {
+    "<sep>": "--",
+    "<section>": "\n"
+}
+def skip_special_tokens(text, special_tokens):
+    for token in special_tokens:
+        text = text.replace(token, '')
+
+    return text
+
+def target_postprocessing(texts, special_tokens):
+    if not isinstance(texts, list):
+        texts = [texts]
+    
+    new_texts = []
+    for text in texts:
+        text = skip_special_tokens(text, special_tokens)
+
+        for k, v in tokens_map.items():
+            text = text.replace(k, v)
+
+        new_texts.append(text)
+
+    return new_texts
+
+def generation_function(texts):
+    _inputs = texts if isinstance(texts, list) else [texts]
+    inputs = [prefix + inp for inp in _inputs]
+    inputs = tokenizer(
+        inputs, 
+        max_length=256, 
+        padding="max_length", 
+        truncation=True, 
+        return_tensors='jax'
+    )
+
+    input_ids = inputs.input_ids
+    attention_mask = inputs.attention_mask
+
+    output_ids = model.generate(
+        input_ids=input_ids, 
+        attention_mask=attention_mask,
+        **generation_kwargs
+    )
+    generated = output_ids.sequences
+    generated_recipe = target_postprocessing(
+        tokenizer.batch_decode(generated, skip_special_tokens=False),
+        special_tokens
+    )
+    return generated_recipe
+```
 
-... SOON
+```python
+items = [
+    "macaroni, butter, salt, bacon, milk, flour, pepper, cream corn"
+]
+generated = generation_function(items)
+for text in generated:
+    sections = text.split("\n")
+    for section in sections:
+        section = section.strip()
+        if section.startswith("title:"):
+            section = section.replace("title:", "")
+            headline = "TITLE"
+        elif section.startswith("ingredients:"):
+            section = section.replace("ingredients:", "")
+            headline = "INGREDIENTS"
+        elif section.startswith("directions:"):
+            section = section.replace("directions:", "")
+            headline = "DIRECTIONS"
+        
+        section_info = [f"  - {i+1}: {info.strip().capitalize()}" for i, info in enumerate(section.split("--"))]
+        print(f"[{headline}]:")
+        print("\n".join(section_info))
+
+    print("-" * 130)
+```
+
+Output:
+```text
+[TITLE]:
+  - 1: Macaroni and corn
+[INGREDIENTS]:
+  - 1: 2 c. macaroni
+  - 2: 2 tbsp. butter
+  - 3: 1 tsp. salt
+  - 4: 4 slices bacon
+  - 5: 2 c. milk
+  - 6: 2 tbsp. flour
+  - 7: 1/4 tsp. pepper
+  - 8: 1 can cream corn
+[DIRECTIONS]:
+  - 1: Cook macaroni in boiling salted water until tender.
+  - 2: Drain.
+  - 3: Melt butter in saucepan.
+  - 4: Blend in flour, salt and pepper.
+  - 5: Add milk all at once.
+  - 6: Cook and stir until thickened and bubbly.
+  - 7: Stir in corn and bacon.
+  - 8: Pour over macaroni and mix well.
+----------------------------------------------------------------------------------------------------------------------------------
+```
+
+## Evaluation
 
-### Our Results
+The following tables summarize the scores obtained by the **Chef Transformer**. Those marked as (*) are the baseline models.
 
-... SOON
+|      Model      |  BLEU |  WER  | COSIM | ROUGE-2 |
+|:---------------:|:-----:|:-----:|:-----:|:-------:|
+|   Recipe1M+ *   | 0.844 | 0.786 | 0.589 |    -    |
+|   RecipeNLG *   | 0.866 | 0.751 | 0.666 |    -    |
+| ChefTransformer | 0.203 | 0.709 | 0.714 |  0.290  |
 
 
 ## Copyright

config.json

@@ -51,9 +51,9 @@
     },
     "text2text-generation": {
       "early_stopping": true,
-      "max_length": 1024,
-      "repetition_penalty": 1.0,
-      "length_penalty": 1.0,
+      "max_length": 512,
+      "repetition_penalty": 1.2,
+      "length_penalty": 1.2,
       "num_beams": 5,
       "prefix": "items: "
     }

data/test_generated.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:479d8789478d92caafa5f31c7c6dbe0fbe55deb34fb0bda72fe410b77a914427
+size 145654259

src/evaluation.py

@@ -0,0 +1,124 @@
+import numpy as np
+import pandas as pd
+from sklearn.metrics.pairwise import cosine_similarity
+from sklearn.feature_extraction.text import CountVectorizer
+
+from datasets import load_metric
+
+import nltk
+from nltk.tokenize import wordpunct_tokenize
+from nltk.corpus import stopwords
+import nltk.translate.bleu_score as bleu
+from nltk.translate.bleu_score import SmoothingFunction
+import nltk.translate.gleu_score as gleu
+import nltk.translate.meteor_score as meteor
+
+from jiwer import wer, mer
+
+import re
+import math
+from collections import Counter
+import string
+from tqdm import tqdm
+
+
+nltk.download('stopwords')
+stopwords = stopwords.words("english")
+
+
+df = pd.read_csv("./test_generated.csv", sep="\t")
+true_recipes = df["true_recipe"].values.tolist()
+generated_recipes = df["generated_recipe"].values.tolist()
+
+def cleaning(text, rm_sep=True, rm_nl=True, rm_punk_stopwords=True):
+    if rm_sep:
+        text = text.replace("--", " ")
+    
+    if rm_nl:
+        text = text.replace("\n", " ")
+
+    if rm_punk_stopwords:
+        text = " ".join([word.strip() for word in wordpunct_tokenize(text) if word not in string.punctuation and word not in stopwords and word])
+    else:
+        text = " ".join([word.strip() for word in wordpunct_tokenize(text) if word.strip()])
+
+    text = text.lower()
+    return text
+
+X, Y = [], []
+for x, y in tqdm(zip(true_recipes, generated_recipes), total=len(df)):
+    x, y = cleaning(x, True, True, True), cleaning(y, True, True, True)
+
+    if len(x) > 16 and len(y) > 16:
+        X.append(x)
+        Y.append(y)
+
+
+print(f"Sample X: {X[0]}")
+print(f"Sample Y: {Y[0]}")
+
+def get_cosine(vec1, vec2):
+    intersection = set(vec1.keys()) & set(vec2.keys())
+    numerator = sum([vec1[x] * vec2[x] for x in intersection])
+
+    sum1 = sum([vec1[x]**2 for x in vec1.keys()])
+    sum2 = sum([vec2[x]**2 for x in vec2.keys()])
+    denominator = math.sqrt(sum1) * math.sqrt(sum2)
+
+    if not denominator:
+        return 0.0
+    else:
+        return float(numerator) / denominator
+
+def text_to_vector(text):
+    word = re.compile(r'\w+')
+    words = word.findall(text)
+    return Counter(words)
+
+def get_result(content_a, content_b):
+    text1 = content_a
+    text2 = content_b
+
+    vector1 = text_to_vector(text1)
+    vector2 = text_to_vector(text2)
+
+    cosine_result = get_cosine(vector1, vector2)
+    return cosine_result
+
+
+cosim_scores = []
+for i in tqdm(range(len(X))):
+    cosim_scores.append(get_result(X[i], Y[i]))
+
+cosim_score = np.array(cosim_scores).mean()
+print(f"Cosine similarity score: {cosim_score}")  # 0.714542
+
+X, Y = [], []
+for x, y in tqdm(zip(true_recipes, generated_recipes), total=len(df)):
+    x, y = cleaning(x, True, True, False), cleaning(y, True, True, False)
+
+    if len(x) > 16 and len(y) > 16:
+        X.append(x)
+        Y.append(y)
+        
+        
+wer = load_metric("wer")
+wer_score = wer.compute(predictions=Y, references=X)
+print(f"WER score: {wer_score}")  # 0.70938
+
+    
+rouge = load_metric("rouge")
+rouge_score = rouge.compute(predictions=Y, references=X, use_stemmer=True)
+rouge_score = {key: value.mid.fmeasure * 100 for key, value in rouge_score.items()}
+print(f"Rouge score: {rouge_score}")  # {'rouge1': 56.30779082900833, 'rouge2': 29.07704230163075, 'rougeL': 45.812165960365924, 'rougeLsum': 45.813971137090654}
+
+bleu = load_metric("bleu")
+def postprocess_text(preds, labels):
+    preds = [wordpunct_tokenize(pred) for pred in preds]
+    labels = [[wordpunct_tokenize(label)] for label in labels]
+
+    return preds, labels
+
+Y, X = postprocess_text(Y, X)
+bleu_score = bleu.compute(predictions=Y, references=X)["bleu"]
+print(f"BLEU score: {bleu_score}")  # 0.203867

src/prediction.py

@@ -0,0 +1,102 @@
+from transformers import FlaxAutoModelForSeq2SeqLM
+from transformers import AutoTokenizer
+import textwrap
+
+MODEL_NAME_OR_PATH = "flax-community/t5-recipe-generation"
+tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME_OR_PATH, use_fast=True)
+model = FlaxAutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME_OR_PATH)
+
+prefix = "items: "
+# generation_kwargs = {
+#     "max_length": 1024,
+#     "min_length": 128,
+#     "no_repeat_ngram_size": 3,
+#     "do_sample": True,
+#     "top_k": 60,
+#     "top_p": 0.95
+# }
+generation_kwargs = {
+    "max_length": 512,
+    "min_length": 64,
+    "no_repeat_ngram_size": 3,
+    "early_stopping": True,
+    "num_beams": 5,
+    "length_penalty": 1.5,
+}
+
+special_tokens = tokenizer.all_special_tokens
+tokens_map = {
+    "<sep>": "--",
+    "<section>": "\n"
+}
+def skip_special_tokens(text, special_tokens):
+    for token in special_tokens:
+        text = text.replace(token, '')
+
+    return text
+
+def target_postprocessing(texts, special_tokens):
+    if not isinstance(texts, list):
+        texts = [texts]
+    
+    new_texts = []
+    for text in texts:
+        text = skip_special_tokens(text, special_tokens)
+
+        for k, v in tokens_map.items():
+            text = text.replace(k, v)
+
+        new_texts.append(text)
+
+    return new_texts
+
+def generation_function(texts):
+    _inputs = texts if isinstance(texts, list) else [texts]
+    inputs = [prefix + inp for inp in _inputs]
+    inputs = tokenizer(
+        inputs, 
+        max_length=256, 
+        padding="max_length", 
+        truncation=True, 
+        return_tensors='jax'
+    )
+
+    input_ids = inputs.input_ids
+    attention_mask = inputs.attention_mask
+
+    output_ids = model.generate(
+        input_ids=input_ids, 
+        attention_mask=attention_mask,
+        **generation_kwargs
+    )
+    generated = output_ids.sequences
+    generated_recipe = target_postprocessing(
+        tokenizer.batch_decode(generated, skip_special_tokens=False),
+        special_tokens
+    )
+    return generated_recipe
+
+
+items = [
+    "macaroni, butter, salt, bacon, milk, flour, pepper, cream corn"
+]
+generated = generation_function(items)
+for text in generated:
+    sections = text.split("\n")
+    for section in sections:
+        section = section.strip()
+        if section.startswith("title:"):
+            section = section.replace("title:", "")
+            headline = "TITLE"
+        elif section.startswith("ingredients:"):
+            section = section.replace("ingredients:", "")
+            headline = "INGREDIENTS"
+        elif section.startswith("directions:"):
+            section = section.replace("directions:", "")
+            headline = "DIRECTIONS"
+        
+        section_info = [f"  - {i+1}: {info.strip().capitalize()}" for i, info in enumerate(section.split("--"))]
+        print(f"[{headline}]:")
+        print("\n".join(section_info))
+
+    print("-" * 130)