m3hrdadfi commited on
Commit
21ff7c3
·
1 Parent(s): 606a511

Fix data preparation

Browse files
src/preparaing_recipe_nlg_dataset.py CHANGED
@@ -81,7 +81,6 @@ def main():
81
  cache_dir=data_args.cache_dir
82
  )
83
 
84
-
85
  def cleaning(text, item_type="ner"):
86
  # NOTE: DO THE CLEANING LATER
87
  text = normalizer(text, do_lowercase=True)
@@ -93,9 +92,9 @@ def main():
93
  ingredients = item_dict["ingredients"]
94
  steps = item_dict["directions"]
95
 
96
- condition_1 = filter_by_item(ner, 3)
97
- condition_2 = filter_by_length(title, 3)
98
- condition_3 = filter_by_item(ingredients, 3)
99
  condition_4 = filter_by_item(steps, 2)
100
  condition_5 = filter_by_steps(" ".join(steps))
101
 
 
81
  cache_dir=data_args.cache_dir
82
  )
83
 
 
84
  def cleaning(text, item_type="ner"):
85
  # NOTE: DO THE CLEANING LATER
86
  text = normalizer(text, do_lowercase=True)
 
92
  ingredients = item_dict["ingredients"]
93
  steps = item_dict["directions"]
94
 
95
+ condition_1 = filter_by_item(ner, 2)
96
+ condition_2 = filter_by_length(title, 4)
97
+ condition_3 = filter_by_item(ingredients, 2)
98
  condition_4 = filter_by_item(steps, 2)
99
  condition_5 = filter_by_steps(" ".join(steps))
100