readme and gpu

generation_evaluator.py

@@ -2,6 +2,7 @@ import datasets
 import evaluate
 import numpy as np
 import spacy
+import torch
 from alignscore import AlignScore
 
 _CITATION = """\
@@ -53,38 +54,19 @@ _CITATION = """\
 """
 
 _DESCRIPTION = """\
-ROUGE, or Recall-Oriented Understudy for Gisting Evaluation, is a set of metrics and a software package used for
-evaluating automatic summarization and machine translation software in natural language processing.
-The metrics compare an automatically produced summary or translation against a reference or a set of references (human-produced) summary or translation.
+This evaluator computes multiple metrics to assess the quality of generated text. These metrics are the following:
 
-Note that ROUGE is case insensitive, meaning that upper case letters are treated the same way as lower case letters.
+- **ROUGE**: a set of metrics and a software package used for evaluating automatic summarization and machine translation software in natural language processing. The metrics compare an automatically produced summary or translation against a reference or a set of references (human-produced) summary or translation. Note that ROUGE is case insensitive, meaning that upper case letters are treated the same way as lower case letters. This metrics is a wrapper around Google Research reimplementation of ROUGE: https://github.com/google-research/google-research/tree/master/rouge
 
-This metrics is a wrapper around Google Research reimplementation of ROUGE:
-https://github.com/google-research/google-research/tree/master/rouge
+- **BLEU**: evaluates the quality of text which has been machine-translated from one natural language to another. Quality is considered to be the correspondence between a machine's output and that of a human: "the closer a machine translation is to a professional human translation, the better it is" this is the central idea behind BLEU. BLEU was one of the first metrics to claim a high correlation with human judgements of quality, and remains one of the most popular automated and inexpensive metrics. Scores are calculated for individual translated segments—generally sentences—by comparing them with a set of good quality reference translations. Those scores are then averaged over the whole corpus to reach an estimate of the translation's overall quality. Neither intelligibility nor grammatical correctness are not taken into account.
 
-BLEU (Bilingual Evaluation Understudy) is an algorithm for evaluating the quality of text which has been machine-translated from one natural language to another.
-Quality is considered to be the correspondence between a machine's output and that of a human: "the closer a machine translation is to a professional human translation, the better it is"
-this is the central idea behind BLEU. BLEU was one of the first metrics to claim a high correlation with human judgements of quality, and remains one of the most popular automated and inexpensive metrics.
+- **Exact Match**: rate at which the input predicted strings exactly match their references, ignoring any strings input as part of the regexes_to_ignore list.
 
-Scores are calculated for individual translated segments—generally sentences—by comparing them with a set of good quality reference translations.
-Those scores are then averaged over the whole corpus to reach an estimate of the translation's overall quality.
-Neither intelligibility nor grammatical correctness are not taken into account.
+- **BERTScore**: leverages the pre-trained contextual embeddings from BERT and matches words in candidate and reference sentences by cosine similarity. It has been shown to correlate with human judgment on sentence-level and system-level evaluation. Moreover, BERTScore computes precision, recall, and F1 measure, which can be useful for evaluating different language generation tasks. See the project's README at https://github.com/Tiiiger/bert_score#readme for more information.
 
-EXACT MATCH: Returns the rate at which the input predicted strings exactly match their references, ignoring any strings input as part of the regexes_to_ignore list.
+- **AlignScore**: evaluates whether all the information in a piece of text *b* is contained in another piece of text *a* and *b* does not contradict *a*, by leveraging an information alignment function learnt through RoBERTa models. See https://github.com/yuh-zha/AlignScore for more information.
 
-BERTScore leverages the pre-trained contextual embeddings from BERT and matches words in candidate and reference
-sentences by cosine similarity.
-It has been shown to correlate with human judgment on sentence-level and system-level evaluation.
-Moreover, BERTScore computes precision, recall, and F1 measure, which can be useful for evaluating different language
-generation tasks.
-See the project's README at https://github.com/Tiiiger/bert_score#readme for more information.
-
-AlignScore evaluates whether all the information in b is contained in a (b does not contradict a).
-See https://github.com/yuh-zha/AlignScore for more information.
-
-ChrF and ChrF++ are two MT evaluation metrics. They both use the F-score statistic for character n-gram matches,
-and ChrF++ adds word n-grams as well which correlates more strongly with direct assessment. We use the implementation
-that is already present in sacrebleu.
+- **ChrF and ChrF++**: are two MT evaluation metrics. They both use the F-score statistic for character n-gram matches, and ChrF++ adds word n-grams as well which correlates more strongly with direct assessment. We use the implementation that is already present in sacrebleu.
 """
 
 _KWARGS_DESCRIPTION = """
@@ -120,7 +102,7 @@ BERT_SCORE:{
     "hashcode": Hashcode of the library.
 },
 AlignScore:{
-    "score": mean align scores using roberta-base as scorer
+    "score": mean align-scores using roberta-large as scorer
 },
 CHRF:{
     'score' (float): The chrF (chrF++) score,
@@ -131,10 +113,9 @@ CHRF:{
 """
 
 ALIGNSCORE_ARGS = {
-    "model": "roberta-base",
+    "model": "roberta-large",
     "batch_size": 32,
-    "device": "cuda",
-    "ckpt_path": "https://huggingface.co/yzha/AlignScore/resolve/main/AlignScore-base.ckpt",
+    "ckpt_path": "https://huggingface.co/yzha/AlignScore/resolve/main/AlignScore-large.ckpt",
     "evaluation_mode": "nli_sp",
 }
 
@@ -167,9 +148,12 @@ class GenerationEvaluator(evaluate.Metric):
         except OSError:
             spacy.cli.download("en_core_web_sm")
 
-        # Download AlignScore checkpoint
+        # Download AlignScore model and move to GPU if possible
         model_path = dl_manager.download(ALIGNSCORE_ARGS["ckpt_path"])
         ALIGNSCORE_ARGS["ckpt_path"] = model_path
+        ALIGNSCORE_ARGS["device"] = (
+            "cuda:0" if torch.cuda.is_available() else "cpu"
+        )
         self.align_scorer = AlignScore(**ALIGNSCORE_ARGS)
 
     def _compute(self, predictions, references):

gradio_tst.py

@@ -128,8 +128,7 @@ def launch_gradio_widget2(metric):
         outputs=gr.Textbox(label=metric.name),
         description=(
             metric.info.description
-            + "\nIf this is a text-based metric, make sure to wrap you input in double quotes."
-            " Alternatively you can use a JSON-formatted list as input."
+            + "\nThis is a text-based metric, so make sure to wrap you input in double quotes."
         ),
         title=f"Metric: {metric.name}",
         article=parse_readme(local_path / "README.md"),