# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """TODO: Add a description here.""" from __future__ import annotations import re from collections import Counter, namedtuple from typing import Iterable from tqdm.auto import tqdm from sage.evaluation.ruerrant_wrapper import classifier from sage.evaluation.ruerrant_wrapper import merger from errant.annotator import Annotator from errant.commands.compare_m2 import process_edits from errant.commands.compare_m2 import evaluate_edits from errant.commands.compare_m2 import merge_dict from errant.edit import Edit import spacy from spacy.tokenizer import Tokenizer from spacy.util import compile_prefix_regex, compile_infix_regex, compile_suffix_regex import evaluate import datasets # TODO: Add BibTeX citation _CITATION = """\ @InProceedings{huggingface:module, title = {A great new module}, authors={huggingface, Inc.}, year={2020} } """ # TODO: Add description of the module here _DESCRIPTION = """\ This new module is designed to solve this great ML task and is crafted with a lot of care. """ # TODO: Add description of the arguments of the module here _KWARGS_DESCRIPTION = """ Calculates how good are predictions given some references, using certain scores Args: predictions: list of predictions to score. Each predictions should be a string with tokens separated by spaces. references: list of reference for each prediction. Each reference should be a string with tokens separated by spaces. Returns: accuracy: description of the first score, another_score: description of the second score, Examples: Examples should be written in doctest format, and should illustrate how to use the function. >>> my_new_module = evaluate.load("my_new_module") >>> results = my_new_module.compute(references=[0, 1], predictions=[0, 1]) >>> print(results) {'accuracy': 1.0} """ # TODO: Define external resources urls if needed BAD_WORDS_URL = "http://url/to/external/resource/bad_words.txt" def update_spacy_tokenizer(nlp): """ Changes Spacy tokenizer to parse additional patterns. """ infix_re = compile_infix_regex(nlp.Defaults.infixes[:-1] + ["\]\("]) simple_url_re = re.compile(r'''^https?://''') nlp.tokenizer = Tokenizer( nlp.vocab, prefix_search=compile_prefix_regex(nlp.Defaults.prefixes + ['\\\\\"']).search, suffix_search=compile_suffix_regex(nlp.Defaults.suffixes + ['\\\\']).search, infix_finditer=infix_re.finditer, token_match=None, url_match=simple_url_re.match ) return nlp def annotate_errors(self, orig: str, cor: str, merging: str = "rules") -> list[Edit]: """ Overrides `Annotator.annotate()` function to allow multiple errors per token. This is nesessary to parse combined errors, e.g.: ["werd", "Word"] >>> Errors: ["SPELL", "CASE"] The `classify()` method called inside is implemented in ruerrant_classifier.py (also overrides the original classifier). """ alignment = self.annotator.align(orig, cor, False) edits = self.annotator.merge(alignment, merging) classified_edits = [] for edit in edits: classified_edits.extend(self.annotator.classify(edit)) return sorted(classified_edits, key=lambda x: (x[0], x[2])) @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) class RuErrant(evaluate.Metric): """TODO: Short description of my evaluation module.""" def _info(self): # TODO: Specifies the evaluate.EvaluationModuleInfo object return evaluate.MetricInfo( # This is the description that will appear on the modules page. module_type="metric", description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, # This defines the format of each prediction and reference features=datasets.Features( { "sources": datasets.Value("string", id="sequence"), "corrections": datasets.Value("string", id="sequence"), "answers": datasets.Value("string", id="sequence"), } ), # Homepage of the module for documentation homepage="http://module.homepage", # Additional links to the codebase or references codebase_urls=["https://github.com/ai-forever/sage"], reference_urls=["http://path.to.reference.url/new_module"] ) def _download_and_prepare(self, dl_manager): self.annotator = Annotator("ru", nlp=update_spacy_tokenizer(spacy.load("ru_core_news_lg")), merger=merger, classifier=classifier) def _compute(self, sources, corrections, answers): """ Evaluates iterables of sources, hyp and ref corrections with ERRANT metric. Args: sources (Iterable[str]): an iterable of source texts; corrections (Iterable[str]): an iterable of gold corrections for the source texts; answers (Iterable[str]): an iterable of evaluated corrections for the source texts; Returns: dict[str, tuple[float, ...]]: a dict mapping error categories to the corresponding P, R, F1 metric values. """ best_dict = Counter({"tp": 0, "fp": 0, "fn": 0}) best_cats = {} sents = zip(sources, corrections, answers) pb = tqdm(sents, desc="Calculating errant metric", total=len(sources)) for sent_id, sent in enumerate(pb): src = self.annotator.parse(sent[0]) ref = self.annotator.parse(sent[1]) hyp = self.annotator.parse(sent[2]) # Align hyp and ref corrections and annotate errors hyp_edits = self.annotate_errors(src, hyp) ref_edits = self.annotate_errors(src, ref) # Process the edits for detection/correction based on args ProcessingArgs = namedtuple("ProcessingArgs", ["dt", "ds", "single", "multi", "filt", "cse"], defaults=[False, False, False, False, [], True]) processing_args = ProcessingArgs() hyp_dict = process_edits(hyp_edits, processing_args) ref_dict = process_edits(ref_edits, processing_args) # Evaluate edits and get best TP, FP, FN hyp+ref combo. EvaluationArgs = namedtuple("EvaluationArgs", ["beta", "verbose"], defaults=[1.0, False]) evaluation_args = EvaluationArgs() count_dict, cat_dict = evaluate_edits( hyp_dict, ref_dict, best_dict, sent_id, evaluation_args) # Merge these dicts with best_dict and best_cats best_dict += Counter(count_dict) # corpus-level TP, FP, FN best_cats = merge_dict(best_cats, cat_dict) # corpus-level errortype-wise TP, FP, FN cat_prf = {} for cat, values in best_cats.items(): tp, fp, fn = values # fp - extra corrections, fn - missed corrections p = float(tp) / (tp + fp) if tp + fp else 1.0 r = float(tp) / (tp + fn) if tp + fn else 1.0 f = (2 * p * r) / (p + r) if p + r else 0.0 cat_prf[cat] = (p, r, f) for error_category in ["CASE", "PUNCT", "SPELL", "YO"]: if error_category not in cat_prf: cat_prf[error_category] = (1.0, 1.0, 1.0) return cat_prf