initial release

README.md

@@ -0,0 +1,58 @@
+---
+language:
+- "ja"
+tags:
+- "japanese"
+- "wikipedia"
+- "pos"
+- "dependency-parsing"
+datasets:
+- "universal_dependencies"
+license: "cc-by-sa-4.0"
+pipeline_tag: "token-classification"
+widget:
+- text: "全学年にわたって小学校の国語の教科書に挿し絵が用いられている"
+---
+
+# deberta-large-japanese-wikipedia-ud-goeswith
+
+## Model Description
+
+This is a DeBERTa(V2) model pretrained on Japanese Wikipedia and 青空文庫 texts for POS-tagging and dependency-parsing (using `goeswith` for subwords), derived from [deberta-large-japanese-wikipedia](https://huggingface.co/KoichiYasuoka/deberta-large-japanese-wikipedia) and [UD_Japanese-GSDLUW](https://github.com/UniversalDependencies/UD_Japanese-GSDLUW).
+
+## How to Use
+
+```py
+class UDgoeswith(object):
+  def __init__(self,bert):
+    from transformers import AutoTokenizer,AutoModelForTokenClassification
+    self.tokenizer=AutoTokenizer.from_pretrained(bert)
+    self.model=AutoModelForTokenClassification.from_pretrained(bert)
+  def __call__(self,text):
+    import numpy,torch,ufal.chu_liu_edmonds
+    w=self.tokenizer(text,return_offsets_mapping=True)
+    v=w["input_ids"]
+    n=len(v)-1
+    with torch.no_grad():
+      d=self.model(input_ids=torch.tensor([v[0:i]+[self.tokenizer.mask_token_id]+v[i+1:]+[v[i]] for i in range(1,n)]))
+    e=d.logits.numpy()[:,1:n,:]
+    e[:,:,0]=numpy.nan
+    m=numpy.full((n,n),numpy.nan)
+    m[1:,1:]=numpy.nanmax(e,axis=2).transpose()
+    p=numpy.zeros((n,n))
+    p[1:,1:]=numpy.nanargmax(e,axis=2).transpose()
+    for i in range(1,n):
+      m[i,0],m[i,i],p[i,0]=m[i,i],numpy.nan,p[i,i]
+    h=ufal.chu_liu_edmonds.chu_liu_edmonds(m)[0]
+    u="# text = "+text+"\n"
+    v=[(s,e) for s,e in w["offset_mapping"] if s<e]
+    for i,(s,e) in enumerate(v,1):
+      q=self.model.config.id2label[p[i,h[i]]].split("|")
+      u+="\t".join([str(i),text[s:e],"_",q[0],"_","|".join(q[1:-1]),str(h[i]),q[-1],"_","_" if i<len(v) and e<v[i][0] else "SpaceAfter=No"])+"\n"
+    return u+"\n"
+
+nlp=UDgoeswith("KoichiYasuoka/deberta-large-japanese-wikipedia-ud-goeswith")
+print(nlp("全学年にわたって小学校の国語の教科書に挿し絵が用いられている"))
+```
+
+[ufal.chu-liu-edmonds](https://pypi.org/project/ufal.chu-liu-edmonds/) is required.

config.json

@@ -0,0 +1,211 @@
+{
+  "architectures": [
+    "DebertaV2ForTokenClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1024,
+  "id2label": {
+    "0": "-|_|dep",
+    "1": "ADJ|_|acl",
+    "2": "ADJ|_|advcl",
+    "3": "ADJ|_|amod",
+    "4": "ADJ|_|ccomp",
+    "5": "ADJ|_|csubj",
+    "6": "ADJ|_|dep",
+    "7": "ADJ|_|dislocated",
+    "8": "ADJ|_|nmod",
+    "9": "ADJ|_|nsubj",
+    "10": "ADJ|_|obj",
+    "11": "ADJ|_|obl",
+    "12": "ADJ|_|root",
+    "13": "ADP|_|case",
+    "14": "ADP|_|fixed",
+    "15": "ADV|_|advcl",
+    "16": "ADV|_|advmod",
+    "17": "ADV|_|dep",
+    "18": "ADV|_|obj",
+    "19": "ADV|_|root",
+    "20": "AUX|Polarity=Neg|aux",
+    "21": "AUX|_|aux",
+    "22": "AUX|_|cop",
+    "23": "AUX|_|fixed",
+    "24": "AUX|_|root",
+    "25": "CCONJ|_|cc",
+    "26": "DET|_|det",
+    "27": "INTJ|_|discourse",
+    "28": "INTJ|_|root",
+    "29": "NOUN|Polarity=Neg|obl",
+    "30": "NOUN|Polarity=Neg|root",
+    "31": "NOUN|_|acl",
+    "32": "NOUN|_|advcl",
+    "33": "NOUN|_|ccomp",
+    "34": "NOUN|_|compound",
+    "35": "NOUN|_|csubj",
+    "36": "NOUN|_|dislocated",
+    "37": "NOUN|_|nmod",
+    "38": "NOUN|_|nsubj",
+    "39": "NOUN|_|obj",
+    "40": "NOUN|_|obl",
+    "41": "NOUN|_|root",
+    "42": "NUM|_|advcl",
+    "43": "NUM|_|compound",
+    "44": "NUM|_|dislocated",
+    "45": "NUM|_|nmod",
+    "46": "NUM|_|nsubj",
+    "47": "NUM|_|nummod",
+    "48": "NUM|_|obj",
+    "49": "NUM|_|obl",
+    "50": "NUM|_|root",
+    "51": "PART|_|mark",
+    "52": "PRON|_|acl",
+    "53": "PRON|_|advcl",
+    "54": "PRON|_|dislocated",
+    "55": "PRON|_|nmod",
+    "56": "PRON|_|nsubj",
+    "57": "PRON|_|obj",
+    "58": "PRON|_|obl",
+    "59": "PRON|_|root",
+    "60": "PROPN|_|acl",
+    "61": "PROPN|_|advcl",
+    "62": "PROPN|_|compound",
+    "63": "PROPN|_|dislocated",
+    "64": "PROPN|_|nmod",
+    "65": "PROPN|_|nsubj",
+    "66": "PROPN|_|obj",
+    "67": "PROPN|_|obl",
+    "68": "PROPN|_|root",
+    "69": "PUNCT|_|punct",
+    "70": "SCONJ|_|mark",
+    "71": "SYM|_|compound",
+    "72": "SYM|_|dep",
+    "73": "SYM|_|nmod",
+    "74": "SYM|_|obl",
+    "75": "VERB|_|acl",
+    "76": "VERB|_|advcl",
+    "77": "VERB|_|ccomp",
+    "78": "VERB|_|compound",
+    "79": "VERB|_|csubj",
+    "80": "VERB|_|dislocated",
+    "81": "VERB|_|nmod",
+    "82": "VERB|_|obj",
+    "83": "VERB|_|obl",
+    "84": "VERB|_|root",
+    "85": "X|_|dep",
+    "86": "X|_|goeswith",
+    "87": "X|_|nmod"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "label2id": {
+    "-|_|dep": 0,
+    "ADJ|_|acl": 1,
+    "ADJ|_|advcl": 2,
+    "ADJ|_|amod": 3,
+    "ADJ|_|ccomp": 4,
+    "ADJ|_|csubj": 5,
+    "ADJ|_|dep": 6,
+    "ADJ|_|dislocated": 7,
+    "ADJ|_|nmod": 8,
+    "ADJ|_|nsubj": 9,
+    "ADJ|_|obj": 10,
+    "ADJ|_|obl": 11,
+    "ADJ|_|root": 12,
+    "ADP|_|case": 13,
+    "ADP|_|fixed": 14,
+    "ADV|_|advcl": 15,
+    "ADV|_|advmod": 16,
+    "ADV|_|dep": 17,
+    "ADV|_|obj": 18,
+    "ADV|_|root": 19,
+    "AUX|Polarity=Neg|aux": 20,
+    "AUX|_|aux": 21,
+    "AUX|_|cop": 22,
+    "AUX|_|fixed": 23,
+    "AUX|_|root": 24,
+    "CCONJ|_|cc": 25,
+    "DET|_|det": 26,
+    "INTJ|_|discourse": 27,
+    "INTJ|_|root": 28,
+    "NOUN|Polarity=Neg|obl": 29,
+    "NOUN|Polarity=Neg|root": 30,
+    "NOUN|_|acl": 31,
+    "NOUN|_|advcl": 32,
+    "NOUN|_|ccomp": 33,
+    "NOUN|_|compound": 34,
+    "NOUN|_|csubj": 35,
+    "NOUN|_|dislocated": 36,
+    "NOUN|_|nmod": 37,
+    "NOUN|_|nsubj": 38,
+    "NOUN|_|obj": 39,
+    "NOUN|_|obl": 40,
+    "NOUN|_|root": 41,
+    "NUM|_|advcl": 42,
+    "NUM|_|compound": 43,
+    "NUM|_|dislocated": 44,
+    "NUM|_|nmod": 45,
+    "NUM|_|nsubj": 46,
+    "NUM|_|nummod": 47,
+    "NUM|_|obj": 48,
+    "NUM|_|obl": 49,
+    "NUM|_|root": 50,
+    "PART|_|mark": 51,
+    "PRON|_|acl": 52,
+    "PRON|_|advcl": 53,
+    "PRON|_|dislocated": 54,
+    "PRON|_|nmod": 55,
+    "PRON|_|nsubj": 56,
+    "PRON|_|obj": 57,
+    "PRON|_|obl": 58,
+    "PRON|_|root": 59,
+    "PROPN|_|acl": 60,
+    "PROPN|_|advcl": 61,
+    "PROPN|_|compound": 62,
+    "PROPN|_|dislocated": 63,
+    "PROPN|_|nmod": 64,
+    "PROPN|_|nsubj": 65,
+    "PROPN|_|obj": 66,
+    "PROPN|_|obl": 67,
+    "PROPN|_|root": 68,
+    "PUNCT|_|punct": 69,
+    "SCONJ|_|mark": 70,
+    "SYM|_|compound": 71,
+    "SYM|_|dep": 72,
+    "SYM|_|nmod": 73,
+    "SYM|_|obl": 74,
+    "VERB|_|acl": 75,
+    "VERB|_|advcl": 76,
+    "VERB|_|ccomp": 77,
+    "VERB|_|compound": 78,
+    "VERB|_|csubj": 79,
+    "VERB|_|dislocated": 80,
+    "VERB|_|nmod": 81,
+    "VERB|_|obj": 82,
+    "VERB|_|obl": 83,
+    "VERB|_|root": 84,
+    "X|_|dep": 85,
+    "X|_|goeswith": 86,
+    "X|_|nmod": 87
+  },
+  "layer_norm_eps": 1e-07,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "pad_token_id": 1,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 1024,
+  "pos_att_type": null,
+  "position_biased_input": true,
+  "relative_attention": false,
+  "tokenizer_class": "DebertaV2TokenizerFast",
+  "torch_dtype": "float32",
+  "transformers_version": "4.22.0",
+  "type_vocab_size": 0,
+  "vocab_size": 32000
+}

maker.py

@@ -0,0 +1,54 @@
+#! /usr/bin/python3
+src="KoichiYasuoka/deberta-large-japanese-wikipedia"
+tgt="KoichiYasuoka/deberta-large-japanese-wikipedia-ud-goeswith"
+url="https://github.com/UniversalDependencies/UD_Japanese-GSDLUW"
+import os
+d=os.path.basename(url)
+os.system("test -d "+d+" || git clone --depth=1 "+url)
+os.system("for F in train dev test ; do cp "+d+"/*-$F.conllu $F.conllu ; done")
+class UDgoeswithDataset(object):
+  def __init__(self,conllu,tokenizer):
+    self.ids,self.tags,label=[],[],set()
+    with open(conllu,"r",encoding="utf-8") as r:
+      cls,sep,msk=tokenizer.cls_token_id,tokenizer.sep_token_id,tokenizer.mask_token_id
+      dep,c="-|_|dep",[]
+      for s in r:
+        t=s.split("\t")
+        if len(t)==10 and t[0].isdecimal():
+          c.append(t)
+        elif c!=[]:
+          v=tokenizer([t[1] for t in c],add_special_tokens=False)["input_ids"]
+          for i in range(len(v)-1,-1,-1):
+            for j in range(1,len(v[i])):
+              c.insert(i+1,[c[i][0],"_","_","X","_","_",c[i][0],"goeswith","_","_"])
+          y=["0"]+[t[0] for t in c]
+          h=[i if t[6]=="0" else y.index(t[6]) for i,t in enumerate(c,1)]
+          p,v=[t[3]+"|"+t[5]+"|"+t[7] for t in c],sum(v,[])
+          self.ids.append([cls]+v+[sep])
+          self.tags.append([dep]+p+[dep])
+          label=set(sum([self.tags[-1],list(label)],[]))
+          for i,k in enumerate(v):
+            self.ids.append([cls]+v[0:i]+[msk]+v[i+1:]+[sep,k])
+            self.tags.append([dep]+[t if h[j]==i+1 else dep for j,t in enumerate(p)]+[dep,dep])
+          c=[]
+    self.label2id={l:i for i,l in enumerate(sorted(label))}
+  def __call__(*args):
+    label=set(sum([list(t.label2id) for t in args],[]))
+    lid={l:i for i,l in enumerate(sorted(label))}
+    for t in args:
+      t.label2id=lid
+    return lid
+  __len__=lambda self:len(self.ids)
+  __getitem__=lambda self,i:{"input_ids":self.ids[i],"labels":[self.label2id[t] for t in self.tags[i]]}
+from transformers import AutoTokenizer,AutoConfig,AutoModelForTokenClassification,DataCollatorForTokenClassification,TrainingArguments,Trainer
+tkz=AutoTokenizer.from_pretrained(src)
+trainDS=UDgoeswithDataset("train.conllu",tkz)
+devDS=UDgoeswithDataset("dev.conllu",tkz)
+testDS=UDgoeswithDataset("test.conllu",tkz)
+lid=trainDS(devDS,testDS)
+cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()})
+arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=32,output_dir="/tmp",overwrite_output_dir=True,save_total_limit=2,evaluation_strategy="epoch",learning_rate=5e-05,warmup_ratio=0.1)
+trn=Trainer(args=arg,data_collator=DataCollatorForTokenClassification(tkz),model=AutoModelForTokenClassification.from_pretrained(src,config=cfg),train_dataset=trainDS,eval_dataset=devDS)
+trn.train()
+trn.save_model(tgt)
+tkz.save_pretrained(tgt)

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e84633943bc3d30811d4aa0c0248417b8423927962e125c7dd08930a3a69a37e
+size 1342912499

special_tokens_map.json

@@ -0,0 +1,9 @@
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

spm.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b
+size 1

tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "keep_accents": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "split_by_punct": true,
+  "tokenizer_class": "DebertaV2TokenizerFast",
+  "unk_token": "[UNK]"
+}