|
#! /bin/sh |
|
S=KoichiYasuoka/modernbert-base-thai-wikipedia |
|
T=KoichiYasuoka/modernbert-base-thai-wikipedia-upos |
|
D=spaCy-Thai/UD_Thai-Corpora |
|
test -d $D || git clone --depth=1 https://github.com/KoichiYasuoka/spaCy-Thai |
|
nawk ' |
|
BEGIN{ |
|
FS=OFS="\t"; |
|
} |
|
{ |
|
if(NF==10&&$1~/^[1-9][0-9]*$/) |
|
u=u$0"\n"; |
|
else if($0~/^# text =/) |
|
u=u$0"\n"; |
|
else if($0==""){ |
|
f=(FILENAME~/test/)?"test":(FILENAME~/dev/)?"dev":"train"; |
|
print u>f".upos"; |
|
if(u~/\t0\troot\t/) |
|
print u>f".conllu"; |
|
u=""; |
|
} |
|
}' $D/*-ud-*.conllu |
|
python3 -m esupar.train $S $T -1 /tmp train.upos |
|
python3 -m esupar.train $T $T 32 /// train.conllu dev.conllu test.conllu |
|
|
