dicta-il merge

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+vocab.txt filter=lfs diff=lfs merge=lfs -text

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

README.md

@@ -1,3 +1,591 @@
----
-license: cc-by-4.0
----
+---
+license: cc-by-4.0
+language:
+- he
+inference: false
+---
+# DictaBERT: A State-of-the-Art BERT Suite for Modern Hebrew
+
+State-of-the-art language model for parsing Hebrew, released [here](https://arxiv.org/abs/2403.06970).
+
+This is the fine-tuned model for the joint parsing of the following tasks: 
+
+- Prefix Segmentation
+- Morphological Disabmgiuation
+- Lexicographical Analysis (Lemmatization)
+- Syntactical Parsing (Dependency-Tree)
+- Named-Entity Recognition
+
+This model was initialized from dictabert-**large**-joint and tuned on the Hebrew UD Treebank and NEMO corpora, to align the predictions of the model to the tagging methodology in those corpora.
+
+A live demo of the `dictabert-joint` model with instant visualization of the syntax tree can be found [here](https://huggingface.co/spaces/dicta-il/joint-demo).
+
+For a faster model, you can use the equivalent bert-tiny model for this task [here](https://huggingface.co/dicta-il/dictabert-tiny-parse).
+
+For the bert-base models for other tasks, see [here](https://huggingface.co/collections/dicta-il/dictabert-6588e7cc08f83845fc42a18b).
+
+---
+
+The model currently supports 3 types of output:
+
+1. **JSON**: The model returns a JSON object for each sentence in the input, where for each sentence we have the sentence text, the NER entities, and the list of tokens. For each token we include the output from each of the tasks.
+    ```python
+    model.predict(..., output_style='json')
+    ```    
+
+1. **UD**: The model returns the full UD output for each sentence, according to the style of the Hebrew UD Treebank. 
+    ```python
+    model.predict(..., output_style='ud')
+    ```
+
+1. **UD, in the style of IAHLT**: This model returns the full UD output, with slight modifications to match the style of IAHLT. This differences are mostly granularity of some dependency relations, how the suffix of a word is broken up, and implicit definite articles. The actual tagging behavior doesn't change.
+    ```python
+    model.predict(..., output_style='iahlt_ud')
+    ```
+
+---
+
+If you only need the output for one of the tasks, you can tell the model to not initialize some of the heads, for example:
+```python
+model = AutoModel.from_pretrained('dicta-il/dictabert-parse', trust_remote_code=True, do_lex=False)
+```
+
+The list of options are: `do_lex`, `do_syntax`, `do_ner`, `do_prefix`, `do_morph`. 
+
+---
+
+Sample usage:
+
+```python
+from transformers import AutoModel, AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained('dicta-il/dictabert-large-parse')
+model = AutoModel.from_pretrained('dicta-il/dictabert-large-parse', trust_remote_code=True)
+
+model.eval()
+
+sentence = 'בשנת 1948 השלים אפרים קישון את לימודיו בפיסול מתכת ובתולדות האמנות והחל לפרסם מאמרים הומוריסטיים'
+print(model.predict([sentence], tokenizer, output_style='json')) # see below for other return formats
+```
+
+Output:
+```json
+[
+  {
+    "text": "בשנת 1948 השלים אפרים קישון את לימודיו בפיסול מתכת ובתולדות האמנות והחל לפרסם מאמרים הומוריסטיים",
+    "tokens": [
+      {
+        "token": "בשנת",
+        "offsets": {
+          "start": 0,
+          "end": 4
+        },
+        "syntax": {
+          "word": "בשנת",
+          "dep_head_idx": 2,
+          "dep_func": "obl",
+          "dep_head": "השלים"
+        },
+        "seg": [
+          "ב",
+          "שנת"
+        ],
+        "lex": "שנה",
+        "morph": {
+          "token": "בשנת",
+          "pos": "NOUN",
+          "feats": {
+            "Gender": "Fem",
+            "Number": "Sing"
+          },
+          "prefixes": [
+            "ADP"
+          ],
+          "suffix": false
+        }
+      },
+      {
+        "token": "1948",
+        "offsets": {
+          "start": 5,
+          "end": 9
+        },
+        "syntax": {
+          "word": "1948",
+          "dep_head_idx": 0,
+          "dep_func": "compound:smixut",
+          "dep_head": "בשנת"
+        },
+        "seg": [
+          "1948"
+        ],
+        "lex": "1948",
+        "morph": {
+          "token": "1948",
+          "pos": "NUM",
+          "feats": {},
+          "prefixes": [],
+          "suffix": false
+        }
+      },
+      {
+        "token": "השלים",
+        "offsets": {
+          "start": 10,
+          "end": 15
+        },
+        "syntax": {
+          "word": "השלים",
+          "dep_head_idx": -1,
+          "dep_func": "root",
+          "dep_head": "הומוריסטיים"
+        },
+        "seg": [
+          "השלים"
+        ],
+        "lex": "השלים",
+        "morph": {
+          "token": "השלים",
+          "pos": "VERB",
+          "feats": {
+            "Gender": "Masc",
+            "Number": "Sing",
+            "Person": "3",
+            "Tense": "Past"
+          },
+          "prefixes": [],
+          "suffix": false
+        }
+      },
+      {
+        "token": "אפרים",
+        "offsets": {
+          "start": 16,
+          "end": 21
+        },
+        "syntax": {
+          "word": "אפרים",
+          "dep_head_idx": 2,
+          "dep_func": "nsubj",
+          "dep_head": "השלים"
+        },
+        "seg": [
+          "אפרים"
+        ],
+        "lex": "אפרים",
+        "morph": {
+          "token": "אפרים",
+          "pos": "PROPN",
+          "feats": {},
+          "prefixes": [],
+          "suffix": false
+        }
+      },
+      {
+        "token": "קישון",
+        "offsets": {
+          "start": 22,
+          "end": 27
+        },
+        "syntax": {
+          "word": "קישון",
+          "dep_head_idx": 3,
+          "dep_func": "flat:name",
+          "dep_head": "אפרים"
+        },
+        "seg": [
+          "קישון"
+        ],
+        "lex": "קישון",
+        "morph": {
+          "token": "קישון",
+          "pos": "PROPN",
+          "feats": {},
+          "prefixes": [],
+          "suffix": false
+        }
+      },
+      {
+        "token": "את",
+        "offsets": {
+          "start": 28,
+          "end": 30
+        },
+        "syntax": {
+          "word": "את",
+          "dep_head_idx": 6,
+          "dep_func": "case:acc",
+          "dep_head": "לימודיו"
+        },
+        "seg": [
+          "את"
+        ],
+        "lex": "את",
+        "morph": {
+          "token": "את",
+          "pos": "ADP",
+          "feats": {},
+          "prefixes": [],
+          "suffix": false
+        }
+      },
+      {
+        "token": "לימודיו",
+        "offsets": {
+          "start": 31,
+          "end": 38
+        },
+        "syntax": {
+          "word": "לימודיו",
+          "dep_head_idx": 2,
+          "dep_func": "obj",
+          "dep_head": "השלים"
+        },
+        "seg": [
+          "לימודיו"
+        ],
+        "lex": "לימוד",
+        "morph": {
+          "token": "לימודיו",
+          "pos": "NOUN",
+          "feats": {
+            "Gender": "Masc",
+            "Number": "Plur"
+          },
+          "prefixes": [],
+          "suffix": "ADP_PRON",
+          "suffix_feats": {
+            "Gender": "Masc",
+            "Number": "Sing",
+            "Person": "3"
+          }
+        }
+      },
+      {
+        "token": "בפיסול",
+        "offsets": {
+          "start": 39,
+          "end": 45
+        },
+        "syntax": {
+          "word": "בפיסול",
+          "dep_head_idx": 6,
+          "dep_func": "nmod",
+          "dep_head": "לימודיו"
+        },
+        "seg": [
+          "ב",
+          "פיסול"
+        ],
+        "lex": "פיסול",
+        "morph": {
+          "token": "בפיסול",
+          "pos": "NOUN",
+          "feats": {
+            "Gender": "Masc",
+            "Number": "Sing"
+          },
+          "prefixes": [
+            "ADP"
+          ],
+          "suffix": false
+        }
+      },
+      {
+        "token": "מתכת",
+        "offsets": {
+          "start": 46,
+          "end": 50
+        },
+        "syntax": {
+          "word": "מתכת",
+          "dep_head_idx": 7,
+          "dep_func": "compound:smixut",
+          "dep_head": "בפיסול"
+        },
+        "seg": [
+          "מתכת"
+        ],
+        "lex": "מתכת",
+        "morph": {
+          "token": "מתכת",
+          "pos": "NOUN",
+          "feats": {
+            "Gender": "Fem",
+            "Number": "Sing"
+          },
+          "prefixes": [],
+          "suffix": false
+        }
+      },
+      {
+        "token": "ובתולדות",
+        "offsets": {
+          "start": 51,
+          "end": 59
+        },
+        "syntax": {
+          "word": "ובתולדות",
+          "dep_head_idx": 7,
+          "dep_func": "conj",
+          "dep_head": "בפיסול"
+        },
+        "seg": [
+          "וב",
+          "תולדות"
+        ],
+        "lex": "תולדה",
+        "morph": {
+          "token": "ובתולדות",
+          "pos": "NOUN",
+          "feats": {
+            "Gender": "Fem",
+            "Number": "Plur"
+          },
+          "prefixes": [
+            "CCONJ",
+            "ADP"
+          ],
+          "suffix": false
+        }
+      },
+      {
+        "token": "האמנות",
+        "offsets": {
+          "start": 60,
+          "end": 66
+        },
+        "syntax": {
+          "word": "האמנות",
+          "dep_head_idx": 9,
+          "dep_func": "compound:smixut",
+          "dep_head": "ובתולדות"
+        },
+        "seg": [
+          "ה",
+          "אמנות"
+        ],
+        "lex": "אומנות",
+        "morph": {
+          "token": "האמנות",
+          "pos": "NOUN",
+          "feats": {
+            "Gender": "Fem",
+            "Number": "Sing"
+          },
+          "prefixes": [
+            "DET"
+          ],
+          "suffix": false
+        }
+      },
+      {
+        "token": "והחל",
+        "offsets": {
+          "start": 67,
+          "end": 71
+        },
+        "syntax": {
+          "word": "והחל",
+          "dep_head_idx": 2,
+          "dep_func": "conj",
+          "dep_head": "השלים"
+        },
+        "seg": [
+          "ו",
+          "החל"
+        ],
+        "lex": "החל",
+        "morph": {
+          "token": "והחל",
+          "pos": "VERB",
+          "feats": {
+            "Gender": "Masc",
+            "Number": "Sing",
+            "Person": "3",
+            "Tense": "Past"
+          },
+          "prefixes": [
+            "CCONJ"
+          ],
+          "suffix": false
+        }
+      },
+      {
+        "token": "לפרסם",
+        "offsets": {
+          "start": 72,
+          "end": 77
+        },
+        "syntax": {
+          "word": "לפרסם",
+          "dep_head_idx": 11,
+          "dep_func": "xcomp",
+          "dep_head": "והחל"
+        },
+        "seg": [
+          "לפרסם"
+        ],
+        "lex": "פרסם",
+        "morph": {
+          "token": "לפרסם",
+          "pos": "VERB",
+          "feats": {},
+          "prefixes": [],
+          "suffix": false
+        }
+      },
+      {
+        "token": "מאמרים",
+        "offsets": {
+          "start": 78,
+          "end": 84
+        },
+        "syntax": {
+          "word": "מאמרים",
+          "dep_head_idx": 12,
+          "dep_func": "obj",
+          "dep_head": "לפרסם"
+        },
+        "seg": [
+          "מאמרים"
+        ],
+        "lex": "מאמר",
+        "morph": {
+          "token": "מאמרים",
+          "pos": "NOUN",
+          "feats": {
+            "Gender": "Masc",
+            "Number": "Plur"
+          },
+          "prefixes": [],
+          "suffix": false
+        }
+      },
+      {
+        "token": "הומוריסטיים",
+        "offsets": {
+          "start": 85,
+          "end": 96
+        },
+        "syntax": {
+          "word": "הומוריסטיים",
+          "dep_head_idx": 13,
+          "dep_func": "amod",
+          "dep_head": "מאמרים"
+        },
+        "seg": [
+          "הומוריסטיים"
+        ],
+        "lex": "הומוריסטי",
+        "morph": {
+          "token": "הומוריסטיים",
+          "pos": "ADJ",
+          "feats": {
+            "Gender": "Masc",
+            "Number": "Plur"
+          },
+          "prefixes": [],
+          "suffix": false
+        }
+      }
+    ],
+    "root_idx": 2,
+    "ner_entities": [
+      {
+        "phrase": "1948",
+        "label": "TIMEX",
+        "start": 5,
+        "end": 9,
+        "token_start": 1,
+        "token_end": 1
+      },
+      {
+        "phrase": "אפרים קישון",
+        "label": "PER",
+        "start": 16,
+        "end": 27,
+        "token_start": 3,
+        "token_end": 4
+      }
+    ]
+  }
+]
+```
+
+You can also choose to get your response in UD format:
+
+```python
+sentence = 'בשנת 1948 השלים אפרים קישון את לימודיו בפיסול מתכת ובתולדות האמנות והחל לפרסם מאמרים הומוריסטיים'
+print(model.predict([sentence], tokenizer, output_style='ud')) 
+```
+
+Results:
+```json
+[
+  [
+    "# sent_id = 1",
+    "# text = בשנת 1948 השלים אפרים קישון את לימודיו בפיסול מתכת ובתולדות האמנות והחל לפרסם מאמרים הומוריסטיים",
+    "1-2\tבשנת\t_\t_\t_\t_\t_\t_\t_\t_",
+    "1\tב\tב\tADP\tADP\t_\t2\tcase\t_\t_",
+    "2\tשנת\tשנה\tNOUN\tNOUN\tGender=Fem|Number=Sing\t4\tobl\t_\t_",
+    "3\t1948\t1948\tNUM\tNUM\t\t2\tcompound:smixut\t_\t_",
+    "4\tהשלים\tהשלים\tVERB\tVERB\tGender=Masc|Number=Sing|Person=3|Tense=Past\t0\troot\t_\t_",
+    "5\tאפרים\tאפרים\tPROPN\tPROPN\t\t4\tnsubj\t_\t_",
+    "6\tקישון\tקישון\tPROPN\tPROPN\t\t5\tflat:name\t_\t_",
+    "7\tאת\tאת\tADP\tADP\t\t8\tcase:acc\t_\t_",
+    "8-10\tלימודיו\t_\t_\t_\t_\t_\t_\t_\t_",
+    "8\tלימוד_\tלימוד\tNOUN\tNOUN\tGender=Masc|Number=Plur\t4\tobj\t_\t_",
+    "9\t_של_\tשל\tADP\tADP\t_\t10\tcase\t_\t_",
+    "10\t_הוא\tהוא\tPRON\tPRON\tGender=Masc|Number=Sing|Person=3\t8\tnmod:poss\t_\t_",
+    "11-12\tבפיסול\t_\t_\t_\t_\t_\t_\t_\t_",
+    "11\tב\tב\tADP\tADP\t_\t12\tcase\t_\t_",
+    "12\tפיסול\tפיסול\tNOUN\tNOUN\tGender=Masc|Number=Sing\t8\tnmod\t_\t_",
+    "13\tמתכת\tמתכת\tNOUN\tNOUN\tGender=Fem|Number=Sing\t12\tcompound:smixut\t_\t_",
+    "14-16\tובתולדות\t_\t_\t_\t_\t_\t_\t_\t_",
+    "14\tו\tו\tCCONJ\tCCONJ\t_\t16\tcc\t_\t_",
+    "15\tב\tב\tADP\tADP\t_\t16\tcase\t_\t_",
+    "16\tתולדות\tתולדה\tNOUN\tNOUN\tGender=Fem|Number=Plur\t12\tconj\t_\t_",
+    "17-18\tהאמנות\t_\t_\t_\t_\t_\t_\t_\t_",
+    "17\tה\tה\tDET\tDET\t_\t18\tdet\t_\t_",
+    "18\tאמנות\tאומנות\tNOUN\tNOUN\tGender=Fem|Number=Sing\t16\tcompound:smixut\t_\t_",
+    "19-20\tוהחל\t_\t_\t_\t_\t_\t_\t_\t_",
+    "19\tו\tו\tCCONJ\tCCONJ\t_\t20\tcc\t_\t_",
+    "20\tהחל\tהחל\tVERB\tVERB\tGender=Masc|Number=Sing|Person=3|Tense=Past\t4\tconj\t_\t_",
+    "21\tלפרסם\tפרסם\tVERB\tVERB\t\t20\txcomp\t_\t_",
+    "22\tמאמרים\tמאמר\tNOUN\tNOUN\tGender=Masc|Number=Plur\t21\tobj\t_\t_",
+    "23\tהומוריסטיים\tהומוריסטי\tADJ\tADJ\tGender=Masc|Number=Plur\t22\tamod\t_\t_"
+  ]
+]
+```
+
+## Citation
+
+If you use DictaBERT-large-parse in your research, please cite ```MRL Parsing without Tears: The Case of Hebrew```
+
+**BibTeX:**
+
+```bibtex
+@misc{shmidman2024mrl,
+      title={MRL Parsing Without Tears: The Case of Hebrew}, 
+      author={Shaltiel Shmidman and Avi Shmidman and Moshe Koppel and Reut Tsarfaty},
+      year={2024},
+      eprint={2403.06970},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+```
+
+
+## License
+
+Shield: [![CC BY 4.0][cc-by-shield]][cc-by]
+
+This work is licensed under a
+[Creative Commons Attribution 4.0 International License][cc-by].
+
+[![CC BY 4.0][cc-by-image]][cc-by]
+
+[cc-by]: http://creativecommons.org/licenses/by/4.0/
+[cc-by-image]: https://i.creativecommons.org/l/by/4.0/88x31.png
+[cc-by-shield]: https://img.shields.io/badge/License-CC%20BY%204.0-lightgrey.svg

config.json

@@ -0,0 +1,92 @@
+{
+  "architectures": [
+    "BertForJointParsing"
+  ],
+  "auto_map": {
+    "AutoModel": "dicta-il/dictabert-joint--BertForJointParsing.BertForJointParsing"
+  },
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "do_lex": true,
+  "do_morph": true,
+  "do_ner": true,
+  "do_prefix": true,
+  "do_syntax": true,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1024,
+  "id2label": {
+    "0": "B-ANG",
+    "1": "B-DUC",
+    "2": "B-EVE",
+    "3": "B-FAC",
+    "4": "B-GPE",
+    "5": "B-LOC",
+    "6": "B-ORG",
+    "7": "B-PER",
+    "8": "B-WOA",
+    "9": "B-INFORMAL",
+    "10": "B-MISC",
+    "11": "B-TIMEX",
+    "12": "B-TTL",
+    "13": "I-DUC",
+    "14": "I-EVE",
+    "15": "I-FAC",
+    "16": "I-GPE",
+    "17": "I-LOC",
+    "18": "I-ORG",
+    "19": "I-PER",
+    "20": "I-WOA",
+    "21": "I-ANG",
+    "22": "I-INFORMAL",
+    "23": "I-MISC",
+    "24": "I-TIMEX",
+    "25": "I-TTL",
+    "26": "O"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "label2id": {
+    "B-ANG": 0,
+    "B-DUC": 1,
+    "B-EVE": 2,
+    "B-FAC": 3,
+    "B-GPE": 4,
+    "B-INFORMAL": 9,
+    "B-LOC": 5,
+    "B-MISC": 10,
+    "B-ORG": 6,
+    "B-PER": 7,
+    "B-TIMEX": 11,
+    "B-TTL": 12,
+    "B-WOA": 8,
+    "I-ANG": 21,
+    "I-DUC": 13,
+    "I-EVE": 14,
+    "I-FAC": 15,
+    "I-GPE": 16,
+    "I-INFORMAL": 22,
+    "I-LOC": 17,
+    "I-MISC": 23,
+    "I-ORG": 18,
+    "I-PER": 19,
+    "I-TIMEX": 24,
+    "I-TTL": 25,
+    "I-WOA": 20,
+    "O": 26
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "newmodern": true,
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "syntax_head_size": 128,
+  "torch_dtype": "float32",
+  "transformers_version": "4.36.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 128000
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d25a305e90a34d3f74d8651651af2575be03fcee631a239a35fb538a13ed8158
+size 1750709384

special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,63 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "[BLANK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

vocab.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fb90bfa35244d26f0065d1fcd0b5becc3da3d44d616a7e2aacaf6320b9fa2d0
+size 1500244