Upload tokenizer

Browse files

Files changed (7) hide show

.gitattributes +1 -0
README.md +19 -23
added_tokens.json +24 -0
special_tokens_map.json +23 -0
spiece.model +3 -0
tokenizer.json +3 -0
tokenizer_config.json +215 -0

.gitattributes CHANGED Viewed

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text

 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md CHANGED Viewed

@@ -1,36 +1,32 @@
 ---
 license: apache-2.0
-widget:
-- text: >-
-    <Disease> Torsade de pointes ventricular tachycardia during low dose
-    intermittent dobutamine treatment in a patient with dilated cardiomyopathy
-    and congestive heart failure .
-- text: >-
-    <ClinicalEntity> Ecográficamente se observan tres nódulos tumorales
-    independientes y bien delimitados : dos de ellos heterogéneos , sólidos , de
-    20 y 33 mm de diámetros , con áreas quísticas y calcificaciones .
-- text: >-
-    <ClinicalEntity> On notait une hyperlordose lombaire avec une contracture
-    permanente des muscles paravertébraux , de l abdomen et des deux membres
-    inférieurs .
-- text: >-
-    <ClinicalEntity> Nell ’ anamnesi patologica era riferita ipertensione
-    arteriosa controllata con terapia medica
 library_name: transformers
-pipeline_tag: text2text-generation
 tags:
 - medical
 - multilingual
 - medic
 datasets:
 - HiTZ/Multilingual-Medical-Corpus
 - HiTZ/multilingual-abstrct
-language:
-- es
-- en
-- fr
-- it
-base_model: HiTZ/Medical-mT5-large
 ---
 <p align="center">

 ---
+language:
+- es
+- en
+- fr
+- it
 license: apache-2.0
 library_name: transformers
 tags:
 - medical
 - multilingual
 - medic
+base_model: HiTZ/Medical-mT5-large
 datasets:
 - HiTZ/Multilingual-Medical-Corpus
 - HiTZ/multilingual-abstrct
+widget:
+- text: <Disease> Torsade de pointes ventricular tachycardia during low dose intermittent
+    dobutamine treatment in a patient with dilated cardiomyopathy and congestive heart
+    failure .
+- text: '<ClinicalEntity> Ecográficamente se observan tres nódulos tumorales independientes
+    y bien delimitados : dos de ellos heterogéneos , sólidos , de 20 y 33 mm de diámetros
+    , con áreas quísticas y calcificaciones .'
+- text: <ClinicalEntity> On notait une hyperlordose lombaire avec une contracture
+    permanente des muscles paravertébraux , de l abdomen et des deux membres inférieurs
+    .
+- text: <ClinicalEntity> Nell ’ anamnesi patologica era riferita ipertensione arteriosa
+    controllata con terapia medica
+pipeline_tag: text2text-generation
 ---
 <p align="center">

added_tokens.json ADDED Viewed

	@@ -0,0 +1,24 @@

+{
+  "</Chemical>": 250117,
+  "</Claim>": 250114,
+  "</ClinicalEntity>": 250111,
+  "</Dis>": 250112,
+  "</Disease>": 250116,
+  "</DiseaseNCBI>": 250115,
+  "</NORMALIZABLES>": 250121,
+  "</NO_NORMALIZABLES>": 250119,
+  "</PROTEINAS>": 250118,
+  "</Premise>": 250113,
+  "</UNCLEAR>": 250120,
+  "<Chemical>": 250106,
+  "<Claim>": 250103,
+  "<ClinicalEntity>": 250100,
+  "<Dis>": 250101,
+  "<Disease>": 250105,
+  "<DiseaseNCBI>": 250104,
+  "<NORMALIZABLES>": 250110,
+  "<NO_NORMALIZABLES>": 250108,
+  "<PROTEINAS>": 250107,
+  "<Premise>": 250102,
+  "<UNCLEAR>": 250109
+}

special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,23 @@

+{
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

spiece.model ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:ef78f86560d809067d12bac6c09f19a462cb3af3f54d2b8acbba26e1433125d6
+size 4309802

tokenizer.json ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:346fcaa3262c0fc3cd3ec89cae5d4464e61ab5389dc0ca465643337db812afec
+size 16319362

tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,215 @@

+{
+  "add_eos_token": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250100": {
+      "content": "<ClinicalEntity>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "250101": {
+      "content": "<Dis>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "250102": {
+      "content": "<Premise>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "250103": {
+      "content": "<Claim>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "250104": {
+      "content": "<DiseaseNCBI>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "250105": {
+      "content": "<Disease>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "250106": {
+      "content": "<Chemical>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "250107": {
+      "content": "<PROTEINAS>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "250108": {
+      "content": "<NO_NORMALIZABLES>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "250109": {
+      "content": "<UNCLEAR>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "250110": {
+      "content": "<NORMALIZABLES>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "250111": {
+      "content": "</ClinicalEntity>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "250112": {
+      "content": "</Dis>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "250113": {
+      "content": "</Premise>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "250114": {
+      "content": "</Claim>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "250115": {
+      "content": "</DiseaseNCBI>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "250116": {
+      "content": "</Disease>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "250117": {
+      "content": "</Chemical>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "250118": {
+      "content": "</PROTEINAS>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "250119": {
+      "content": "</NO_NORMALIZABLES>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "250120": {
+      "content": "</UNCLEAR>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "250121": {
+      "content": "</NORMALIZABLES>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [],
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "</s>",
+  "extra_ids": 0,
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "T5Tokenizer",
+  "unk_token": "<unk>"
+}