add tokenizer

Browse files

Files changed (6) hide show

added_tokens.json +237 -0
merges.txt +0 -0
special_tokens_map.json +15 -0
tokenizer.json +0 -0
tokenizer_config.json +65 -0
vocab.json +0 -0

added_tokens.json ADDED Viewed

	@@ -0,0 +1,237 @@

+{
+  "2021": 50347,
+  "2025": 50403,
+  "2030": 50339,
+  "2050": 50401,
+  "Agreement": 50344,
+  "BNP": 50370,
+  "BlackRock": 50480,
+  "CO2": 50365,
+  "Carbon": 50358,
+  "Committee": 50329,
+  "Corporate": 50408,
+  "ESG": 50360,
+  "Eni": 50476,
+  "Finance": 50493,
+  "GHG": 50322,
+  "Governance": 50308,
+  "Greenhouse": 50463,
+  "IFC": 50475,
+  "Investment": 50369,
+  "Operational": 50486,
+  "Paribas": 50385,
+  "Principles": 50494,
+  "Risk": 50319,
+  "Sustainability": 50362,
+  "Sustainable": 50341,
+  "achieve": 50379,
+  "achieving": 50498,
+  "across": 50299,
+  "activities": 50283,
+  "addition": 50287,
+  "additional": 50367,
+  "adverse": 50411,
+  "adversely": 50497,
+  "affect": 50320,
+  "already": 50496,
+  "annual": 50342,
+  "annually": 50488,
+  "approach": 50311,
+  "areas": 50318,
+  "assess": 50378,
+  "assessment": 50351,
+  "asset": 50325,
+  "basis": 50473,
+  "bond": 50418,
+  "bonds": 50452,
+  "buildings": 50404,
+  "businesses": 50309,
+  "caused": 50410,
+  "cement": 50492,
+  "challenges": 50400,
+  "claims": 50470,
+  "clients": 50289,
+  "commitment": 50337,
+  "commitments": 50465,
+  "committed": 50334,
+  "communities": 50389,
+  "companies": 50271,
+  "compared": 50467,
+  "conditions": 50376,
+  "considered": 50479,
+  "construction": 50462,
+  "consumption": 50314,
+  "continued": 50443,
+  "contribute": 50405,
+  "corporate": 50307,
+  "costs": 50275,
+  "countries": 50328,
+  "criteria": 50477,
+  "customer": 50395,
+  "customers": 50279,
+  "decision": 50481,
+  "decisions": 50412,
+  "deliver": 50437,
+  "developing": 50381,
+  "developments": 50474,
+  "dioxide": 50471,
+  "directly": 50427,
+  "disasters": 50432,
+  "distribution": 50429,
+  "economy": 50304,
+  "efforts": 50324,
+  "electricity": 50291,
+  "emerging": 50489,
+  "emission": 50388,
+  "emissions": 50266,
+  "employees": 50332,
+  "engagement": 50397,
+  "ensure": 50399,
+  "environmental": 50268,
+  "equity": 50434,
+  "equivalent": 50457,
+  "experience": 50448,
+  "exposed": 50386,
+  "exposure": 50343,
+  "facilities": 50336,
+  "factors": 50301,
+  "failure": 50499,
+  "finance": 50298,
+  "financing": 50294,
+  "fiscal": 50436,
+  "flooding": 50407,
+  "floods": 50449,
+  "following": 50371,
+  "footprint": 50338,
+  "fossil": 50374,
+  "fuels": 50375,
+  "funds": 50398,
+  "further": 50315,
+  "goals": 50348,
+  "governance": 50345,
+  "greater": 50444,
+  "greenhouse": 50323,
+  "identify": 50377,
+  "impacts": 50272,
+  "included": 50393,
+  "increase": 50280,
+  "increased": 50293,
+  "increases": 50446,
+  "indirect": 50445,
+  "industry": 50292,
+  "infrastructure": 50310,
+  "initiative": 50424,
+  "initiatives": 50316,
+  "innovative": 50491,
+  "institutions": 50450,
+  "insurance": 50296,
+  "invested": 50439,
+  "investing": 50392,
+  "investment": 50269,
+  "investments": 50282,
+  "investors": 50356,
+  "launched": 50440,
+  "least": 50487,
+  "legislation": 50485,
+  "lending": 50368,
+  "loan": 50396,
+  "loans": 50417,
+  "longer": 50431,
+  "losses": 50346,
+  "mainly": 50466,
+  "manage": 50326,
+  "managing": 50421,
+  "manufacturing": 50425,
+  "materials": 50357,
+  "methodology": 50483,
+  "metrics": 50442,
+  "mitigate": 50428,
+  "mortgage": 50459,
+  "objectives": 50387,
+  "operate": 50415,
+  "operating": 50372,
+  "operational": 50302,
+  "operations": 50270,
+  "opportunities": 50278,
+  "others": 50472,
+  "overall": 50366,
+  "particular": 50390,
+  "partners": 50482,
+  "patterns": 50438,
+  "planning": 50435,
+  "plans": 50383,
+  "plants": 50350,
+  "policies": 50353,
+  "portfolio": 50277,
+  "portfolios": 50363,
+  "possible": 50420,
+  "potential": 50281,
+  "practices": 50394,
+  "prices": 50484,
+  "processes": 50354,
+  "profitability": 50454,
+  "provide": 50313,
+  "provides": 50409,
+  "purchase": 50478,
+  "reduce": 50284,
+  "reduced": 50422,
+  "reducing": 50359,
+  "reduction": 50306,
+  "regarding": 50413,
+  "regulations": 50321,
+  "regulatory": 50295,
+  "relating": 50461,
+  "renewable": 50274,
+  "reputation": 50327,
+  "reputational": 50469,
+  "requirements": 50335,
+  "resilience": 50433,
+  "resulting": 50361,
+  "returns": 50495,
+  "risks": 50265,
+  "sands": 50441,
+  "scenario": 50290,
+  "scenarios": 50330,
+  "sectors": 50312,
+  "several": 50416,
+  "severity": 50468,
+  "significantly": 50458,
+  "society": 50364,
+  "solar": 50340,
+  "solutions": 50355,
+  "sources": 50380,
+  "stakeholders": 50384,
+  "standards": 50349,
+  "statements": 50453,
+  "strategic": 50300,
+  "strategies": 50391,
+  "strategy": 50286,
+  "suppliers": 50402,
+  "supply": 50297,
+  "supporting": 50490,
+  "sustainability": 50285,
+  "sustainable": 50288,
+  "systems": 50333,
+  "taken": 50460,
+  "targets": 50305,
+  "technologies": 50352,
+  "temperature": 50423,
+  "therefore": 50456,
+  "tonnes": 50447,
+  "towards": 50414,
+  "transactions": 50451,
+  "transition": 50273,
+  "transport": 50419,
+  "unable": 50464,
+  "uncertainty": 50406,
+  "various": 50382,
+  "vehicles": 50455,
+  "waste": 50331,
+  "–": 50317,
+  "—": 50430,
+  "‘": 50426,
+  "’": 50267,
+  "“": 50373,
+  "•": 50276,
+  "€": 50303
+}

merges.txt ADDED Viewed

The diff for this file is too large to render. See raw diff

special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,15 @@

+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,65 @@

+{
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "mask_token": {
+    "__type": "AddedToken",
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "model_max_length": 512,
+  "name_or_path": "./envclaim-climatebert",
+  "pad_token": {
+    "__type": "AddedToken",
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "special_tokens_map_file": "pre_model/14102021_roberta/special_tokens_map.json",
+  "tokenizer_class": "RobertaTokenizer",
+  "trim_offsets": true,
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

vocab.json ADDED Viewed

The diff for this file is too large to render. See raw diff