feat(tokenizer): expose merge ranks and special tokens for GGUF

tokenization_arcade100k.py

@@ -113,7 +113,7 @@ class Arcade100kTokenizer(PreTrainedTokenizer):
         super().__init__(errors=errors, **kwargs)
         self._tiktoken_config = _arcade100k(vocab_file)
         self.tokenizer = tiktoken.Encoding(**self._tiktoken_config)
-        self.errors = errors
+
         # TODO: Remove this assertion
         assert (
             len(self.tokenizer._mergeable_ranks)
@@ -126,6 +126,9 @@ class Arcade100kTokenizer(PreTrainedTokenizer):
         self.decoder.update({i: n for n, i in self.tokenizer._special_tokens.items()})
         self.eos_token = self.decoder[self.tokenizer.eot_token]
         self.pad_token = self.decoder[self.tokenizer.eot_token]
+        # Expose for convenience
+        self.mergeable_ranks = self.tokenizer._mergeable_ranks
+        self.special_tokens = self.tokenizer._special_tokens
 
     def __len__(self):
         return self.tokenizer.n_vocab
@@ -270,4 +273,4 @@ class Arcade100kTokenizer(PreTrainedTokenizer):
             token_ids = [token_ids]
         if skip_special_tokens:
             token_ids = [i for i in token_ids if i < self.tokenizer.eot_token]
-        return self.tokenizer.decode(token_ids)
+        return self.tokenizer.decode(token_ids)