Update tiktoken.py

tiktoken.py

@@ -3,7 +3,6 @@
 from functools import lru_cache
 from typing import Any, Dict, List, Optional, Tuple
 
-import torch
 from transformers import PreTrainedTokenizer
 
 DEFAULT_SYSTEM_PROMPT = """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible."""
@@ -200,8 +199,6 @@ class TiktokenTokenizerWrapper(PreTrainedTokenizer):
             '{% endif %}'
             '{% if (add_generation_prompt == true and loop.last) %}'
             "{{ '\n' + '<|im_start|>' + 'assistant' + '\n' }}"
-            "{% elif (message['role'] == 'assistant') %}"
-            '{{ eos_token }}'
             '{% endif %}'
             '{% endfor %}')
         template = template.replace(
@@ -358,19 +355,5 @@ class TiktokenTokenizerWrapper(PreTrainedTokenizer):
 
         return self.add_tokens(actual_new_tokens, special_tokens=True)
 
-    def construct_logit_tensor(self, logprobs: Dict[str,
-                                                    float]) -> torch.Tensor:
-        """Construct tensor of shape (vocab_size,) mapping words to logprobs.
 
-        Args:
-            logprobs (Dict[str, float]): Dictionary mapping tokens to log probabilities assigned to them by the model.
-        """
-        tensor = torch.tensor([min(logprobs.values()) - 1] * (self.vocab_size))
-        for k in logprobs:
-            encoding = self(k)['input_ids']
-            idx = encoding[0]
-            tensor[idx] = logprobs[k]
-        return tensor
-
-
-TiktokenTokenizerWrapper.register_for_auto_class()
+TiktokenTokenizerWrapper.register_for_auto_class()