Hugging Face's logo

Spaces:
jamino30
/
bpe-visualizer
Sleeping

App Files Community
jamino30 commited on
Commit
d4bbd8e
•
1 Parent(s): d5ce4c4

Upload folder using huggingface_hub

Browse files
Files changed (2) hide show
  1. dune-20256.model +1 -1
  2. xsbpe/base.py +1 -1
dune-20256.model CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:5aceeff36b589168f40e87d22befcc69879312882c3f8c16f4df412bd9c95cef
3
  size 174006
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a0a77bec25eb609ecfc68bbf1adfa9f503a26866c7638fd1a08201671f07e56b
3
  size 174006
xsbpe/base.py CHANGED
@@ -66,7 +66,7 @@ class Tokenizer:
66
  # Similar to sentencepiece model saving
67
  model_file = file_prefix + '.model'
68
  with open(model_file, 'w') as f:
69
- f.write('xsgpe v1\n')
70
  f.write(f'{self.pattern}\n')
71
  f.write(f'{len(self.special_tokens)}\n')
72
  for special, idx in self.special_tokens.items():
 
66
  # Similar to sentencepiece model saving
67
  model_file = file_prefix + '.model'
68
  with open(model_file, 'w') as f:
69
+ f.write('xsbpe v1\n')
70
  f.write(f'{self.pattern}\n')
71
  f.write(f'{len(self.special_tokens)}\n')
72
  for special, idx in self.special_tokens.items():