bglearning commited on
Commit
5008e92
·
1 Parent(s): 588a02c

Add title and preamble

Browse files
Files changed (1) hide show
  1. app.py +12 -0
app.py CHANGED
@@ -26,6 +26,18 @@ def main():
26
  tokenizer = AutoTokenizer.from_pretrained(selected_model)
27
  return tokenizer
28
 
 
 
 
 
 
 
 
 
 
 
 
 
29
  col1, col2 = st.columns([1, 2])
30
  with col1:
31
  selected_model = st.selectbox("Select a tokenizer", models, key=1)
 
26
  tokenizer = AutoTokenizer.from_pretrained(selected_model)
27
  return tokenizer
28
 
29
+ st.markdown(
30
+ """
31
+ ## TAPAS Tokenization Visualization
32
+
33
+ [TAPAS](https://huggingface.co/docs/transformers/model_doc/tapas) models work on Tables.
34
+ The tool below is to help visualize how the table is tokenized and give total (+ row-wise)
35
+ token counts.
36
+
37
+ Implementation adapted from `tokenizers.tools.EncodingVisualizer`.
38
+ """
39
+ )
40
+
41
  col1, col2 = st.columns([1, 2])
42
  with col1:
43
  selected_model = st.selectbox("Select a tokenizer", models, key=1)