Spaces:
Runtime error
Runtime error
Commit
·
5008e92
1
Parent(s):
588a02c
Add title and preamble
Browse files
app.py
CHANGED
@@ -26,6 +26,18 @@ def main():
|
|
26 |
tokenizer = AutoTokenizer.from_pretrained(selected_model)
|
27 |
return tokenizer
|
28 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
29 |
col1, col2 = st.columns([1, 2])
|
30 |
with col1:
|
31 |
selected_model = st.selectbox("Select a tokenizer", models, key=1)
|
|
|
26 |
tokenizer = AutoTokenizer.from_pretrained(selected_model)
|
27 |
return tokenizer
|
28 |
|
29 |
+
st.markdown(
|
30 |
+
"""
|
31 |
+
## TAPAS Tokenization Visualization
|
32 |
+
|
33 |
+
[TAPAS](https://huggingface.co/docs/transformers/model_doc/tapas) models work on Tables.
|
34 |
+
The tool below is to help visualize how the table is tokenized and give total (+ row-wise)
|
35 |
+
token counts.
|
36 |
+
|
37 |
+
Implementation adapted from `tokenizers.tools.EncodingVisualizer`.
|
38 |
+
"""
|
39 |
+
)
|
40 |
+
|
41 |
col1, col2 = st.columns([1, 2])
|
42 |
with col1:
|
43 |
selected_model = st.selectbox("Select a tokenizer", models, key=1)
|