File size: 5,434 Bytes
8defc61
 
 
 
206ed66
8defc61
206ed66
604feee
 
206ed66
 
ab3e62e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
206ed66
 
 
 
 
 
 
 
8defc61
 
 
 
 
 
 
 
206ed66
 
 
 
8defc61
 
206ed66
 
 
 
 
ab3e62e
 
 
 
 
 
 
 
206ed66
 
 
 
 
ab3e62e
 
 
 
206ed66
ab3e62e
206ed66
 
 
 
 
 
ab3e62e
 
206ed66
 
 
 
 
768757a
 
 
 
3563942
768757a
 
 
 
 
 
206ed66
768757a
8defc61
604feee
 
 
206ed66
 
 
8defc61
 
206ed66
8defc61
 
206ed66
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8defc61
c464f06
 
ab3e62e
c464f06
604feee
8defc61
 
 
 
604feee
 
8defc61
206ed66
8defc61
 
 
 
604feee
 
206ed66
604feee
 
206ed66
604feee
 
 
 
 
8defc61
 
604feee
 
 
 
 
 
8defc61
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
# Hint: this cheatsheet is magic! https://cheat-sheet.streamlit.app/
import constants
import pandas as pd
import streamlit as st
import matplotlib.pyplot as plt
from transformers import BertForSequenceClassification, AutoTokenizer

import altair as alt
from altair import X, Y, Scale
import base64

import re


def preprocess_text(arabic_text):
    """Apply preprocessing to the given Arabic text.

    Args:
        arabic_text: The Arabic text to be preprocessed.

    Returns:
        The preprocessed Arabic text.
    """
    no_urls = re.sub(
        r"(https|http)?:\/\/(\w|\.|\/|\?|\=|\&|\%)*\b",
        "",
        arabic_text,
        flags=re.MULTILINE,
    )
    no_english = re.sub(r"[a-zA-Z]", "", no_urls)

    return no_english


@st.cache_data
def render_svg(svg):
    """Renders the given svg string."""
    b64 = base64.b64encode(svg.encode("utf-8")).decode("utf-8")
    html = rf'<p align="center"> <img src="data:image/svg+xml;base64,{b64}"/> </p>'
    c = st.container()
    c.write(html, unsafe_allow_html=True)


@st.cache_data
def convert_df(df):
    # IMPORTANT: Cache the conversion to prevent computation on every rerun
    return df.to_csv(index=None).encode("utf-8")


@st.cache_resource
def load_model(model_name):
    model = BertForSequenceClassification.from_pretrained(model_name)
    return model


tokenizer = AutoTokenizer.from_pretrained(constants.MODEL_NAME)
model = load_model(constants.MODEL_NAME)


def compute_ALDi(sentences):
    """Computes the ALDi score for the given sentences.

    Args:
        sentences: A list of Arabic sentences.

    Returns:
        A list of ALDi scores for the given sentences.
    """
    progress_text = "Computing ALDi..."
    my_bar = st.progress(0, text=progress_text)

    BATCH_SIZE = 4
    output_logits = []

    preprocessed_sentences = [preprocess_text(s) for s in sentences]

    for first_index in range(0, len(preprocessed_sentences), BATCH_SIZE):
        inputs = tokenizer(
            preprocessed_sentences[first_index : first_index + BATCH_SIZE],
            return_tensors="pt",
            padding=True,
        )
        outputs = model(**inputs).logits.reshape(-1).tolist()
        output_logits = output_logits + [max(min(o, 1), 0) for o in outputs]
        my_bar.progress(
            min((first_index + BATCH_SIZE) / len(preprocessed_sentences), 1),
            text=progress_text,
        )
    my_bar.empty()
    return output_logits


@st.cache_data
def render_metadata():
    """Renders the metadata."""
    html = r"""<p align="center">
        <a href="https://huggingface.co/AMR-KELEG/Sentence-ALDi"><img alt="HuggingFace Model" src="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Model-8A2BE2"></a>
        <a href="https://github.com/AMR-KELEG/ALDi"><img alt="GitHub" src="https://img.shields.io/badge/%F0%9F%93%A6%20GitHub-orange"></a>
        <a href="https://arxiv.org/abs/2310.13747"><img alt="arXiv" src="https://img.shields.io/badge/arXiv-2310.13747-b31b1b.svg"></a>
        </p>"""
    c = st.container()
    c.write(html, unsafe_allow_html=True)

render_svg(open("assets/ALDi_logo.svg").read())
render_metadata()

tab1, tab2 = st.tabs(["Input a Sentence", "Upload a File"])

with tab1:
    sent = st.text_input(
        "Arabic Sentence:", placeholder="Enter an Arabic sentence.", on_change=None
    )

    # TODO: Check if this is needed!
    clicked = st.button("Submit")

    if sent:
        ALDi_score = compute_ALDi([sent])[0]

        ORANGE_COLOR = "#FF8000"
        fig, ax = plt.subplots(figsize=(8, 1))
        fig.patch.set_facecolor("none")
        ax.set_facecolor("none")

        ax.spines["left"].set_color(ORANGE_COLOR)
        ax.spines["bottom"].set_color(ORANGE_COLOR)
        ax.tick_params(axis="x", colors=ORANGE_COLOR)

        ax.spines[["right", "top"]].set_visible(False)

        ax.barh(y=[0], width=[ALDi_score], color=ORANGE_COLOR)
        ax.set_xlim(0, 1)
        ax.set_ylim(-1, 1)
        ax.set_title(f"ALDi score is: {round(ALDi_score, 3)}", color=ORANGE_COLOR)
        ax.get_yaxis().set_visible(False)
        ax.set_xlabel("ALDi score", color=ORANGE_COLOR)
        st.pyplot(fig)

        print(sent)
        with open("logs.txt", "a") as f:
            f.write(sent + "\n")

with tab2:
    file = st.file_uploader("Upload a file", type=["txt"])
    if file is not None:
        df = pd.read_csv(file, sep="\t", header=None)
        df.columns = ["Sentence"]
        df.reset_index(drop=True, inplace=True)

        # TODO: Run the model
        df["ALDi"] = compute_ALDi(df["Sentence"].tolist())

        # A horizontal rule
        st.markdown("""---""")

        chart = (
            alt.Chart(df.reset_index())
            .mark_area(color="darkorange", opacity=0.5)
            .encode(
                x=X(field="index", title="Sentence Index"),
                y=Y("ALDi", scale=Scale(domain=[0, 1])),
            )
        )
        st.altair_chart(chart.interactive(), use_container_width=True)

        col1, col2 = st.columns([4, 1])

        with col1:
            # Display the output
            st.table(
                df,
            )

        with col2:
            # Add a download button
            csv = convert_df(df)
            st.download_button(
                label=":file_folder: Download predictions as CSV",
                data=csv,
                file_name="ALDi_scores.csv",
                mime="text/csv",
            )