first commit

.gitignore

@@ -0,0 +1,4 @@
+/env
+/__pycache__/
+
+.env

app.py

@@ -0,0 +1,77 @@
+import os
+import streamlit as st
+from elasticsearch import Elasticsearch
+import numpy as np
+import pandas as pd
+from sklearn.preprocessing import StandardScaler
+from sklearn.manifold import TSNE
+import plotly.express as plx
+
+def compare():
+    if len(multiselect) == 0: return
+    target_field = f"{model}_features"
+    ids = [documents[title] for title in multiselect]
+
+    results = []
+    for id in ids:
+        results.append(es.search(
+            index="sentences",
+            query={
+                "constant_score" : { 
+                    "filter" : {
+                        "term" : { 
+                            "document": id
+                        }
+                    }
+                }
+            },
+            size=limit
+        ))
+
+
+    features = []
+    classes = []
+    sentences = []
+    for result, title in zip(results, multiselect):
+        features.append(np.asarray([sent["_source"][target_field] for sent in result["hits"]["hits"]]))
+        classes.extend([title]*len(result["hits"]["hits"]))
+        sentences.extend([sent["_source"]["sentence"] for sent in result["hits"]["hits"]])
+    
+    features = np.concatenate(features)
+
+    scaler = StandardScaler()
+    features = scaler.fit_transform(features)
+    tsne = TSNE(n_components=2, metric="cosine", init="pca")
+    features = tsne.fit_transform(features)
+
+    classes = [c[:10]+"..." for c in classes]
+
+    df = pd.DataFrame.from_dict(dict(
+        x=features[:, 0],
+        y=features[:, 1],
+        classes=classes,
+        sentences=sentences
+    ))
+    
+    
+    st.plotly_chart(plx.scatter(
+        data_frame=df,
+        x="x",
+        y="y",
+        color="classes",
+        hover_name="sentences"
+    ))
+
+es = Elasticsearch(os.environ["ELASTIC_HOST"], basic_auth=os.environ["ELASTIC_AUTH"].split(":"))
+
+results = es.search(index="documents", query={"match_all":{}})
+results = [result["_source"] for result in results["hits"]["hits"]]
+
+documents = {f"{result['title']} - {result['author']}": result['id'] for result in results}
+
+st.sidebar.title("Semantic compare")
+st.sidebar.write("Select 2 or more documents from the SERICA library to semantically compare them")
+multiselect = st.sidebar.multiselect("Documents", list(documents.keys()))
+model = st.sidebar.selectbox("Model", ["LaBSE"])
+limit = st.sidebar.number_input("Sentences per document", 1000)
+st.sidebar.button("Compare", on_click=compare)

requirements.txt

@@ -0,0 +1,16 @@
+certifi==2022.6.15
+elastic-transport==8.1.2
+elasticsearch==8.3.3
+joblib==1.1.0
+numpy==1.23.1
+pandas==1.4.3
+plotly==5.9.0
+python-dateutil==2.8.2
+pytz==2022.1
+scikit-learn==1.1.1
+scipy==1.9.0
+six==1.16.0
+sklearn==0.0
+tenacity==8.0.1
+threadpoolctl==3.1.0
+urllib3==1.26.11