improvements

app.py

@@ -1,29 +1,33 @@
 import streamlit as st
 import pandas as pd
 import numpy as np
-#import torch
-#from transformers import AlbertTokenizer, AlbertModel
-#from sklearn.metrics.pairwise import cosine_similarity
-
-#tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')
-#model = AlbertModel.from_pretrained("albert-base-v2")
-
-#def get_embedding(input_text):
-#    encoded_input = tokenizer(input_text, return_tensors='pt')
-#    input_ids = encoded_input.input_ids
-#    input_num_tokens = input_ids.shape[1]
-#
-#    #print( "Number of input tokens: " + str(input_num_tokens))
-#    #print("Length of input: " + str(len(input_text)))
-#
-#    list_of_tokens = tokenizer.convert_ids_to_tokens(input_ids.view(-1).tolist())
-#
-#    #print( "Tokens : " + ' '.join(list_of_tokens))
-#    with torch.no_grad():
-#        output = model(**encoded_input)
-#
-#    embedding = output.last_hidden_state[0][0]
-#    return embedding.tolist()
+import torch
+from transformers import AlbertTokenizer, AlbertModel
+from sklearn.metrics.pairwise import cosine_similarity
+
+# base is smaller, vs large
+model_size='base'
+tokenizer = AlbertTokenizer.from_pretrained('albert-' + model_size + '-v2')
+model = AlbertModel.from_pretrained('albert-' + model_size + '-v2')
+
+def get_embedding(input_text):
+    encoded_input = tokenizer(input_text, return_tensors='pt')
+    input_ids = encoded_input.input_ids
+    input_num_tokens = input_ids.shape[1]
+
+    print( "Number of input tokens: " + str(input_num_tokens))
+    print("Length of input: " + str(len(input_text)))
+
+    list_of_tokens = tokenizer.convert_ids_to_tokens(input_ids.view(-1).tolist())
+
+    print( "Tokens : " + ' '.join(list_of_tokens))
+    with torch.no_grad():
+
+        outputs = model(**encoded_input)
+        last_hidden_states = outputs[0]
+        sentence_embedding = torch.mean(last_hidden_states[0], dim=0)
+        #sentence_embedding = output.last_hidden_state[0][0]
+        return sentence_embedding.tolist()
 
 st.title('Upload the Address Dataset')
 
@@ -33,8 +37,8 @@ uploaded_file = st.file_uploader('Choose a file', type='xlsx')
 
 
 if uploaded_file is not None:
-    data_caqh = pd.read_excel(uploaded_file, sheet_name='CAQH')
-    data_ndb = pd.read_excel(uploaded_file, sheet_name='NDB')
+    data_caqh = pd.read_excel(uploaded_file, sheet_name='CAQH', dtype=str)
+    data_ndb = pd.read_excel(uploaded_file, sheet_name='NDB', dtype=str)
 
     # Data cleaning CAQH
     data_caqh['postalcode'] = data_caqh['postalcode'].astype(str).apply(lambda x: x[:5] + '-' + x[5:] if len(x) > 5 and not '-' in x else x)
@@ -52,17 +56,35 @@ if uploaded_file is not None:
                                       + data_ndb['zip_pls_4_cd'].astype(str))
 
     data_ndb['full-addr'] = data_ndb['adr_ln_1_txt'].astype(str).str.strip() + ', ' \
+                            + data_ndb['city'].astype(str) + ', ' \
                             + data_ndb['st_cd'].astype(str) + ', ' + data_ndb['zip_cd_zip_pls_4_cd']
 
-    # Add a matched column
+    # App
+    data_caqh['embedding'] = data_caqh['full-addr'].apply(get_embedding)
+    data_ndb['embedding'] = data_ndb['full-addr'].apply(get_embedding)
+
     data_caqh['matched-addr'] = ''
 
-    # App
-    #data_caqh['embed'] = data_caqh['full-addr'].apply(get_embedding)
+    for i, row in data_caqh.iterrows():
+        max_similarity = 0
+        matched_row = None
+        for j, ndb_row in data_ndb.iterrows():
+            sim = cosine_similarity([row['embedding']], [ndb_row['embedding']])
+            if sim > max_similarity:
+                max_similarity = sim
+                matched_row = ndb_row
+        if max_similarity >= 0.98:
+            data_caqh.at[i, 'matched-addr'] = matched_row['full-addr']
+            data_caqh.at[i, 'similarity-score'] = max_similarity
+        else:
+            data_caqh.at[i, 'matched-addr'] = 'No Matches'
 
     st.dataframe(data_caqh)
     st.dataframe(data_ndb)
 
+    # calculate the embedding of each item.
+
+    #st.dataframe(data_caqh)
     # Do some matching
     #data_caqh.loc[data_caqh['full-addr'] == '1000 Vale Terrace, Vista, CA, 92084', 'matched-addr'] = '456 Main St'
     #time.sleep(10)

data.py

@@ -1,5 +1,6 @@
 from transformers import AlbertTokenizer, AlbertModel
 from sklearn.metrics.pairwise import cosine_similarity
+import torch
 
 # base
 # large
@@ -11,6 +12,7 @@ a2 = "112 Mountain Blvd Ext, Warren, NJ 07059"
 a3 = "1677 NJ-27 #2, Edison, NJ 08817"
 a4 = "5078 S Maryland Pkwy, Las Vegas, NV 89119"
 a5 = "65 Mountain Boulevard Ext, Warren, NJ 07059"
+a6 = "123 Broad St, New York, NY, 10304-2345"
 
 def get_embedding(input_text):
     encoded_input = tokenizer(input_text, return_tensors='pt')
@@ -23,16 +25,20 @@ def get_embedding(input_text):
     list_of_tokens = tokenizer.convert_ids_to_tokens(input_ids.view(-1).tolist())
 
     print( "Tokens : " + ' '.join(list_of_tokens))
-    output = model(**encoded_input)
+    with torch.no_grad():
 
-    embedding = output.last_hidden_state[0][0]
-    return embedding.tolist()
+        outputs = model(**encoded_input)
+        last_hidden_states = outputs[0]
+        sentence_embedding = torch.mean(last_hidden_states[0], dim=0)
+        #sentence_embedding = output.last_hidden_state[0][0]
+        return sentence_embedding.tolist()
 
 e1 = get_embedding(a1)
 e2 = get_embedding(a2)
 #e3 = get_embedding(a3)
 e4 = get_embedding(a4)
 e5 = get_embedding(a5)
+e6 = get_embedding(a6)
 
 print(f"a1 \"{a1}\" to \"{a2}\" a2")
 print(cosine_similarity([e1], [e2]))