minor updates

.gitignore

@@ -0,0 +1 @@
+output/

app.py

@@ -9,37 +9,42 @@ from sklearn.metrics.pairwise import cosine_similarity
 from io import BytesIO
 
 # base is smaller, vs large
-model_size='base'
-tokenizer = AlbertTokenizer.from_pretrained('albert-' + model_size + '-v2')
-model = AlbertModel.from_pretrained('albert-' + model_size + '-v2')
+#model_size='base'
+#tokenizer = AlbertTokenizer.from_pretrained('albert-' + model_size + '-v2')
+#model = AlbertModel.from_pretrained('albert-' + model_size + '-v2')
+
+# For baseline 'sentence-transformers/paraphrase-albert-base-v2'
+model_name = 'output/training_OnlineConstrativeLoss-2023-03-10_11-17-15'
+
+similarity_threshold = 0.9
 
-model_sbert = SentenceTransformer('sentence-transformers/paraphrase-albert-base-v2')
 # for regular burt 0.98
 
-similarity_threshold = 0.8
+model_sbert = SentenceTransformer(model_name)
+
 
 def get_sbert_embedding(input_text):
     embedding = model_sbert.encode(input_text)
     return embedding.tolist()
 
-def get_embedding(input_text):
-    encoded_input = tokenizer(input_text, return_tensors='pt')
-    input_ids = encoded_input.input_ids
-    #input_num_tokens = input_ids.shape[1]
-
-    #print( "Number of input tokens: " + str(input_num_tokens))
-    #print("Length of input: " + str(len(input_text)))
-
-    list_of_tokens = tokenizer.convert_ids_to_tokens(input_ids.view(-1).tolist())
-
-    #print( "Tokens : " + ' '.join(list_of_tokens))
-    with torch.no_grad():
-
-        outputs = model(**encoded_input)
-        last_hidden_states = outputs[0]
-        sentence_embedding = torch.mean(last_hidden_states[0], dim=0)
-        #sentence_embedding = output.last_hidden_state[0][0]
-        return sentence_embedding.tolist()
+#def get_embedding(input_text):
+#    encoded_input = tokenizer(input_text, return_tensors='pt')
+#    input_ids = encoded_input.input_ids
+#    #input_num_tokens = input_ids.shape[1]
+#
+#    #print( "Number of input tokens: " + str(input_num_tokens))
+#    #print("Length of input: " + str(len(input_text)))
+#
+#    list_of_tokens = tokenizer.convert_ids_to_tokens(input_ids.view(-1).tolist())
+#
+#    #print( "Tokens : " + ' '.join(list_of_tokens))
+#    with torch.no_grad():
+#
+#        outputs = model(**encoded_input)
+#        last_hidden_states = outputs[0]
+#        sentence_embedding = torch.mean(last_hidden_states[0], dim=0)
+#        #sentence_embedding = output.last_hidden_state[0][0]
+#        return sentence_embedding.tolist()
 
 st.set_page_config(layout="wide")
 st.title('Upload the Address Dataset')
@@ -58,7 +63,7 @@ if uploaded_file is not None:
     data_caqh['postalcode'] = data_caqh['postalcode'].astype(str).apply(lambda x: x[:5] + '-' + x[5:] if len(x) > 5 and not '-' in x else x)
     data_caqh['full-addr'] = data_caqh['address1'].astype(str) + ', ' \
                              + np.where(data_caqh['address2'].isnull(),  '' , data_caqh['address2'].astype(str)+ ', ')  \
-                             + data_caqh['city'].astype(str) + ' '\
+                             + data_caqh['city'].astype(str) + ', '\
                              + data_caqh['state'].astype(str) + ' ' \
                              + data_caqh['postalcode'].astype(str)
 
@@ -75,7 +80,7 @@ if uploaded_file is not None:
                                       + data_ndb['zip_pls_4_cd'].astype(str))
 
     data_ndb['full-addr'] = data_ndb['adr_ln_1_txt'].astype(str).str.strip() + ', ' \
-                            + data_ndb['cty_nm'].astype(str).str.strip() + '  ' \
+                            + data_ndb['cty_nm'].astype(str).str.strip() + ',  ' \
                             + data_ndb['st_cd'].astype(str) + ' ' + data_ndb['zip_cd_zip_pls_4_cd']
 
     # Calculate similarity For CAQH

data.py

@@ -9,7 +9,7 @@ from sentence_transformers import SentenceTransformer
 #tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')
 #model = AlbertModel.from_pretrained("albert-base-v2")
 #'sentence-transformers/paraphrase-albert-base-v2'
-model_name = 'output/training_OnlineConstrativeLoss-2023-03-09_23-55-34'
+model_name = 'output/training_OnlineConstrativeLoss-2023-03-10_11-17-15'
 model_sbert = SentenceTransformer(model_name)
 
 def get_sbert_embedding(input_text):
@@ -58,22 +58,22 @@ e8 = get_sbert_embedding(a8)
 e8x = get_sbert_embedding(a8x)
 e9 = get_sbert_embedding(a9)
 e10 = get_sbert_embedding(a10)
-print(f"a1 \"{a1}\" to \"{a2}\" a2")
+print(f"a1 \"{a1}\" to \"{a2}\" a2 - expected Different")
 print(cosine_similarity([e1], [e2]))
-print(f"a1 \"{a1}\" to \"{a4}\" a4")
+print(f"a1 \"{a1}\" to \"{a4}\" a4 - expected Different")
 print(cosine_similarity([e1], [e4]))
-print(f"a1 \"{a1}\" to \"{a5}\" a5")
+print(f"a1 \"{a1}\" to \"{a5}\" a5 - expected Same")
 print(cosine_similarity([e1], [e5]))
 
-print(f"a7 \"{a7}\" to \"{a8}\" a8")
+print(f"a7 \"{a7}\" to \"{a8}\" a8 - expected Different")
 print(cosine_similarity([e7], [e8]))
-print(f"a7 \"{a7}\" to \"{a8x}\" a8x")
+print(f"a7 \"{a7}\" to \"{a8x}\" a8x - expected Different")
 print(cosine_similarity([e7], [e8x]))
 
-print(f"a7 \"{a7}\" to \"{a9}\" a9")
+print(f"a7 \"{a7}\" to \"{a9}\" a9 - expected Same")
 print(cosine_similarity([e7], [e9]))
 
-print(f"a7 \"{a7}\" to \"{a10}\" a10")
+print(f"a7 \"{a7}\" to \"{a10}\" a10 - expected Same")
 print(cosine_similarity([e7], [e10]))
 # with base
 #a1 to a2

data_set_training.csv

@@ -106,7 +106,8 @@ Valley Healthcare System 1600 Fort Benning Rd, Columbus, GA 31903|1600 Fort Benn
 Valley Healthcare System 1600 Fort Benning Rd, Columbus, GA 31903|1600 Fort Benning Rd, Valley Healthcare System, Columbus, GA 31903|1
 Memorial Satilla Health, 1900 Tebeau St, Waycross, GA 31501|1900 Tebeau St, Waycross, GA 31501|1
 VA Medical Center 2002 Holcombe Blvd, Houston, TX 77030|VA Medical Center 2002 Holcombe Boulevard, Houston, TX 77030|1
-
+1839 E Capitol Ave, Bismarck, ND 58501|1839 East Capitol Avenue, Bismarck, ND 58501|1
+1839 E Capitol Ave, Bismarck, ND 58501|1912 East Capitol Avenue, Bismarck, ND 58501|0
 
 
 

dev_set_training.csv

@@ -4,4 +4,12 @@ address1|address2|are_same
 1061 Schmidt Ln, North Brunswick Township, NJ 08902|1061 Schmidt Lane, North Brunswick Township, NJ 08902|1
 1061 Schmidt Ln, North Brunswick Township, NJ 08902|934 Schmidt Ln, North Brunswick Township, NJ 08902|0
 5844 N Orange Blossom Trail, Orlando, FL 32810|5844 North Orange Blossom Trail, Orlando, FL 32810-9635|1
-6701 Fannin St #1400, Houston, TX 77030|6701 Fannin Ste #1400, Houston, TX 77030|1
+6701 Fannin St #1400, Houston, TX 77030|6701 Fannin Ste #1400, Houston, TX 77030|1
+14143 Winecup Ln, Houston, TX 77047|14121 Winecup Lane, Houston, TX 77047|0
+440 TECHNOLOGY CENTER DRIVE, Boston, MA 10034|440 Technology Center Dr., Boston, MA 10034|1
+440 TECHNOLOGY CENTER DRIVE, Boston, MA 10034|440 Technology Center Dr., Boston, MA 10034-0345|1
+440 TECHNOLOGY CENTER DRIVE, Boston, MA 10034|87 Technology Center Drive, Boston, MA 10034|0
+440 TECHNOLOGY CENTER DRIVE, Boston, MA 10034|200 Technology Center Drive, Boston, MA 10034|0
+65 Mountain Blvd Ext, Warren, NJ 07059|65 Mountain Boulevard Ext, Warren, NJ 07059|1
+65 Mountain Blvd Ext, Warren, NJ 07059|5078 S Maryland Pkwy, Las Vegas, NV 89119|0
+65 Mountain Blvd Ext, Warren, NJ 07059|112 Mountain Blvd Ext, Warren, NJ 07059|0

train.py

@@ -24,7 +24,7 @@ logger = logging.getLogger(__name__)
 
 
 #As base model, we use DistilBERT-base that was pre-trained on NLI and STSb data
-model = SentenceTransformer('albert-base-v2')
+model = SentenceTransformer('sentence-transformers/paraphrase-albert-base-v2')
 num_epochs = 10
 train_batch_size = 8