Update app.py

added duplicate names removal functionality

app.py

@@ -309,6 +309,10 @@ TIME"""
         # dedupe the entities but only on exact match of values as occasional it will assign an ORG entity to PER
         deduplicated_entities = {frozenset(item.values()):
                                      item for item in article_entity_list}.values()
+
+        #to remove duplicate names
+        fuzzy_remove_duplicate_ent(deduplicated_entities, threshold = 85, limit = 1)
+        
         # create a dataframe from the entities
         for record in deduplicated_entities:
             record_df = pd.DataFrame(record.items(), columns=["entity", "description"])
@@ -318,6 +322,20 @@ TIME"""
         print('______________________')
         return self.entity_df
 
+    
+    def fuzzy_remove_duplicate_ent(deduped_ents, threshold=85, limit=1):
+      search_space = list(deduped_ents)
+
+      for ent in deduped_ents:
+        duplicates_found = process.extract(ent, search_space.remove(ent), limit =1) # process.extract return the ent match amongst search_space with it's score
+        duplicates_found = [entity[0] for entity in duplicates_found if entity[1]> threshold]
+
+        if (len(duplicates_found) >0 ):
+          deduped_ents =[entity for entity in deduped_ents if entity not in duplicates_found]
+
+      return deduped_ents
+
+    
     def entity_json(self):
         """Returns a JSON representation of an entity defined by the `entity_df` dataframe. The `entity_json` function
         will return a JSON object with the following fields:
@@ -444,6 +462,10 @@ if st.session_state['parsed']:
                                                             'GPE':'Political Location',
                                                             'NORP':'Political or Religious Groups',
                                                             'LOC':'Non Political Location'})
+
+    # Replace name with zeroth element of related to
+    df_to_st['Name'] = df_to_st['Related to'].apply(lambda x : x[0])
+    
     gb = GridOptionsBuilder.from_dataframe(df_to_st)
     gb.configure_pagination(paginationAutoPageSize=True) #Add pagination
     gb.configure_side_bar() #Add a sidebar