Update app.py

app.py

@@ -23,7 +23,9 @@ if "visibility" not in st.session_state:
     st.session_state.disabled = False
 
 #nlp = en_core_web_lg.load()
-nlp = spacy.load("en_ner_bc5cdr_md")
+#nlp = spacy.load("en_ner_bc5cdr_md")
+nlp0 = spacy.load("en_core_sci_sm")
+nlp1 = spacy.load("en_ner_bc5cdr_md")
 
 st.set_page_config(page_title ='Clinical Note Summarization', 
                    #page_icon= "Notes",
@@ -150,7 +152,95 @@ def genEntities(ann, entity):
                              entlist = ",".join(ent)
                              st.markdown(f'<p style="background-color:{ent_col[entity]};color:#080808;font-size:16px;">{entlist}</p>', unsafe_allow_html=True)
                                                                        
-
+#lemmatizing the notes to capture all forms of negation(e.g., deny: denies, denying)
+def lemmatize(note, nlp):
+    doc = nlp(note)
+    lemNote = [wd.lemma_ for wd in doc]
+    return " ".join(lemNote)
+
+#function to modify options for displacy NER visualization
+def get_entity_options():
+    entities = ["DISEASE", "CHEMICAL", "NEG_ENTITY"]
+    colors = {'DISEASE': 'pink', 'CHEMICAL': 'orange', "NEG_ENTITY":'white'}
+    options = {"ents": entities, "colors": colors}    
+    return options
+
+#adding a new pipeline component to identify negation
+def neg_model(nlp_model):
+    nlp = spacy.load(nlp_model, disable = ['parser'])
+#     nlp.add_pipe(nlp.create_pipe('sentencizer'))
+    nlp.add_pipe('sentencizer')
+#     negex = Negex(nlp)
+    nlp.add_pipe(
+    "negex",
+    config={
+        "chunk_prefix": ["no"],
+    },
+    last=True)
+    return nlp
+"""
+Negspacy sets a new attribute e._.negex to True if a negative concept is encountered
+"""
+def negation_handling(nlp_model, note, neg_model):
+    results = []
+    nlp = neg_model(nlp_model) 
+    note = note.split(".") #sentence tokenizing based on delimeter 
+    note = [n.strip() for n in note] #removing extra spaces at the begining and end of sentence
+    for t in note:
+        doc = nlp(t)
+        for e in doc.ents:
+            rs = str(e._.negex)
+            if rs == "True": 
+                results.append(e.text)
+    return results
+
+#function to identify span objects of matched negative phrases from text
+def match(nlp,terms,label):
+    patterns = [nlp.make_doc(text) for text in terms]
+    matcher = PhraseMatcher(nlp.vocab)
+    matcher.add(label, None, *patterns)
+    return matcher
+
+#replacing the labels for identified negative entities
+def overwrite_ent_lbl(matcher, doc):
+    matches = matcher(doc)
+    seen_tokens = set()
+    new_entities = []
+    entities = doc.ents
+    for match_id, start, end in matches:
+        if start not in seen_tokens and end - 1 not in seen_tokens:
+            new_entities.append(Span(doc, start, end, label=match_id))
+            entities = [e for e in entities if not (e.start < end and e.end > start)]
+            seen_tokens.update(range(start, end))
+    doc.ents = tuple(entities) + tuple(new_entities)
+    return doc
+
+#deduplicate repeated entities 
+def dedupe(items):
+    seen = set()
+    for item in items:
+        item = str(item).strip()
+        if item not in seen:
+            yield item
+            seen.add(item)
+
+lem_clinical_note= lemmatize(text, nlp0)
+#creating a doc object using BC5CDR model
+doc = nlp1(lem_clinical_note)
+options = get_entity_options()
+
+#list of negative concepts from clinical note identified by negspacy
+results0 = negation_handling("en_ner_bc5cdr_md", lem_clinical_note, neg_model)
+
+matcher = match(nlp1, results0,"NEG_ENTITY")
+
+#doc0: new doc object with added "NEG_ENTITY label"
+doc0 = overwrite_ent_lbl(matcher,doc)
+
+#visualizing identified Named Entities in clinical input text 
+displacy.render(doc0, style='ent', options=options)
+
+##=== end of NER tagger===
 def visualize (run_text,output):
     text =''
     splitruntext = [x for x in runtext.split('.')]
@@ -185,6 +275,7 @@ def run_model(input_text):
         st.write('Summary')
 
 
+    
     st.success(output)
 
 doc = nlp(str(original_text2))