Spaces:
Build error
Build error
Update app.py
Browse files
app.py
CHANGED
@@ -1,4 +1,7 @@
|
|
1 |
import streamlit as st
|
|
|
|
|
|
|
2 |
import pandas as pd
|
3 |
import numpy as np
|
4 |
from math import ceil
|
@@ -13,67 +16,73 @@ from spacy.tokens import Span
|
|
13 |
#import en_ner_bc5cdr_md
|
14 |
import re
|
15 |
from streamlit.components.v1 import html
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
16 |
|
17 |
-
|
18 |
-
st.session_state.load_state = False
|
19 |
-
|
20 |
-
#if "button_clicked" not in st.session_state:
|
21 |
-
# st.session_state.button_clicked = True
|
22 |
-
#
|
23 |
-
#if "daily_button_clicked" not in st.session_state:
|
24 |
-
# st.session_state.daily_button_clicked = False
|
25 |
-
|
26 |
-
if "past_button_clicked" not in st.session_state:
|
27 |
-
st.session_state.past_button_clicked = False
|
28 |
-
|
29 |
-
|
30 |
-
#nlp = en_core_web_lg.load()
|
31 |
-
nlp = spacy.load("en_ner_bc5cdr_md")
|
32 |
-
|
33 |
-
st.set_page_config(page_title ='Patient Inpatient Progression Dashboard',
|
34 |
-
#page_icon= "Notes",
|
35 |
-
layout='wide')
|
36 |
-
st.title('Patient Inpatient Progression Dashboard')
|
37 |
-
st.markdown(
|
38 |
-
"""
|
39 |
-
<style>
|
40 |
-
[data-testid="stSidebar"][aria-expanded="true"] > div:first-child {
|
41 |
-
width: 400px;
|
42 |
-
}
|
43 |
-
[data-testid="stSidebar"][aria-expanded="false"] > div:first-child {
|
44 |
-
width: 400px;
|
45 |
-
margin-left: -230px;
|
46 |
-
}
|
47 |
-
</style>
|
48 |
-
""",
|
49 |
-
unsafe_allow_html=True,
|
50 |
-
)
|
51 |
-
st.sidebar.markdown('Using transformer model')
|
52 |
-
|
53 |
## ======== Loading dataset ========
|
54 |
## Loading in Admission Dataset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
55 |
df = pd.read_csv('shpi25nov.csv')
|
56 |
df.sort_values(by='SUBJECT_ID',ascending = True, inplace=True)
|
57 |
|
58 |
-
# Loading in Admission chief Complaint and diagnosis
|
59 |
df2 = pd.read_csv('cohort_cc_adm_diag.csv')
|
60 |
|
61 |
-
# Loading in Dischare History
|
62 |
df3 = pd.read_csv('cohort_past_history_12072022.csv')
|
63 |
df3.sort_values(by='CHARTDATE',ascending = False, inplace=True)
|
64 |
|
65 |
-
# Loading in Daily Narrative
|
66 |
df4 = pd.read_csv('24houreventsFulltextwdifference.csv')
|
67 |
-
df4.sort_values(by=['
|
68 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
69 |
|
70 |
# combining both data into one
|
71 |
df = pd.merge(df, df2, on=['HADM_ID','SUBJECT_ID'])
|
72 |
-
|
73 |
# Deleting admission chief complaint and diagnosis after combining
|
74 |
del df2
|
75 |
|
76 |
-
# Remove decimal point from Admission ID
|
77 |
df['HADM_ID'] = df['HADM_ID'].astype(str).apply(lambda x: x.replace('.0',''))
|
78 |
df3['HADM_ID'] = df3['HADM_ID'].astype(str).apply(lambda x: x.replace('.0',''))
|
79 |
df4['HADM_ID'] = df4['HADM_ID'].astype(str).apply(lambda x: x.replace('.0',''))
|
@@ -96,10 +105,33 @@ df3.rename(columns={'SUBJECT_ID':'Patient_ID',
|
|
96 |
'HADM_ID':'PAST_Admission_ID',
|
97 |
'INDEX_HADM_ID':'Admission_ID'}, inplace = True)
|
98 |
|
99 |
-
df4.rename(columns={'
|
100 |
-
'
|
101 |
-
'
|
|
|
|
|
|
|
102 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
103 |
|
104 |
#Filter selection
|
105 |
st.sidebar.header("Search for Patient:")
|
@@ -112,18 +144,11 @@ HospitalAdmission = st.sidebar.selectbox(' ', admissionid)
|
|
112 |
pastHistoryEpDate = df3['CHARTDATE_HADM_ID'].loc[(df3['Patient_ID'] == patient) & (df3['Admission_ID']== HospitalAdmission)]
|
113 |
countOfAdmission = len(pastHistoryEpDate)
|
114 |
|
115 |
-
|
116 |
# List of Model available
|
117 |
-
model = st.sidebar.selectbox('Select Model', ('BertSummarizer','BertGPT2','t5seq2eq','t5','gensim','pysummarizer'))
|
|
|
|
|
118 |
|
119 |
-
# ===== to display selected patient and admission id on main page
|
120 |
-
col3,col4 = st.columns(2)
|
121 |
-
patientid = col3.write(f"Patient ID: {patient} ")
|
122 |
-
admissionid =col4.write(f"Admission ID: {HospitalAdmission} ")
|
123 |
-
|
124 |
-
runtext = ''
|
125 |
-
inputNote ='Input note here:'
|
126 |
-
# Query out relevant Clinical notes
|
127 |
original_text = df.query(
|
128 |
"Patient_ID == @patient & Admission_ID == @HospitalAdmission"
|
129 |
)
|
@@ -132,40 +157,58 @@ original_text2 = original_text['Original_Text'].values
|
|
132 |
AdmissionChiefCom = original_text['Admission_Chief_Complaint'].values
|
133 |
diagnosis =original_text['DIAGNOSIS'].values
|
134 |
reference_text = original_text['Reference_text'].values
|
|
|
135 |
|
136 |
-
|
137 |
-
dailyNoteChange =df4[['STORETIME','_24_Hour_Events']].loc[(df4['Patient_ID'] == patient) & (df4['Admission_ID']==HospitalAdmission) & df4['_24_Hour_Events'].notnull()]
|
138 |
|
139 |
dailyNoteChange.rename(columns={'STORETIME':'Time of Record',
|
140 |
-
'
|
141 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
142 |
dailyNote = dailyNote.unique()
|
143 |
|
144 |
-
|
145 |
-
|
146 |
-
|
147 |
-
|
148 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
149 |
with st.container():
|
150 |
with col1:
|
151 |
btnAdmission = st.button("🏥 Admission")
|
152 |
-
inputNote = "Input Admission Note"
|
153 |
-
|
154 |
with col2:
|
155 |
btnDailyNarrative = st.button('📆Daily Narrative')
|
156 |
-
|
157 |
-
# btnDischargePlan = st.button('🗒️Discharge Plan')
|
158 |
-
# if btnDischargePlan:
|
159 |
-
# inputNote = "Input Discharge Plan"
|
160 |
-
# with col4:
|
161 |
-
# btnSocialNotes = st.button('📝Social Notes')
|
162 |
-
# if btnSocialNotes:
|
163 |
-
# inputNote = "Input Social Note"
|
164 |
-
with col5:
|
165 |
btnPastHistory = st.button('📇Past History (6 Mths)')
|
166 |
|
167 |
|
168 |
-
|
169 |
##======================== Start of NER Tagging ========================
|
170 |
|
171 |
#lemmatizing the notes to capture all forms of negation(e.g., deny: denies, denying)
|
@@ -234,11 +277,10 @@ def dedupe(items):
|
|
234 |
if item not in seen:
|
235 |
yield item
|
236 |
seen.add(item)
|
|
|
|
|
237 |
|
238 |
|
239 |
-
|
240 |
-
##======================== End of NER Tagging ========================
|
241 |
-
|
242 |
def run_model(input_text):
|
243 |
if model == "BertSummarizer":
|
244 |
output = original_text['BertSummarizer2s'].values
|
@@ -268,14 +310,175 @@ def run_model(input_text):
|
|
268 |
|
269 |
|
270 |
st.success(output)
|
|
|
|
|
271 |
|
272 |
-
|
273 |
-
|
274 |
-
|
275 |
-
|
276 |
-
|
277 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
278 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
279 |
with st.container():
|
280 |
with col6:
|
281 |
|
@@ -297,98 +500,23 @@ if btnPastHistory or st.session_state["past_button_clicked"]:
|
|
297 |
else:
|
298 |
#runtext = historyAdmission['hospital_course_processed'].values[0]
|
299 |
runtext = historyAdmission['hospital_course_processed'].values[0]
|
300 |
-
|
301 |
-
if btnAdmission:
|
302 |
-
#st.session_state["daily_button_clicked"] = False
|
303 |
-
#st.session_state["past_button_clicked"] = False
|
304 |
-
#st.session_state["button_clicked"] = True
|
305 |
-
runtext =st.text_area(inputNote, str(original_text2)[1:-1], height=300)
|
306 |
-
|
307 |
-
#if btnDailyNarrative:
|
308 |
-
#st.session_state["button_clicked"] = False
|
309 |
-
#st.session_state["past_button_clicked"] = False
|
310 |
-
#st.session_state["daily_button_clicked"] = True
|
311 |
-
|
312 |
-
|
313 |
|
|
|
|
|
|
|
|
|
314 |
|
|
|
|
|
315 |
|
316 |
-
|
317 |
-
#creating a doc object using BC5CDR model
|
318 |
-
doc = nlp(lem_clinical_note)
|
319 |
-
options = get_entity_options()
|
320 |
-
|
321 |
-
#list of negative concepts from clinical note identified by negspacy
|
322 |
-
results0 = negation_handling(lem_clinical_note, neg_model)
|
323 |
-
|
324 |
-
matcher = match(nlp, results0,"NEG_ENTITY")
|
325 |
-
|
326 |
-
#doc0: new doc object with added "NEG_ENTITY label"
|
327 |
-
doc0 = overwrite_ent_lbl(matcher,doc)
|
328 |
-
|
329 |
-
#visualizing identified Named Entities in clinical input text
|
330 |
-
ent_html = displacy.render(doc0, style='ent', options=options)
|
331 |
|
|
|
|
|
332 |
|
|
|
|
|
333 |
|
334 |
-
col1, col2 = st.columns([1,1])
|
335 |
-
|
336 |
-
#to not show summary and references text for Past History and Daily Narrative
|
337 |
-
if btnAdmission :
|
338 |
-
|
339 |
-
#st.session_state["daily_button_clicked"] = False
|
340 |
-
st.session_state["past_button_clicked"] = False
|
341 |
-
#st.session_state["button_clicked"] = True
|
342 |
-
|
343 |
-
|
344 |
-
with st.container():
|
345 |
-
with col1:
|
346 |
-
st.button('Summarize')
|
347 |
-
run_model(runtext)
|
348 |
-
#sentences=runtext.split('.')
|
349 |
-
st.text_area('Reference text', str(reference_text), height=150)
|
350 |
-
with col2:
|
351 |
-
st.button('NER')
|
352 |
-
# ===== Adding the Disease/Chemical into a list =====
|
353 |
-
problem_entities = list(dedupe([t for t in doc0.ents if t.label_ == 'DISEASE']))
|
354 |
-
medication_entities = list(dedupe([t for t in doc0.ents if t.label_ == 'CHEMICAL']))
|
355 |
-
st.markdown('**CHIEF COMPLAINT:**')
|
356 |
-
st.write(str(AdmissionChiefCom)[1:-1])
|
357 |
-
st.markdown('**ADMISSION DIAGNOSIS:**')
|
358 |
-
st.markdown(str(diagnosis)[1:-1].capitalize())
|
359 |
-
st.markdown('**PROBLEM/ISSUE**')
|
360 |
-
#st.markdown(problem_entities)
|
361 |
-
st.markdown(f'<p style="background-color:PINK;color:#080808;font-size:16px;">{str(problem_entities)[1:-1]}</p>', unsafe_allow_html=True)
|
362 |
-
#genEntities(trans_df, 'DISEASE')
|
363 |
-
st.markdown('**MEDICATION**')
|
364 |
-
st.markdown(f'<p style="background-color:orange;color:#080808;font-size:16px;">{str(medication_entities)[1:-1]}</p>', unsafe_allow_html=True)
|
365 |
-
#genEntities(trans_df, 'CHEMICAL')
|
366 |
-
#st.table(trans_df)
|
367 |
-
st.markdown('**NER**')
|
368 |
-
with st.expander("See NER Details"):
|
369 |
-
st.markdown(ent_html, unsafe_allow_html=True)
|
370 |
-
|
371 |
-
|
372 |
-
elif btnDailyNarrative :
|
373 |
-
# st.session_state["daily_button_clicked"] = True
|
374 |
-
st.session_state["past_button_clicked"] = False
|
375 |
-
# st.session_state["button_clicked"] = False
|
376 |
-
|
377 |
-
with st.container():
|
378 |
-
st.markdown('Daily Progress Note (24 hour event only):')
|
379 |
-
st.markdown(str(dailyNote)[1:-1])
|
380 |
-
|
381 |
-
|
382 |
-
with st.container():
|
383 |
-
styler = dailyNoteChange.style.hide_index()
|
384 |
-
st.write(styler.to_html(), unsafe_allow_html=True)
|
385 |
-
st.markdown(f'<p style="color:#828080;font-size:12px;">*Current prototype displays only a single section within the daily progress note, could also potentially include all sections within each progress note and allow user to select the section changes the user wants to look at</p>', unsafe_allow_html=True)
|
386 |
-
|
387 |
-
#else:
|
388 |
-
elif btnPastHistory or st.session_state["past_button_clicked"]:
|
389 |
-
st.session_state["past_button_clicked"] = True
|
390 |
-
# st.session_state["button_clicked"] = False
|
391 |
-
# st.session_state["daily_button_clicked"] = False
|
392 |
# ===== Adding the Disease/Chemical into a list =====
|
393 |
problem_entities = list(dedupe([t for t in doc0.ents if t.label_ == 'DISEASE']))
|
394 |
medication_entities = list(dedupe([t for t in doc0.ents if t.label_ == 'CHEMICAL']))
|
@@ -429,4 +557,40 @@ elif btnPastHistory or st.session_state["past_button_clicked"]:
|
|
429 |
with st.expander('Full Discharge Summary'):
|
430 |
#st.write("line 1 \n line 2 \n line 3")
|
431 |
fulldischargesummary = historyAdmission['TEXT'].values[0]
|
432 |
-
st.write(fulldischargesummary)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
import streamlit as st
|
2 |
+
import streamlit.components as components
|
3 |
+
from annotated_text import annotated_text, annotation
|
4 |
+
from htbuilder import h3
|
5 |
import pandas as pd
|
6 |
import numpy as np
|
7 |
from math import ceil
|
|
|
16 |
#import en_ner_bc5cdr_md
|
17 |
import re
|
18 |
from streamlit.components.v1 import html
|
19 |
+
import pickle
|
20 |
+
from functools import reduce
|
21 |
+
import operator
|
22 |
+
import itertools
|
23 |
+
from itertools import chain
|
24 |
+
from collections import Counter
|
25 |
+
from collections import OrderedDict
|
26 |
|
27 |
+
### ========== Loading Dataset ==========
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
28 |
## ======== Loading dataset ========
|
29 |
## Loading in Admission Dataset
|
30 |
+
## df = Admission
|
31 |
+
## df2 = Admission Chief Complaint and Diagnosis
|
32 |
+
## df3 = Discharge History
|
33 |
+
## df4 = Daily Narrative
|
34 |
+
# #=================================
|
35 |
+
nlp = spacy.load("en_ner_bc5cdr_md")
|
36 |
+
|
37 |
df = pd.read_csv('shpi25nov.csv')
|
38 |
df.sort_values(by='SUBJECT_ID',ascending = True, inplace=True)
|
39 |
|
|
|
40 |
df2 = pd.read_csv('cohort_cc_adm_diag.csv')
|
41 |
|
|
|
42 |
df3 = pd.read_csv('cohort_past_history_12072022.csv')
|
43 |
df3.sort_values(by='CHARTDATE',ascending = False, inplace=True)
|
44 |
|
|
|
45 |
df4 = pd.read_csv('24houreventsFulltextwdifference.csv')
|
46 |
+
#df4.sort_values(by=['hadmid','DATETIME'],ascending = True, inplace=True)
|
47 |
+
|
48 |
+
# Loading in Daily Narrative - refreshed full 24 hr text
|
49 |
+
df5 = pd.read_csv('24hourevents10Jan.csv')
|
50 |
+
df5.sort_values(by=['hadmid','DATETIME'],ascending = True, inplace=True)
|
51 |
+
|
52 |
+
#Append the updated 24 hr text and changes column
|
53 |
+
df5.rename(columns={'hadmid':'HADM_ID',
|
54 |
+
'DATETIME':'STORETIME'}, inplace = True)
|
55 |
+
df4 = pd.merge(df4[['HADM_ID','DESCRIPTION','SUBJECT_ID','CHARTTIME','STORETIME','CGID','TEXT','checks','_24_Hour_Events','Full_24_Hour_Events']],df5[['HADM_ID','STORETIME','full_24 Hour Events:','24 Hour Events:']], on = ['HADM_ID','STORETIME'], how = 'left')
|
56 |
+
|
57 |
+
hr24event_pattern = re.compile('((24 Hour Events):\\n(?s).*?Allergies:)')
|
58 |
+
|
59 |
+
#there are some records with full_24 Hour Events: null, hence replaced these text with the extracted text from the progress note
|
60 |
+
df4['hr24event_extracted'] = ''
|
61 |
+
for (idx, row) in df4.iterrows():
|
62 |
+
try:
|
63 |
+
text = df4['TEXT'][idx]
|
64 |
+
df4['hr24event_extracted'][idx] = re.findall(hr24event_pattern,text)
|
65 |
+
df4['hr24event_extracted'][idx] = [x for x in chain.from_iterable(df4['hr24event_extracted'][idx])]
|
66 |
+
except:
|
67 |
+
df4['hr24event_extracted'][idx] = ''
|
68 |
+
|
69 |
+
df4 = df4.reset_index(drop=True)
|
70 |
+
df4['hr24event_extracted'] = df4['hr24event_extracted'].apply(' '.join)
|
71 |
+
df4['hr24event_extracted'] = df4['hr24event_extracted'].str.replace('\s+[a-z]+:\\n', ' ')
|
72 |
+
df4['hr24event_extracted'] = df4['hr24event_extracted'].str.replace('24 Hour Events:|24 Hour Events|Allergies:', '')
|
73 |
+
df4['hr24event_extracted'] = df4['hr24event_extracted'].str.replace('\s+', ' ')
|
74 |
+
df4['hr24event_extracted'] = df4['hr24event_extracted'].str.replace('\.\s+\.', '.')
|
75 |
+
df4['hr24event_extracted'] = df4['hr24event_extracted'].replace(r"^ +| +$", r"", regex=True)
|
76 |
+
|
77 |
+
df4.loc[df4['full_24 Hour Events:'].isnull(),'full_24 Hour Events:'] = df4['hr24event_extracted']
|
78 |
+
df4.loc[df4['24 Hour Events:'].isnull(),'24 Hour Events:'] = df4['_24_Hour_Events']
|
79 |
|
80 |
# combining both data into one
|
81 |
df = pd.merge(df, df2, on=['HADM_ID','SUBJECT_ID'])
|
|
|
82 |
# Deleting admission chief complaint and diagnosis after combining
|
83 |
del df2
|
84 |
|
85 |
+
# Remove decimal point from Admission ID and format words
|
86 |
df['HADM_ID'] = df['HADM_ID'].astype(str).apply(lambda x: x.replace('.0',''))
|
87 |
df3['HADM_ID'] = df3['HADM_ID'].astype(str).apply(lambda x: x.replace('.0',''))
|
88 |
df4['HADM_ID'] = df4['HADM_ID'].astype(str).apply(lambda x: x.replace('.0',''))
|
|
|
105 |
'HADM_ID':'PAST_Admission_ID',
|
106 |
'INDEX_HADM_ID':'Admission_ID'}, inplace = True)
|
107 |
|
108 |
+
df4.rename(columns={'HADM_ID':'Admission_ID',
|
109 |
+
'full_24 Hour Events:':'Full Text',
|
110 |
+
'24 Hour Events:':'Change_Note',
|
111 |
+
'past_24 Hour Events:':'Past_Change_Note'}, inplace = True)
|
112 |
+
|
113 |
+
df4["Full Text"] =df4["Full Text"].replace('["[','').replace(']"]','')
|
114 |
|
115 |
+
## ========== Setting up Streamlit Sidebar ==========
|
116 |
+
st.set_page_config(page_title ='Patient Inpatient Progression Dashboard',
|
117 |
+
#page_icon= "Notes",
|
118 |
+
layout='wide')
|
119 |
+
st.title('Patient Inpatient Progression Dashboard')
|
120 |
+
st.markdown(
|
121 |
+
"""
|
122 |
+
<style>
|
123 |
+
[data-testid="stSidebar"][aria-expanded="true"] > div:first-child {
|
124 |
+
width: 400px;
|
125 |
+
}
|
126 |
+
[data-testid="stSidebar"][aria-expanded="false"] > div:first-child {
|
127 |
+
width: 400px;
|
128 |
+
margin-left: -230px;
|
129 |
+
}
|
130 |
+
</style>
|
131 |
+
""",
|
132 |
+
unsafe_allow_html=True,
|
133 |
+
)
|
134 |
+
st.sidebar.markdown('Using transformer model')
|
135 |
|
136 |
#Filter selection
|
137 |
st.sidebar.header("Search for Patient:")
|
|
|
144 |
pastHistoryEpDate = df3['CHARTDATE_HADM_ID'].loc[(df3['Patient_ID'] == patient) & (df3['Admission_ID']== HospitalAdmission)]
|
145 |
countOfAdmission = len(pastHistoryEpDate)
|
146 |
|
|
|
147 |
# List of Model available
|
148 |
+
#model = st.sidebar.selectbox('Select Model', ('BertSummarizer','BertGPT2','t5seq2eq','t5','gensim','pysummarizer'))
|
149 |
+
model = 'BertSummarizer'
|
150 |
+
st.sidebar.markdown('Model: ' + model)
|
151 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
152 |
original_text = df.query(
|
153 |
"Patient_ID == @patient & Admission_ID == @HospitalAdmission"
|
154 |
)
|
|
|
157 |
AdmissionChiefCom = original_text['Admission_Chief_Complaint'].values
|
158 |
diagnosis =original_text['DIAGNOSIS'].values
|
159 |
reference_text = original_text['Reference_text'].values
|
160 |
+
dailyNoteChange =df4[['STORETIME','Change_Note','Full Text']].loc[(df4['Admission_ID']==HospitalAdmission) & df4['_24_Hour_Events'].notnull()]
|
161 |
|
162 |
+
dailyNoteFull =df4[['STORETIME','Change_Note','Full Text']].loc[(df4['Admission_ID']==HospitalAdmission) & df4['_24_Hour_Events'].notnull()]
|
|
|
163 |
|
164 |
dailyNoteChange.rename(columns={'STORETIME':'Time of Record',
|
165 |
+
'Change_Note':'Note Changes'}, inplace = True)
|
166 |
+
|
167 |
+
#dailyNoteChange['Time of Record'] = pd.to_datetime(dailyNoteChange['Time of Record'])
|
168 |
+
|
169 |
+
dailyNoteChange['TimeDiff'] = pd.to_datetime(dailyNoteChange["Time of Record"], format='%Y/%m/%d %H:%M')
|
170 |
+
#dailyNoteChange['TimeDiff'] = pd.to_datetime(dailyNoteChange["Time of Record"], format='%d/%m/%Y %H:%M')
|
171 |
+
dailyNoteChange['TimeDiff'] = dailyNoteChange['TimeDiff'] -dailyNoteChange['TimeDiff'].shift()
|
172 |
+
dailyNoteChange['TimeDiff'] = dailyNoteChange['TimeDiff'].fillna(pd.Timedelta(seconds=0))
|
173 |
+
dailyNoteChange['TimeDiff']= dailyNoteChange['TimeDiff'].dt.total_seconds().div(60).astype(int)
|
174 |
+
dailyNoteChange['Hour'] = dailyNoteChange['TimeDiff'] // 60
|
175 |
+
dailyNoteChange['Mins'] = dailyNoteChange['TimeDiff']- dailyNoteChange['Hour'] * 60
|
176 |
+
dailyNoteChange["TimeDiff"] = dailyNoteChange['Hour'].astype(str) + " hours " + dailyNoteChange['Mins'].astype(str) + " Mins"
|
177 |
+
del dailyNoteChange['Hour']
|
178 |
+
del dailyNoteChange['Mins']
|
179 |
+
|
180 |
+
dailyNoteChange["PreviousRecord"] = dailyNoteChange["Time of Record"].shift()
|
181 |
+
|
182 |
+
dailyNoteChange.sort_values(by=['Time of Record'],ascending = False, inplace=True)
|
183 |
+
|
184 |
+
dailyNoteFull.rename(columns={'STORETIME':'Time of Record',
|
185 |
+
'Change_Note':'Note Changes'}, inplace = True)
|
186 |
+
|
187 |
+
dailyNote = df4['Full Text'].loc[(df4['Admission_ID']==HospitalAdmission)]
|
188 |
dailyNote = dailyNote.unique()
|
189 |
|
190 |
+
try:
|
191 |
+
mindate = min(dailyNoteFull['Time of Record'])
|
192 |
+
except:
|
193 |
+
mindate = ''
|
194 |
+
|
195 |
+
# ===== to display selected patient and admission id on main page
|
196 |
+
col3,col4 = st.columns(2)
|
197 |
+
patientid = col3.write(f"Patient ID: {patient} ")
|
198 |
+
admissionid =col4.write(f"Admission ID: {HospitalAdmission} ")
|
199 |
+
|
200 |
+
##========= Buttons to the 3 tabs ======== Temp disabled Discharge Plan and Social Notes
|
201 |
+
col1, col2, col3 = st.columns([1,1,1])
|
202 |
+
#col6, col7 =st.columns([2,2])
|
203 |
with st.container():
|
204 |
with col1:
|
205 |
btnAdmission = st.button("🏥 Admission")
|
|
|
|
|
206 |
with col2:
|
207 |
btnDailyNarrative = st.button('📆Daily Narrative')
|
208 |
+
with col3:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
209 |
btnPastHistory = st.button('📇Past History (6 Mths)')
|
210 |
|
211 |
|
|
|
212 |
##======================== Start of NER Tagging ========================
|
213 |
|
214 |
#lemmatizing the notes to capture all forms of negation(e.g., deny: denies, denying)
|
|
|
277 |
if item not in seen:
|
278 |
yield item
|
279 |
seen.add(item)
|
280 |
+
|
281 |
+
##======================== End of NER Tagging ========================
|
282 |
|
283 |
|
|
|
|
|
|
|
284 |
def run_model(input_text):
|
285 |
if model == "BertSummarizer":
|
286 |
output = original_text['BertSummarizer2s'].values
|
|
|
310 |
|
311 |
|
312 |
st.success(output)
|
313 |
+
|
314 |
+
def Admission():
|
315 |
|
316 |
+
with st.container():
|
317 |
+
|
318 |
+
runtext =st.text_area('History of presenting illnesses at admission', str(original_text2)[1:-1], height=300)
|
319 |
+
lem_clinical_note= lemmatize(runtext, nlp)
|
320 |
+
#creating a doc object using BC5CDR model
|
321 |
+
doc = nlp(lem_clinical_note)
|
322 |
+
options = get_entity_options()
|
323 |
+
|
324 |
+
#list of negative concepts from clinical note identified by negspacy
|
325 |
+
results0 = negation_handling(lem_clinical_note, neg_model)
|
326 |
+
|
327 |
+
matcher = match(nlp, results0,"NEG_ENTITY")
|
328 |
+
|
329 |
+
#doc0: new doc object with added "NEG_ENTITY label"
|
330 |
+
doc0 = overwrite_ent_lbl(matcher,doc)
|
331 |
+
|
332 |
+
#visualizing identified Named Entities in clinical input text
|
333 |
+
ent_html = displacy.render(doc0, style='ent', options=options)
|
334 |
+
|
335 |
+
col1, col2 = st.columns([1,1])
|
336 |
+
with st.container():
|
337 |
+
with col1:
|
338 |
+
st.button('Summarize')
|
339 |
+
run_model(runtext)
|
340 |
+
|
341 |
+
with col2:
|
342 |
+
st.button('NER')
|
343 |
+
# ===== Adding the Disease/Chemical into a list =====
|
344 |
+
problem_entities = list(dedupe([t for t in doc0.ents if t.label_ == 'DISEASE']))
|
345 |
+
medication_entities = list(dedupe([t for t in doc0.ents if t.label_ == 'CHEMICAL']))
|
346 |
+
st.markdown('**CHIEF COMPLAINT:**')
|
347 |
+
st.write(str(AdmissionChiefCom)[1:-1])
|
348 |
+
st.markdown('**ADMISSION DIAGNOSIS:**')
|
349 |
+
st.markdown(str(diagnosis)[1:-1].capitalize())
|
350 |
+
st.markdown('**PROBLEM/ISSUE**')
|
351 |
+
#st.markdown(problem_entities)
|
352 |
+
st.markdown(f'<p style="background-color:PINK;color:#080808;font-size:16px;">{str(problem_entities)[1:-1]}</p>', unsafe_allow_html=True)
|
353 |
+
#genEntities(trans_df, 'DISEASE')
|
354 |
+
st.markdown('**MEDICATION**')
|
355 |
+
st.markdown(f'<p style="background-color:orange;color:#080808;font-size:16px;">{str(medication_entities)[1:-1]}</p>', unsafe_allow_html=True)
|
356 |
+
#genEntities(trans_df, 'CHEMICAL')
|
357 |
+
#st.table(trans_df)
|
358 |
+
st.markdown('**NER**')
|
359 |
+
with st.expander("See NER Details"):
|
360 |
+
st.markdown(ent_html, unsafe_allow_html=True)
|
361 |
+
|
362 |
+
alphabets= "([A-Za-z])"
|
363 |
+
prefixes = "(mr|st|mrs|ms|dr)[.]"
|
364 |
+
suffixes = "(inc|ltd|jr|sr|co)"
|
365 |
+
starters = "(mr|mrs|ms|dr|he\s|she\s|it\s|they\s|their\s|our\s|we\s|but\s|however\s|that\s|this\s|wherever)"
|
366 |
+
acronyms = "([A-Z][.][A-Z][.](?:[A-Z][.])?)"
|
367 |
+
websites = "[.](com|net|org|io|gov)"
|
368 |
+
digits = "([0-9])"
|
369 |
+
|
370 |
+
def split_into_sentences(text):
|
371 |
+
# text = str(text)
|
372 |
+
text = " " + text + " "
|
373 |
+
text = text.replace("\n"," ")
|
374 |
+
# text = text.replace("[0-9]{4}-[0-9]{1,2}-[0-9]{1,2} [0-9]{2}:[0-9]{2}:[0-9]{2}"," ")
|
375 |
+
text = re.sub(prefixes,"\\1<prd>",text)
|
376 |
+
text = re.sub(websites,"<prd>\\1",text)
|
377 |
+
text = re.sub(digits + "[.]" + digits,"\\1<prd>\\2",text)
|
378 |
+
if "..." in text: text = text.replace("...","<prd><prd><prd>")
|
379 |
+
if "Ph.D" in text: text = text.replace("Ph.D.","Ph<prd>D<prd>")
|
380 |
+
text = re.sub("\s" + alphabets + "[.] "," \\1<prd> ",text)
|
381 |
+
text = re.sub(acronyms+" "+starters,"\\1<stop> \\2",text)
|
382 |
+
text = re.sub(alphabets + "[.]" + alphabets + "[.]" + alphabets + "[.]","\\1<prd>\\2<prd>\\3<prd>",text)
|
383 |
+
text = re.sub(alphabets + "[.]" + alphabets + "[.]","\\1<prd>\\2<prd>",text)
|
384 |
+
text = re.sub(" "+suffixes+"[.] "+starters," \\1<stop> \\2",text)
|
385 |
+
text = re.sub(" "+suffixes+"[.]"," \\1<prd>",text)
|
386 |
+
text = re.sub(" " + alphabets + "[.]"," \\1<prd>",text)
|
387 |
+
if "”" in text: text = text.replace(".”","”.")
|
388 |
+
if "\"" in text: text = text.replace(".\"","\".")
|
389 |
+
if "!" in text: text = text.replace("!\"","\"!")
|
390 |
+
if "?" in text: text = text.replace("?\"","\"?")
|
391 |
+
text = text.replace(".",".<stop>")
|
392 |
+
text = text.replace("?","?<stop>")
|
393 |
+
text = text.replace("!","!<stop>")
|
394 |
+
text = text.replace("[0-9]{2}:[0-9]{2}:[0-9]{2}:","[0-9]{2}:[0-9]{2}:[0-9]{2}:<stop>")
|
395 |
+
text = text.replace("[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}\s[0-9]{2}:[0-9]{2}:[0-9]{2}","[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}\s[0-9]{2}:[0-9]{2}:[0-9]{2}<stop>")
|
396 |
+
# text = text.replace("-","-<stop>")
|
397 |
+
# text = text.replace("- -","- -<stop>")
|
398 |
+
text = text.replace("<br><br>","<stop><br><br>")
|
399 |
+
text = text.replace("<prd>",".")
|
400 |
+
sentences = text.split("<stop>")
|
401 |
+
# sentences = text.split('-')
|
402 |
+
# sentences = sentences[:-1]
|
403 |
+
sentences = [s.strip() for s in sentences]
|
404 |
+
return sentences
|
405 |
+
|
406 |
+
def DailyNarrative():
|
407 |
+
with st.container():
|
408 |
+
dailyNarrativeTime= st.selectbox('',dailyNoteChange['Time of Record'])
|
409 |
+
|
410 |
+
if df4[['Change_Note']].loc[(df4['Admission_ID']==HospitalAdmission) & (df4['STORETIME'] == dailyNarrativeTime)].size != 0:
|
411 |
+
changeNote = df4[['Change_Note']].loc[(df4['Admission_ID']==HospitalAdmission) & (df4['STORETIME'] == dailyNarrativeTime)].values[0]
|
412 |
+
else:
|
413 |
+
changeNote = 'No records'
|
414 |
|
415 |
+
|
416 |
+
if dailyNoteChange['TimeDiff'].loc[(dailyNoteChange['Time of Record']==dailyNarrativeTime)].empty:
|
417 |
+
changeNoteTime = 'No records'
|
418 |
+
previousRecord = ' '
|
419 |
+
else:
|
420 |
+
changeNoteTime =dailyNoteChange['TimeDiff'].loc[(dailyNoteChange['Time of Record']==dailyNarrativeTime)].values[0]
|
421 |
+
previousRecord =dailyNoteChange['PreviousRecord'].loc[(dailyNoteChange['Time of Record']==dailyNarrativeTime)].values[0]
|
422 |
+
|
423 |
+
if dailyNarrativeTime == mindate:
|
424 |
+
changeNote = 'Nil'
|
425 |
+
else:
|
426 |
+
changeNote = str(changeNote).replace('["[','').replace(']"]','').replace("'","").replace('"','').replace(',','').replace('\\','').replace('[','').replace(']','').replace('\\','')
|
427 |
+
changeNote = changeNote.strip("[-,]").strip("")
|
428 |
+
changeNote = ' '.join(changeNote.split())
|
429 |
+
# changeNote_split = re.split(r'(?<=[^A-Z].[.?]) +(?=[A-Z])|-', changeNote)
|
430 |
+
# changeNote_split = [x.strip(' ') for x in changeNote_split]
|
431 |
+
|
432 |
+
changeNote_split = split_into_sentences(changeNote)
|
433 |
+
changeNote_split = [x for x in changeNote_split if x]
|
434 |
+
|
435 |
+
latestRecord = dailyNoteChange['Time of Record'].max()
|
436 |
+
st.markdown('Changes: ' + changeNote)
|
437 |
+
st.markdown('Changes recorded from previous record at ' + str(previousRecord) + ' , ' + str(changeNoteTime) + ' ago')
|
438 |
+
|
439 |
+
if df4[['Full Text']].loc[(df4['Admission_ID']==HospitalAdmission) & (df4['STORETIME'] == dailyNarrativeTime)].empty:
|
440 |
+
dailyNarrativeText = 'No Records'
|
441 |
+
else:
|
442 |
+
dailyNoteChange.sort_values(by='Time of Record',ascending = True, inplace=True)
|
443 |
+
dailyNoteChange["Combined"] = ''
|
444 |
+
count = 0
|
445 |
+
text =''
|
446 |
+
for index, row in dailyNoteChange.iterrows():
|
447 |
+
text = '[**' + str(row['Time of Record']) + '**]' + ':<stop> ' + row['Full Text'] + '<br>' + '<br>' + text
|
448 |
+
dailyNoteChange['Combined'].iloc[count] = text
|
449 |
+
count = count + 1
|
450 |
+
dailyNarrativeText =dailyNoteChange[['Combined']].loc[(dailyNoteChange['Time of Record'] == dailyNarrativeTime)].values[0]
|
451 |
+
#dailyNarrativeText =df4[['Full Text']].loc[(df4['Admission_ID']==HospitalAdmission) & (df4['DATETIME'] == dailyNarrativeTime)].values[0]
|
452 |
+
|
453 |
+
|
454 |
+
dailyNarrativeText = str(dailyNarrativeText).replace('["[','').replace(']"]','').replace("'","").replace(',','').replace('"','').replace('[','').replace(']','').replace('\\','')
|
455 |
+
dailyNarrativeText = dailyNarrativeText.strip("[-,]").strip(" ")
|
456 |
+
dailyNarrativeText = ' '.join(dailyNarrativeText.split())
|
457 |
+
# dailyNarrativeText_split = re.split(r'(?<=[^A-Z].[.?]) +(?=[A-Z])|-|<br><br>', dailyNarrativeText)
|
458 |
+
# dailyNarrativeText_split = [x.strip(' ') for x in dailyNarrativeText_split]
|
459 |
+
|
460 |
+
dailyNarrativeText_split = split_into_sentences(dailyNarrativeText)
|
461 |
+
|
462 |
+
#st.table(dailyNoteChange) # testing to see if data calculate correctly
|
463 |
+
|
464 |
+
with st.expander("See in detail"):
|
465 |
+
|
466 |
+
|
467 |
+
ls = []
|
468 |
+
|
469 |
+
for sent in dailyNarrativeText_split:
|
470 |
+
if sent in changeNote_split:
|
471 |
+
sent = sent.replace(str(sent),str(annotation(sent)))
|
472 |
+
ls.append(sent)
|
473 |
+
else:
|
474 |
+
ls.append(sent)
|
475 |
+
highlight = ' '.join(ls)
|
476 |
+
st.markdown(highlight, unsafe_allow_html=True)
|
477 |
+
|
478 |
+
|
479 |
+
|
480 |
+
def PastHistory():
|
481 |
+
col6, col7 =st.columns([2,2])
|
482 |
with st.container():
|
483 |
with col6:
|
484 |
|
|
|
500 |
else:
|
501 |
#runtext = historyAdmission['hospital_course_processed'].values[0]
|
502 |
runtext = historyAdmission['hospital_course_processed'].values[0]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
503 |
|
504 |
+
lem_clinical_note= lemmatize(runtext, nlp)
|
505 |
+
#creating a doc object using BC5CDR model
|
506 |
+
doc = nlp(lem_clinical_note)
|
507 |
+
options = get_entity_options()
|
508 |
|
509 |
+
#list of negative concepts from clinical note identified by negspacy
|
510 |
+
results0 = negation_handling(lem_clinical_note, neg_model)
|
511 |
|
512 |
+
matcher = match(nlp, results0,"NEG_ENTITY")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
513 |
|
514 |
+
#doc0: new doc object with added "NEG_ENTITY label"
|
515 |
+
doc0 = overwrite_ent_lbl(matcher,doc)
|
516 |
|
517 |
+
#visualizing identified Named Entities in clinical input text
|
518 |
+
ent_html = displacy.render(doc0, style='ent', options=options)
|
519 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
520 |
# ===== Adding the Disease/Chemical into a list =====
|
521 |
problem_entities = list(dedupe([t for t in doc0.ents if t.label_ == 'DISEASE']))
|
522 |
medication_entities = list(dedupe([t for t in doc0.ents if t.label_ == 'CHEMICAL']))
|
|
|
557 |
with st.expander('Full Discharge Summary'):
|
558 |
#st.write("line 1 \n line 2 \n line 3")
|
559 |
fulldischargesummary = historyAdmission['TEXT'].values[0]
|
560 |
+
st.write(fulldischargesummary)
|
561 |
+
|
562 |
+
if "load_state" not in st.session_state:
|
563 |
+
st.session_state.load_state = False
|
564 |
+
|
565 |
+
if "button_clicked" not in st.session_state:
|
566 |
+
st.session_state.button_clicked = False
|
567 |
+
|
568 |
+
if "admission_button_clicked" not in st.session_state:
|
569 |
+
st.session_state.admission_button_clicked = False
|
570 |
+
|
571 |
+
if "daily_button_clicked" not in st.session_state:
|
572 |
+
st.session_state.daily_button_clicked = False
|
573 |
+
|
574 |
+
if "past_button_clicked" not in st.session_state:
|
575 |
+
st.session_state.past_button_clicked = False
|
576 |
+
|
577 |
+
|
578 |
+
|
579 |
+
if btnAdmission or st.session_state["button_clicked"]:
|
580 |
+
st.session_state["admission_button_clicked"] = True
|
581 |
+
st.session_state["daily_button_clicked"] = False
|
582 |
+
st.session_state["past_button_clicked"] = False
|
583 |
+
Admission()
|
584 |
+
|
585 |
+
if btnDailyNarrative or st.session_state["daily_button_clicked"]:
|
586 |
+
st.session_state["daily_button_clicked"] = True
|
587 |
+
st.session_state["admission_button_clicked"] = False
|
588 |
+
st.session_state["past_button_clicked"] = False
|
589 |
+
DailyNarrative()
|
590 |
+
|
591 |
+
|
592 |
+
if btnPastHistory or st.session_state["past_button_clicked"]:
|
593 |
+
st.session_state["past_button_clicked"] = True
|
594 |
+
st.session_state["daily_button_clicked"] = False
|
595 |
+
st.session_state["admission_button_clicked"] = False
|
596 |
+
PastHistory()
|