carisackc commited on
Commit
ef1d0aa
·
1 Parent(s): 8a1a51e

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +337 -173
app.py CHANGED
@@ -1,4 +1,7 @@
1
  import streamlit as st
 
 
 
2
  import pandas as pd
3
  import numpy as np
4
  from math import ceil
@@ -13,67 +16,73 @@ from spacy.tokens import Span
13
  #import en_ner_bc5cdr_md
14
  import re
15
  from streamlit.components.v1 import html
 
 
 
 
 
 
 
16
 
17
- if "load_state" not in st.session_state:
18
- st.session_state.load_state = False
19
-
20
- #if "button_clicked" not in st.session_state:
21
- # st.session_state.button_clicked = True
22
- #
23
- #if "daily_button_clicked" not in st.session_state:
24
- # st.session_state.daily_button_clicked = False
25
-
26
- if "past_button_clicked" not in st.session_state:
27
- st.session_state.past_button_clicked = False
28
-
29
-
30
- #nlp = en_core_web_lg.load()
31
- nlp = spacy.load("en_ner_bc5cdr_md")
32
-
33
- st.set_page_config(page_title ='Patient Inpatient Progression Dashboard',
34
- #page_icon= "Notes",
35
- layout='wide')
36
- st.title('Patient Inpatient Progression Dashboard')
37
- st.markdown(
38
- """
39
- <style>
40
- [data-testid="stSidebar"][aria-expanded="true"] > div:first-child {
41
- width: 400px;
42
- }
43
- [data-testid="stSidebar"][aria-expanded="false"] > div:first-child {
44
- width: 400px;
45
- margin-left: -230px;
46
- }
47
- </style>
48
- """,
49
- unsafe_allow_html=True,
50
- )
51
- st.sidebar.markdown('Using transformer model')
52
-
53
  ## ======== Loading dataset ========
54
  ## Loading in Admission Dataset
 
 
 
 
 
 
 
55
  df = pd.read_csv('shpi25nov.csv')
56
  df.sort_values(by='SUBJECT_ID',ascending = True, inplace=True)
57
 
58
- # Loading in Admission chief Complaint and diagnosis
59
  df2 = pd.read_csv('cohort_cc_adm_diag.csv')
60
 
61
- # Loading in Dischare History
62
  df3 = pd.read_csv('cohort_past_history_12072022.csv')
63
  df3.sort_values(by='CHARTDATE',ascending = False, inplace=True)
64
 
65
- # Loading in Daily Narrative
66
  df4 = pd.read_csv('24houreventsFulltextwdifference.csv')
67
- df4.sort_values(by=['SUBJECT_ID','HADM_ID','STORETIME'],ascending = True, inplace=True)
68
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
69
 
70
  # combining both data into one
71
  df = pd.merge(df, df2, on=['HADM_ID','SUBJECT_ID'])
72
-
73
  # Deleting admission chief complaint and diagnosis after combining
74
  del df2
75
 
76
- # Remove decimal point from Admission ID
77
  df['HADM_ID'] = df['HADM_ID'].astype(str).apply(lambda x: x.replace('.0',''))
78
  df3['HADM_ID'] = df3['HADM_ID'].astype(str).apply(lambda x: x.replace('.0',''))
79
  df4['HADM_ID'] = df4['HADM_ID'].astype(str).apply(lambda x: x.replace('.0',''))
@@ -96,10 +105,33 @@ df3.rename(columns={'SUBJECT_ID':'Patient_ID',
96
  'HADM_ID':'PAST_Admission_ID',
97
  'INDEX_HADM_ID':'Admission_ID'}, inplace = True)
98
 
99
- df4.rename(columns={'SUBJECT_ID':'Patient_ID',
100
- 'HADM_ID':'Admission_ID',
101
- 'Full_24_Hour_Events':'Full Text'}, inplace = True)
 
 
 
102
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
103
 
104
  #Filter selection
105
  st.sidebar.header("Search for Patient:")
@@ -112,18 +144,11 @@ HospitalAdmission = st.sidebar.selectbox(' ', admissionid)
112
  pastHistoryEpDate = df3['CHARTDATE_HADM_ID'].loc[(df3['Patient_ID'] == patient) & (df3['Admission_ID']== HospitalAdmission)]
113
  countOfAdmission = len(pastHistoryEpDate)
114
 
115
-
116
  # List of Model available
117
- model = st.sidebar.selectbox('Select Model', ('BertSummarizer','BertGPT2','t5seq2eq','t5','gensim','pysummarizer'))
 
 
118
 
119
- # ===== to display selected patient and admission id on main page
120
- col3,col4 = st.columns(2)
121
- patientid = col3.write(f"Patient ID: {patient} ")
122
- admissionid =col4.write(f"Admission ID: {HospitalAdmission} ")
123
-
124
- runtext = ''
125
- inputNote ='Input note here:'
126
- # Query out relevant Clinical notes
127
  original_text = df.query(
128
  "Patient_ID == @patient & Admission_ID == @HospitalAdmission"
129
  )
@@ -132,40 +157,58 @@ original_text2 = original_text['Original_Text'].values
132
  AdmissionChiefCom = original_text['Admission_Chief_Complaint'].values
133
  diagnosis =original_text['DIAGNOSIS'].values
134
  reference_text = original_text['Reference_text'].values
 
135
 
136
- #dailyNoteChange = df4['_24_Hour_Events'].loc[(df4['Patient_ID'] == patient) & (df3['Admission_ID']== HospitalAdmission)]
137
- dailyNoteChange =df4[['STORETIME','_24_Hour_Events']].loc[(df4['Patient_ID'] == patient) & (df4['Admission_ID']==HospitalAdmission) & df4['_24_Hour_Events'].notnull()]
138
 
139
  dailyNoteChange.rename(columns={'STORETIME':'Time of Record',
140
- '_24_Hour_Events':'Note Changes'}, inplace = True)
141
- dailyNote = df4['Full Text'].loc[(df4['Patient_ID'] == patient) & (df4['Admission_ID']==HospitalAdmission)]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
142
  dailyNote = dailyNote.unique()
143
 
144
-
145
- ##========= Buttons to the 5 tabs ======== Temp disabled Discharge Plan and Social Notes
146
- ##col1, col2, col3, col4, col5 = st.columns([1,1,1,1,1]) -- to uncomment and comment below line to include discharge plan and social notes
147
- col1, col2, col5 = st.columns([1,1,1])
148
- col6, col7 =st.columns([2,2])
 
 
 
 
 
 
 
 
149
  with st.container():
150
  with col1:
151
  btnAdmission = st.button("🏥 Admission")
152
- inputNote = "Input Admission Note"
153
-
154
  with col2:
155
  btnDailyNarrative = st.button('📆Daily Narrative')
156
- # with col3:what
157
- # btnDischargePlan = st.button('🗒️Discharge Plan')
158
- # if btnDischargePlan:
159
- # inputNote = "Input Discharge Plan"
160
- # with col4:
161
- # btnSocialNotes = st.button('📝Social Notes')
162
- # if btnSocialNotes:
163
- # inputNote = "Input Social Note"
164
- with col5:
165
  btnPastHistory = st.button('📇Past History (6 Mths)')
166
 
167
 
168
-
169
  ##======================== Start of NER Tagging ========================
170
 
171
  #lemmatizing the notes to capture all forms of negation(e.g., deny: denies, denying)
@@ -234,11 +277,10 @@ def dedupe(items):
234
  if item not in seen:
235
  yield item
236
  seen.add(item)
 
 
237
 
238
 
239
-
240
- ##======================== End of NER Tagging ========================
241
-
242
  def run_model(input_text):
243
  if model == "BertSummarizer":
244
  output = original_text['BertSummarizer2s'].values
@@ -268,14 +310,175 @@ def run_model(input_text):
268
 
269
 
270
  st.success(output)
 
 
271
 
272
- ##========= on Past History Tab =========
273
-
274
- if btnPastHistory or st.session_state["past_button_clicked"]:
275
- #st.session_state["button_clicked"] = False
276
- #st.session_state["daily_button_clicked"] = False
277
- st.session_state["past_button_clicked"] = True
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
278
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
279
  with st.container():
280
  with col6:
281
 
@@ -297,98 +500,23 @@ if btnPastHistory or st.session_state["past_button_clicked"]:
297
  else:
298
  #runtext = historyAdmission['hospital_course_processed'].values[0]
299
  runtext = historyAdmission['hospital_course_processed'].values[0]
300
-
301
- if btnAdmission:
302
- #st.session_state["daily_button_clicked"] = False
303
- #st.session_state["past_button_clicked"] = False
304
- #st.session_state["button_clicked"] = True
305
- runtext =st.text_area(inputNote, str(original_text2)[1:-1], height=300)
306
-
307
- #if btnDailyNarrative:
308
- #st.session_state["button_clicked"] = False
309
- #st.session_state["past_button_clicked"] = False
310
- #st.session_state["daily_button_clicked"] = True
311
-
312
-
313
 
 
 
 
 
314
 
 
 
315
 
316
- lem_clinical_note= lemmatize(runtext, nlp)
317
- #creating a doc object using BC5CDR model
318
- doc = nlp(lem_clinical_note)
319
- options = get_entity_options()
320
-
321
- #list of negative concepts from clinical note identified by negspacy
322
- results0 = negation_handling(lem_clinical_note, neg_model)
323
-
324
- matcher = match(nlp, results0,"NEG_ENTITY")
325
-
326
- #doc0: new doc object with added "NEG_ENTITY label"
327
- doc0 = overwrite_ent_lbl(matcher,doc)
328
-
329
- #visualizing identified Named Entities in clinical input text
330
- ent_html = displacy.render(doc0, style='ent', options=options)
331
 
 
 
332
 
 
 
333
 
334
- col1, col2 = st.columns([1,1])
335
-
336
- #to not show summary and references text for Past History and Daily Narrative
337
- if btnAdmission :
338
-
339
- #st.session_state["daily_button_clicked"] = False
340
- st.session_state["past_button_clicked"] = False
341
- #st.session_state["button_clicked"] = True
342
-
343
-
344
- with st.container():
345
- with col1:
346
- st.button('Summarize')
347
- run_model(runtext)
348
- #sentences=runtext.split('.')
349
- st.text_area('Reference text', str(reference_text), height=150)
350
- with col2:
351
- st.button('NER')
352
- # ===== Adding the Disease/Chemical into a list =====
353
- problem_entities = list(dedupe([t for t in doc0.ents if t.label_ == 'DISEASE']))
354
- medication_entities = list(dedupe([t for t in doc0.ents if t.label_ == 'CHEMICAL']))
355
- st.markdown('**CHIEF COMPLAINT:**')
356
- st.write(str(AdmissionChiefCom)[1:-1])
357
- st.markdown('**ADMISSION DIAGNOSIS:**')
358
- st.markdown(str(diagnosis)[1:-1].capitalize())
359
- st.markdown('**PROBLEM/ISSUE**')
360
- #st.markdown(problem_entities)
361
- st.markdown(f'<p style="background-color:PINK;color:#080808;font-size:16px;">{str(problem_entities)[1:-1]}</p>', unsafe_allow_html=True)
362
- #genEntities(trans_df, 'DISEASE')
363
- st.markdown('**MEDICATION**')
364
- st.markdown(f'<p style="background-color:orange;color:#080808;font-size:16px;">{str(medication_entities)[1:-1]}</p>', unsafe_allow_html=True)
365
- #genEntities(trans_df, 'CHEMICAL')
366
- #st.table(trans_df)
367
- st.markdown('**NER**')
368
- with st.expander("See NER Details"):
369
- st.markdown(ent_html, unsafe_allow_html=True)
370
-
371
-
372
- elif btnDailyNarrative :
373
- # st.session_state["daily_button_clicked"] = True
374
- st.session_state["past_button_clicked"] = False
375
- # st.session_state["button_clicked"] = False
376
-
377
- with st.container():
378
- st.markdown('Daily Progress Note (24 hour event only):')
379
- st.markdown(str(dailyNote)[1:-1])
380
-
381
-
382
- with st.container():
383
- styler = dailyNoteChange.style.hide_index()
384
- st.write(styler.to_html(), unsafe_allow_html=True)
385
- st.markdown(f'<p style="color:#828080;font-size:12px;">*Current prototype displays only a single section within the daily progress note, could also potentially include all sections within each progress note and allow user to select the section changes the user wants to look at</p>', unsafe_allow_html=True)
386
-
387
- #else:
388
- elif btnPastHistory or st.session_state["past_button_clicked"]:
389
- st.session_state["past_button_clicked"] = True
390
- # st.session_state["button_clicked"] = False
391
- # st.session_state["daily_button_clicked"] = False
392
  # ===== Adding the Disease/Chemical into a list =====
393
  problem_entities = list(dedupe([t for t in doc0.ents if t.label_ == 'DISEASE']))
394
  medication_entities = list(dedupe([t for t in doc0.ents if t.label_ == 'CHEMICAL']))
@@ -429,4 +557,40 @@ elif btnPastHistory or st.session_state["past_button_clicked"]:
429
  with st.expander('Full Discharge Summary'):
430
  #st.write("line 1 \n line 2 \n line 3")
431
  fulldischargesummary = historyAdmission['TEXT'].values[0]
432
- st.write(fulldischargesummary)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  import streamlit as st
2
+ import streamlit.components as components
3
+ from annotated_text import annotated_text, annotation
4
+ from htbuilder import h3
5
  import pandas as pd
6
  import numpy as np
7
  from math import ceil
 
16
  #import en_ner_bc5cdr_md
17
  import re
18
  from streamlit.components.v1 import html
19
+ import pickle
20
+ from functools import reduce
21
+ import operator
22
+ import itertools
23
+ from itertools import chain
24
+ from collections import Counter
25
+ from collections import OrderedDict
26
 
27
+ ### ========== Loading Dataset ==========
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  ## ======== Loading dataset ========
29
  ## Loading in Admission Dataset
30
+ ## df = Admission
31
+ ## df2 = Admission Chief Complaint and Diagnosis
32
+ ## df3 = Discharge History
33
+ ## df4 = Daily Narrative
34
+ # #=================================
35
+ nlp = spacy.load("en_ner_bc5cdr_md")
36
+
37
  df = pd.read_csv('shpi25nov.csv')
38
  df.sort_values(by='SUBJECT_ID',ascending = True, inplace=True)
39
 
 
40
  df2 = pd.read_csv('cohort_cc_adm_diag.csv')
41
 
 
42
  df3 = pd.read_csv('cohort_past_history_12072022.csv')
43
  df3.sort_values(by='CHARTDATE',ascending = False, inplace=True)
44
 
 
45
  df4 = pd.read_csv('24houreventsFulltextwdifference.csv')
46
+ #df4.sort_values(by=['hadmid','DATETIME'],ascending = True, inplace=True)
47
+
48
+ # Loading in Daily Narrative - refreshed full 24 hr text
49
+ df5 = pd.read_csv('24hourevents10Jan.csv')
50
+ df5.sort_values(by=['hadmid','DATETIME'],ascending = True, inplace=True)
51
+
52
+ #Append the updated 24 hr text and changes column
53
+ df5.rename(columns={'hadmid':'HADM_ID',
54
+ 'DATETIME':'STORETIME'}, inplace = True)
55
+ df4 = pd.merge(df4[['HADM_ID','DESCRIPTION','SUBJECT_ID','CHARTTIME','STORETIME','CGID','TEXT','checks','_24_Hour_Events','Full_24_Hour_Events']],df5[['HADM_ID','STORETIME','full_24 Hour Events:','24 Hour Events:']], on = ['HADM_ID','STORETIME'], how = 'left')
56
+
57
+ hr24event_pattern = re.compile('((24 Hour Events):\\n(?s).*?Allergies:)')
58
+
59
+ #there are some records with full_24 Hour Events: null, hence replaced these text with the extracted text from the progress note
60
+ df4['hr24event_extracted'] = ''
61
+ for (idx, row) in df4.iterrows():
62
+ try:
63
+ text = df4['TEXT'][idx]
64
+ df4['hr24event_extracted'][idx] = re.findall(hr24event_pattern,text)
65
+ df4['hr24event_extracted'][idx] = [x for x in chain.from_iterable(df4['hr24event_extracted'][idx])]
66
+ except:
67
+ df4['hr24event_extracted'][idx] = ''
68
+
69
+ df4 = df4.reset_index(drop=True)
70
+ df4['hr24event_extracted'] = df4['hr24event_extracted'].apply(' '.join)
71
+ df4['hr24event_extracted'] = df4['hr24event_extracted'].str.replace('\s+[a-z]+:\\n', ' ')
72
+ df4['hr24event_extracted'] = df4['hr24event_extracted'].str.replace('24 Hour Events:|24 Hour Events|Allergies:', '')
73
+ df4['hr24event_extracted'] = df4['hr24event_extracted'].str.replace('\s+', ' ')
74
+ df4['hr24event_extracted'] = df4['hr24event_extracted'].str.replace('\.\s+\.', '.')
75
+ df4['hr24event_extracted'] = df4['hr24event_extracted'].replace(r"^ +| +$", r"", regex=True)
76
+
77
+ df4.loc[df4['full_24 Hour Events:'].isnull(),'full_24 Hour Events:'] = df4['hr24event_extracted']
78
+ df4.loc[df4['24 Hour Events:'].isnull(),'24 Hour Events:'] = df4['_24_Hour_Events']
79
 
80
  # combining both data into one
81
  df = pd.merge(df, df2, on=['HADM_ID','SUBJECT_ID'])
 
82
  # Deleting admission chief complaint and diagnosis after combining
83
  del df2
84
 
85
+ # Remove decimal point from Admission ID and format words
86
  df['HADM_ID'] = df['HADM_ID'].astype(str).apply(lambda x: x.replace('.0',''))
87
  df3['HADM_ID'] = df3['HADM_ID'].astype(str).apply(lambda x: x.replace('.0',''))
88
  df4['HADM_ID'] = df4['HADM_ID'].astype(str).apply(lambda x: x.replace('.0',''))
 
105
  'HADM_ID':'PAST_Admission_ID',
106
  'INDEX_HADM_ID':'Admission_ID'}, inplace = True)
107
 
108
+ df4.rename(columns={'HADM_ID':'Admission_ID',
109
+ 'full_24 Hour Events:':'Full Text',
110
+ '24 Hour Events:':'Change_Note',
111
+ 'past_24 Hour Events:':'Past_Change_Note'}, inplace = True)
112
+
113
+ df4["Full Text"] =df4["Full Text"].replace('["[','').replace(']"]','')
114
 
115
+ ## ========== Setting up Streamlit Sidebar ==========
116
+ st.set_page_config(page_title ='Patient Inpatient Progression Dashboard',
117
+ #page_icon= "Notes",
118
+ layout='wide')
119
+ st.title('Patient Inpatient Progression Dashboard')
120
+ st.markdown(
121
+ """
122
+ <style>
123
+ [data-testid="stSidebar"][aria-expanded="true"] > div:first-child {
124
+ width: 400px;
125
+ }
126
+ [data-testid="stSidebar"][aria-expanded="false"] > div:first-child {
127
+ width: 400px;
128
+ margin-left: -230px;
129
+ }
130
+ </style>
131
+ """,
132
+ unsafe_allow_html=True,
133
+ )
134
+ st.sidebar.markdown('Using transformer model')
135
 
136
  #Filter selection
137
  st.sidebar.header("Search for Patient:")
 
144
  pastHistoryEpDate = df3['CHARTDATE_HADM_ID'].loc[(df3['Patient_ID'] == patient) & (df3['Admission_ID']== HospitalAdmission)]
145
  countOfAdmission = len(pastHistoryEpDate)
146
 
 
147
  # List of Model available
148
+ #model = st.sidebar.selectbox('Select Model', ('BertSummarizer','BertGPT2','t5seq2eq','t5','gensim','pysummarizer'))
149
+ model = 'BertSummarizer'
150
+ st.sidebar.markdown('Model: ' + model)
151
 
 
 
 
 
 
 
 
 
152
  original_text = df.query(
153
  "Patient_ID == @patient & Admission_ID == @HospitalAdmission"
154
  )
 
157
  AdmissionChiefCom = original_text['Admission_Chief_Complaint'].values
158
  diagnosis =original_text['DIAGNOSIS'].values
159
  reference_text = original_text['Reference_text'].values
160
+ dailyNoteChange =df4[['STORETIME','Change_Note','Full Text']].loc[(df4['Admission_ID']==HospitalAdmission) & df4['_24_Hour_Events'].notnull()]
161
 
162
+ dailyNoteFull =df4[['STORETIME','Change_Note','Full Text']].loc[(df4['Admission_ID']==HospitalAdmission) & df4['_24_Hour_Events'].notnull()]
 
163
 
164
  dailyNoteChange.rename(columns={'STORETIME':'Time of Record',
165
+ 'Change_Note':'Note Changes'}, inplace = True)
166
+
167
+ #dailyNoteChange['Time of Record'] = pd.to_datetime(dailyNoteChange['Time of Record'])
168
+
169
+ dailyNoteChange['TimeDiff'] = pd.to_datetime(dailyNoteChange["Time of Record"], format='%Y/%m/%d %H:%M')
170
+ #dailyNoteChange['TimeDiff'] = pd.to_datetime(dailyNoteChange["Time of Record"], format='%d/%m/%Y %H:%M')
171
+ dailyNoteChange['TimeDiff'] = dailyNoteChange['TimeDiff'] -dailyNoteChange['TimeDiff'].shift()
172
+ dailyNoteChange['TimeDiff'] = dailyNoteChange['TimeDiff'].fillna(pd.Timedelta(seconds=0))
173
+ dailyNoteChange['TimeDiff']= dailyNoteChange['TimeDiff'].dt.total_seconds().div(60).astype(int)
174
+ dailyNoteChange['Hour'] = dailyNoteChange['TimeDiff'] // 60
175
+ dailyNoteChange['Mins'] = dailyNoteChange['TimeDiff']- dailyNoteChange['Hour'] * 60
176
+ dailyNoteChange["TimeDiff"] = dailyNoteChange['Hour'].astype(str) + " hours " + dailyNoteChange['Mins'].astype(str) + " Mins"
177
+ del dailyNoteChange['Hour']
178
+ del dailyNoteChange['Mins']
179
+
180
+ dailyNoteChange["PreviousRecord"] = dailyNoteChange["Time of Record"].shift()
181
+
182
+ dailyNoteChange.sort_values(by=['Time of Record'],ascending = False, inplace=True)
183
+
184
+ dailyNoteFull.rename(columns={'STORETIME':'Time of Record',
185
+ 'Change_Note':'Note Changes'}, inplace = True)
186
+
187
+ dailyNote = df4['Full Text'].loc[(df4['Admission_ID']==HospitalAdmission)]
188
  dailyNote = dailyNote.unique()
189
 
190
+ try:
191
+ mindate = min(dailyNoteFull['Time of Record'])
192
+ except:
193
+ mindate = ''
194
+
195
+ # ===== to display selected patient and admission id on main page
196
+ col3,col4 = st.columns(2)
197
+ patientid = col3.write(f"Patient ID: {patient} ")
198
+ admissionid =col4.write(f"Admission ID: {HospitalAdmission} ")
199
+
200
+ ##========= Buttons to the 3 tabs ======== Temp disabled Discharge Plan and Social Notes
201
+ col1, col2, col3 = st.columns([1,1,1])
202
+ #col6, col7 =st.columns([2,2])
203
  with st.container():
204
  with col1:
205
  btnAdmission = st.button("🏥 Admission")
 
 
206
  with col2:
207
  btnDailyNarrative = st.button('📆Daily Narrative')
208
+ with col3:
 
 
 
 
 
 
 
 
209
  btnPastHistory = st.button('📇Past History (6 Mths)')
210
 
211
 
 
212
  ##======================== Start of NER Tagging ========================
213
 
214
  #lemmatizing the notes to capture all forms of negation(e.g., deny: denies, denying)
 
277
  if item not in seen:
278
  yield item
279
  seen.add(item)
280
+
281
+ ##======================== End of NER Tagging ========================
282
 
283
 
 
 
 
284
  def run_model(input_text):
285
  if model == "BertSummarizer":
286
  output = original_text['BertSummarizer2s'].values
 
310
 
311
 
312
  st.success(output)
313
+
314
+ def Admission():
315
 
316
+ with st.container():
317
+
318
+ runtext =st.text_area('History of presenting illnesses at admission', str(original_text2)[1:-1], height=300)
319
+ lem_clinical_note= lemmatize(runtext, nlp)
320
+ #creating a doc object using BC5CDR model
321
+ doc = nlp(lem_clinical_note)
322
+ options = get_entity_options()
323
+
324
+ #list of negative concepts from clinical note identified by negspacy
325
+ results0 = negation_handling(lem_clinical_note, neg_model)
326
+
327
+ matcher = match(nlp, results0,"NEG_ENTITY")
328
+
329
+ #doc0: new doc object with added "NEG_ENTITY label"
330
+ doc0 = overwrite_ent_lbl(matcher,doc)
331
+
332
+ #visualizing identified Named Entities in clinical input text
333
+ ent_html = displacy.render(doc0, style='ent', options=options)
334
+
335
+ col1, col2 = st.columns([1,1])
336
+ with st.container():
337
+ with col1:
338
+ st.button('Summarize')
339
+ run_model(runtext)
340
+
341
+ with col2:
342
+ st.button('NER')
343
+ # ===== Adding the Disease/Chemical into a list =====
344
+ problem_entities = list(dedupe([t for t in doc0.ents if t.label_ == 'DISEASE']))
345
+ medication_entities = list(dedupe([t for t in doc0.ents if t.label_ == 'CHEMICAL']))
346
+ st.markdown('**CHIEF COMPLAINT:**')
347
+ st.write(str(AdmissionChiefCom)[1:-1])
348
+ st.markdown('**ADMISSION DIAGNOSIS:**')
349
+ st.markdown(str(diagnosis)[1:-1].capitalize())
350
+ st.markdown('**PROBLEM/ISSUE**')
351
+ #st.markdown(problem_entities)
352
+ st.markdown(f'<p style="background-color:PINK;color:#080808;font-size:16px;">{str(problem_entities)[1:-1]}</p>', unsafe_allow_html=True)
353
+ #genEntities(trans_df, 'DISEASE')
354
+ st.markdown('**MEDICATION**')
355
+ st.markdown(f'<p style="background-color:orange;color:#080808;font-size:16px;">{str(medication_entities)[1:-1]}</p>', unsafe_allow_html=True)
356
+ #genEntities(trans_df, 'CHEMICAL')
357
+ #st.table(trans_df)
358
+ st.markdown('**NER**')
359
+ with st.expander("See NER Details"):
360
+ st.markdown(ent_html, unsafe_allow_html=True)
361
+
362
+ alphabets= "([A-Za-z])"
363
+ prefixes = "(mr|st|mrs|ms|dr)[.]"
364
+ suffixes = "(inc|ltd|jr|sr|co)"
365
+ starters = "(mr|mrs|ms|dr|he\s|she\s|it\s|they\s|their\s|our\s|we\s|but\s|however\s|that\s|this\s|wherever)"
366
+ acronyms = "([A-Z][.][A-Z][.](?:[A-Z][.])?)"
367
+ websites = "[.](com|net|org|io|gov)"
368
+ digits = "([0-9])"
369
+
370
+ def split_into_sentences(text):
371
+ # text = str(text)
372
+ text = " " + text + " "
373
+ text = text.replace("\n"," ")
374
+ # text = text.replace("[0-9]{4}-[0-9]{1,2}-[0-9]{1,2} [0-9]{2}:[0-9]{2}:[0-9]{2}"," ")
375
+ text = re.sub(prefixes,"\\1<prd>",text)
376
+ text = re.sub(websites,"<prd>\\1",text)
377
+ text = re.sub(digits + "[.]" + digits,"\\1<prd>\\2",text)
378
+ if "..." in text: text = text.replace("...","<prd><prd><prd>")
379
+ if "Ph.D" in text: text = text.replace("Ph.D.","Ph<prd>D<prd>")
380
+ text = re.sub("\s" + alphabets + "[.] "," \\1<prd> ",text)
381
+ text = re.sub(acronyms+" "+starters,"\\1<stop> \\2",text)
382
+ text = re.sub(alphabets + "[.]" + alphabets + "[.]" + alphabets + "[.]","\\1<prd>\\2<prd>\\3<prd>",text)
383
+ text = re.sub(alphabets + "[.]" + alphabets + "[.]","\\1<prd>\\2<prd>",text)
384
+ text = re.sub(" "+suffixes+"[.] "+starters," \\1<stop> \\2",text)
385
+ text = re.sub(" "+suffixes+"[.]"," \\1<prd>",text)
386
+ text = re.sub(" " + alphabets + "[.]"," \\1<prd>",text)
387
+ if "”" in text: text = text.replace(".”","”.")
388
+ if "\"" in text: text = text.replace(".\"","\".")
389
+ if "!" in text: text = text.replace("!\"","\"!")
390
+ if "?" in text: text = text.replace("?\"","\"?")
391
+ text = text.replace(".",".<stop>")
392
+ text = text.replace("?","?<stop>")
393
+ text = text.replace("!","!<stop>")
394
+ text = text.replace("[0-9]{2}:[0-9]{2}:[0-9]{2}:","[0-9]{2}:[0-9]{2}:[0-9]{2}:<stop>")
395
+ text = text.replace("[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}\s[0-9]{2}:[0-9]{2}:[0-9]{2}","[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}\s[0-9]{2}:[0-9]{2}:[0-9]{2}<stop>")
396
+ # text = text.replace("-","-<stop>")
397
+ # text = text.replace("- -","- -<stop>")
398
+ text = text.replace("<br><br>","<stop><br><br>")
399
+ text = text.replace("<prd>",".")
400
+ sentences = text.split("<stop>")
401
+ # sentences = text.split('-')
402
+ # sentences = sentences[:-1]
403
+ sentences = [s.strip() for s in sentences]
404
+ return sentences
405
+
406
+ def DailyNarrative():
407
+ with st.container():
408
+ dailyNarrativeTime= st.selectbox('',dailyNoteChange['Time of Record'])
409
+
410
+ if df4[['Change_Note']].loc[(df4['Admission_ID']==HospitalAdmission) & (df4['STORETIME'] == dailyNarrativeTime)].size != 0:
411
+ changeNote = df4[['Change_Note']].loc[(df4['Admission_ID']==HospitalAdmission) & (df4['STORETIME'] == dailyNarrativeTime)].values[0]
412
+ else:
413
+ changeNote = 'No records'
414
 
415
+
416
+ if dailyNoteChange['TimeDiff'].loc[(dailyNoteChange['Time of Record']==dailyNarrativeTime)].empty:
417
+ changeNoteTime = 'No records'
418
+ previousRecord = ' '
419
+ else:
420
+ changeNoteTime =dailyNoteChange['TimeDiff'].loc[(dailyNoteChange['Time of Record']==dailyNarrativeTime)].values[0]
421
+ previousRecord =dailyNoteChange['PreviousRecord'].loc[(dailyNoteChange['Time of Record']==dailyNarrativeTime)].values[0]
422
+
423
+ if dailyNarrativeTime == mindate:
424
+ changeNote = 'Nil'
425
+ else:
426
+ changeNote = str(changeNote).replace('["[','').replace(']"]','').replace("'","").replace('"','').replace(',','').replace('\\','').replace('[','').replace(']','').replace('\\','')
427
+ changeNote = changeNote.strip("[-,]").strip("")
428
+ changeNote = ' '.join(changeNote.split())
429
+ # changeNote_split = re.split(r'(?<=[^A-Z].[.?]) +(?=[A-Z])|-', changeNote)
430
+ # changeNote_split = [x.strip(' ') for x in changeNote_split]
431
+
432
+ changeNote_split = split_into_sentences(changeNote)
433
+ changeNote_split = [x for x in changeNote_split if x]
434
+
435
+ latestRecord = dailyNoteChange['Time of Record'].max()
436
+ st.markdown('Changes: ' + changeNote)
437
+ st.markdown('Changes recorded from previous record at ' + str(previousRecord) + ' , ' + str(changeNoteTime) + ' ago')
438
+
439
+ if df4[['Full Text']].loc[(df4['Admission_ID']==HospitalAdmission) & (df4['STORETIME'] == dailyNarrativeTime)].empty:
440
+ dailyNarrativeText = 'No Records'
441
+ else:
442
+ dailyNoteChange.sort_values(by='Time of Record',ascending = True, inplace=True)
443
+ dailyNoteChange["Combined"] = ''
444
+ count = 0
445
+ text =''
446
+ for index, row in dailyNoteChange.iterrows():
447
+ text = '[**' + str(row['Time of Record']) + '**]' + ':<stop> ' + row['Full Text'] + '<br>' + '<br>' + text
448
+ dailyNoteChange['Combined'].iloc[count] = text
449
+ count = count + 1
450
+ dailyNarrativeText =dailyNoteChange[['Combined']].loc[(dailyNoteChange['Time of Record'] == dailyNarrativeTime)].values[0]
451
+ #dailyNarrativeText =df4[['Full Text']].loc[(df4['Admission_ID']==HospitalAdmission) & (df4['DATETIME'] == dailyNarrativeTime)].values[0]
452
+
453
+
454
+ dailyNarrativeText = str(dailyNarrativeText).replace('["[','').replace(']"]','').replace("'","").replace(',','').replace('"','').replace('[','').replace(']','').replace('\\','')
455
+ dailyNarrativeText = dailyNarrativeText.strip("[-,]").strip(" ")
456
+ dailyNarrativeText = ' '.join(dailyNarrativeText.split())
457
+ # dailyNarrativeText_split = re.split(r'(?<=[^A-Z].[.?]) +(?=[A-Z])|-|<br><br>', dailyNarrativeText)
458
+ # dailyNarrativeText_split = [x.strip(' ') for x in dailyNarrativeText_split]
459
+
460
+ dailyNarrativeText_split = split_into_sentences(dailyNarrativeText)
461
+
462
+ #st.table(dailyNoteChange) # testing to see if data calculate correctly
463
+
464
+ with st.expander("See in detail"):
465
+
466
+
467
+ ls = []
468
+
469
+ for sent in dailyNarrativeText_split:
470
+ if sent in changeNote_split:
471
+ sent = sent.replace(str(sent),str(annotation(sent)))
472
+ ls.append(sent)
473
+ else:
474
+ ls.append(sent)
475
+ highlight = ' '.join(ls)
476
+ st.markdown(highlight, unsafe_allow_html=True)
477
+
478
+
479
+
480
+ def PastHistory():
481
+ col6, col7 =st.columns([2,2])
482
  with st.container():
483
  with col6:
484
 
 
500
  else:
501
  #runtext = historyAdmission['hospital_course_processed'].values[0]
502
  runtext = historyAdmission['hospital_course_processed'].values[0]
 
 
 
 
 
 
 
 
 
 
 
 
 
503
 
504
+ lem_clinical_note= lemmatize(runtext, nlp)
505
+ #creating a doc object using BC5CDR model
506
+ doc = nlp(lem_clinical_note)
507
+ options = get_entity_options()
508
 
509
+ #list of negative concepts from clinical note identified by negspacy
510
+ results0 = negation_handling(lem_clinical_note, neg_model)
511
 
512
+ matcher = match(nlp, results0,"NEG_ENTITY")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
513
 
514
+ #doc0: new doc object with added "NEG_ENTITY label"
515
+ doc0 = overwrite_ent_lbl(matcher,doc)
516
 
517
+ #visualizing identified Named Entities in clinical input text
518
+ ent_html = displacy.render(doc0, style='ent', options=options)
519
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
520
  # ===== Adding the Disease/Chemical into a list =====
521
  problem_entities = list(dedupe([t for t in doc0.ents if t.label_ == 'DISEASE']))
522
  medication_entities = list(dedupe([t for t in doc0.ents if t.label_ == 'CHEMICAL']))
 
557
  with st.expander('Full Discharge Summary'):
558
  #st.write("line 1 \n line 2 \n line 3")
559
  fulldischargesummary = historyAdmission['TEXT'].values[0]
560
+ st.write(fulldischargesummary)
561
+
562
+ if "load_state" not in st.session_state:
563
+ st.session_state.load_state = False
564
+
565
+ if "button_clicked" not in st.session_state:
566
+ st.session_state.button_clicked = False
567
+
568
+ if "admission_button_clicked" not in st.session_state:
569
+ st.session_state.admission_button_clicked = False
570
+
571
+ if "daily_button_clicked" not in st.session_state:
572
+ st.session_state.daily_button_clicked = False
573
+
574
+ if "past_button_clicked" not in st.session_state:
575
+ st.session_state.past_button_clicked = False
576
+
577
+
578
+
579
+ if btnAdmission or st.session_state["button_clicked"]:
580
+ st.session_state["admission_button_clicked"] = True
581
+ st.session_state["daily_button_clicked"] = False
582
+ st.session_state["past_button_clicked"] = False
583
+ Admission()
584
+
585
+ if btnDailyNarrative or st.session_state["daily_button_clicked"]:
586
+ st.session_state["daily_button_clicked"] = True
587
+ st.session_state["admission_button_clicked"] = False
588
+ st.session_state["past_button_clicked"] = False
589
+ DailyNarrative()
590
+
591
+
592
+ if btnPastHistory or st.session_state["past_button_clicked"]:
593
+ st.session_state["past_button_clicked"] = True
594
+ st.session_state["daily_button_clicked"] = False
595
+ st.session_state["admission_button_clicked"] = False
596
+ PastHistory()