app file

app.py

@@ -0,0 +1,572 @@
+# Ref https://github.com/ezzcodeezzlife/dalle2-in-python
+# Ref https://towardsdatascience.com/speech-to-text-with-openais-whisper-53d5cea9005e
+# Ref https://python.plainenglish.io/creating-an-awesome-web-app-with-python-and-streamlit-728fe100cf7
+import logging
+import logging.handlers
+import queue
+import threading
+import time
+import urllib.request
+from collections import deque
+from pathlib import Path
+from typing import List
+# import whisper
+import av
+import numpy as np
+import pydub
+import streamlit as st
+from tqdm import tqdm
+from streamlit_webrtc import WebRtcMode, webrtc_streamer
+
+from dalle2 import Dalle2
+from PIL import Image
+
+HERE = Path(__file__).parent
+
+logger = logging.getLogger(__name__)
+
+# Initialize the OpenAI API with your API key
+import openai
+from openai import OpenAI
+api_key = "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"
+client = OpenAI(api_key=api_key)
+
+prompt = """I am a doctor, I would like you to check my prescription:
+medical history: Hypertension, Type 2 Diabetes, and Asthma.
+symptoms: Persistent cough, fever, and fatigue.
+My prescription: Lisinopril 10mg daily, Metformin 500mg twice daily, and Albuterol as needed for asthma attacks.
+Drug contexts:
+- Lisinopril: Ingredients: ACE inhibitor. Adverse effects: Dizziness, dry cough, elevated blood potassium levels.
+- Metformin: Ingredients: Oral antihyperglycemic agent. Adverse effects: Stomach upset, diarrhea, low blood sugar.
+- Albuterol: Ingredients: Bronchodilator. Adverse effects: Tremors, nervousness, increased heart rate.
+
+Please answer the following questions in concise point form, taking into account the provided drug context:
+- Possible interactions between prescribed drugs?
+- Adverse effect of given drugs that are specifically related to patient’s pre-existing conditions and medical history?
+
+At the end of your answer, evaluate the level of dangerousness of this treatment, based on interactions and adverse effects. Dangerousness is categorized as: LOW, MEDIUM, HIGH
+Your answer should look like this:
+`
+* interactions:
+- <interaction 1>
+- <interaction 2>
+- ...
+
+* adverse effects:
+- <adverse effect 1>
+- <adverse effect 2>
+- ...`
+
+* dangerousness: <LOW / MEDIUM / HIGH>
+
+Note that you don't have to include any interactions or adverse effect, only those that are necessary.
+"""
+def get_drug_info_string(drug_names):
+    # Make the drug_to_info dictionary into a string with each line of the form drug: info
+    drug_info_string = ""
+    for drug in drug_names:
+        info = search_openfda_drug(drug)
+        drug_info_string += drug + ": " + str(trim_openfda_response(search_openfda_drug(drug))) + "\r\n"
+    return drug_info_string
+import requests
+
+def trim_openfda_response(json_response):
+    """Trim the openFDA JSON response to include only specific fields.
+
+    Parameters:
+    - json_response (dict): The raw JSON response from the openFDA API.
+
+    Returns:
+    - dict: A trimmed version of the JSON response.
+    """
+
+    # List of desired fields
+    desired_fields = [
+        "spl_product_data_elements",
+        "boxed_warning",
+        "contraindications",
+        "drug_interactions",
+        "adverse_reactions",
+        "warnings"
+    ]
+
+    trimmed_response = {}
+
+    # Check if results are present in the response
+    if 'results' in json_response:
+        for field in desired_fields:
+            if field in json_response['results'][0]:
+                trimmed_response[field] = json_response['results'][0][field]
+
+    return trimmed_response
+
+def search_openfda_drug(drug_name):
+    """Search for a drug in the openFDA database.
+
+    Parameters:
+    - drug_name (str): The name of the drug to search for.
+
+    Returns:
+    - dict: The JSON response from the openFDA API containing drug information, or None if there's an error.
+    """
+
+    base_url = "https://api.fda.gov/drug/label.json"
+    query = f"?search=openfda.generic_name:{drug_name}&limit=1"
+
+    try:
+        response = requests.get(base_url + query)
+
+        # Check for successful request
+        if response.status_code == 200:
+            return response.json()
+
+    except requests.RequestException:
+        # If any request-related exception occurs, simply return None
+        print(f"Error encountered searching for drug {drug_name} with code {response.status_code}.")
+
+    return None
+
+def ask_gpt(question, model="gpt-3.5-turbo"):
+    """
+    Query the GPT-3.5 Turbo model with a given question.
+
+    Parameters:
+    - question (str): The input question or prompt for the model.
+
+    Returns:
+    - str: The model's response.
+    """
+
+    response = client.chat.completions.create(
+        model=model,
+        # model="gpt-4-vision-preview",
+        messages=[
+            {"role": "system", "content": "You are a knowledgeable medical database designed to provide concise and direct answers to medical questions."},
+            {"role": "user", "content": question}
+        ]
+    )
+
+    # print(response.choices[0])
+    # return response.choices.message['content']
+    return response.choices[0].message.content
+
+def parse_gpt(question):
+    """
+    Query the GPT-3.5 Turbo model with a given question.
+
+    Parameters:
+    - question (str): The input question or prompt for the model.
+
+    Returns:
+    - str: The model's response.
+    """
+
+    # 1 parse text to replace critical information in the prompt
+    # 2 send parsed text in OpenAPI
+
+    response = openai.ChatCompletion.create(
+        model="gpt-3.5-turbo",
+        messages=[
+            {"role": "system", "content": "You are a knowledgeable medical database designed to provide concise and direct answers to medical questions."},
+            {"role": "user", "content": question}
+        ]
+    )
+
+    return response.choices[0].message['content']
+
+# This code is based on https://github.com/streamlit/demo-self-driving/blob/230245391f2dda0cb464008195a470751c01770b/streamlit_app.py#L48  # noqa: E501
+def download_file(url, download_to: Path, expected_size=None):
+    # Don't download the file twice.
+    # (If possible, verify the download using the file length.)
+    if download_to.exists():
+        if expected_size:
+            if download_to.stat().st_size == expected_size:
+                return
+        else:
+            st.info(f"{url} is already downloaded.")
+            if not st.button("Download again?"):
+                return
+
+    download_to.parent.mkdir(parents=True, exist_ok=True)
+
+    # These are handles to two visual elements to animate.
+    weights_warning, progress_bar = None, None
+    try:
+        weights_warning = st.warning("Downloading %s..." % url)
+        progress_bar = st.progress(0)
+        with open(download_to, "wb") as output_file:
+            with urllib.request.urlopen(url) as response:
+                length = int(response.info()["Content-Length"])
+                counter = 0.0
+                MEGABYTES = 2.0 ** 20.0
+                while True:
+                    data = response.read(8192)
+                    if not data:
+                        break
+                    counter += len(data)
+                    output_file.write(data)
+
+                    # We perform animation by overwriting the elements.
+                    weights_warning.warning(
+                        "Downloading %s... (%6.2f/%6.2f MB)"
+                        % (url, counter / MEGABYTES, length / MEGABYTES)
+                    )
+                    progress_bar.progress(min(counter / length, 1.0))
+    # Finally, we remove these visual elements by calling .empty().
+    finally:
+        if weights_warning is not None:
+            weights_warning.empty()
+        if progress_bar is not None:
+            progress_bar.empty()
+
+
+def main():
+    st.header("openFDA Medical Records Evaluation")
+    st.markdown(
+        """
+This demo app is using [DeepSpeech](https://github.com/mozilla/DeepSpeech),
+an open speech-to-text engine.
+
+A pre-trained model released with
+[v0.9.3](https://github.com/mozilla/DeepSpeech/releases/tag/v0.9.3),
+trained on American English is being served.
+"""
+    )
+
+    # https://github.com/mozilla/DeepSpeech/releases/tag/v0.9.3
+    MODEL_URL = "https://github.com/mozilla/DeepSpeech/releases/download/v0.9.3/deepspeech-0.9.3-models.pbmm"  # noqa
+    LANG_MODEL_URL = "https://github.com/mozilla/DeepSpeech/releases/download/v0.9.3/deepspeech-0.9.3-models.scorer"  # noqa
+    MODEL_LOCAL_PATH = HERE / "models/deepspeech-0.9.3-models.pbmm"
+    LANG_MODEL_LOCAL_PATH = HERE / "models/deepspeech-0.9.3-models.scorer"
+
+    #download_file(MODEL_URL, MODEL_LOCAL_PATH, expected_size=188915987)
+    #download_file(LANG_MODEL_URL, LANG_MODEL_LOCAL_PATH, expected_size=953363776)
+
+    lm_alpha = 0.931289039105002
+    lm_beta = 1.1834137581510284
+    beam = 100
+
+    medical_text_page = "Medical text evaluation" # summarize notes and identify risk from notes (useful for change in doctors)
+    voice_to_text_page = "Voice to medical text"  # use voice to text and identify risk, could be use in case
+    image_to_text_page = "Image to medical text"  # use image to text and identify risk, could be use in case
+    all_in_one_page = "All modalities"            # use all modalities
+    sound_only_page = "Sound only (sendonly)"
+    with_video_page = "With video (sendrecv)"
+    text_only_page = "Text only for DALLE2"
+    app_mode = st.selectbox(
+        "Choose the app mode", 
+        # [sound_only_page, with_video_page, text_only_page, medical_text_page]
+        [medical_text_page, voice_to_text_page, image_to_text_page, all_in_one_page]
+    )
+
+
+
+    if app_mode == sound_only_page:
+        app_sst(
+            str(MODEL_LOCAL_PATH), str(LANG_MODEL_LOCAL_PATH), lm_alpha, lm_beta, beam
+        )
+    elif app_mode == with_video_page:
+        app_sst_with_video(
+            str(MODEL_LOCAL_PATH), str(LANG_MODEL_LOCAL_PATH), lm_alpha, lm_beta, beam
+        )
+    elif app_mode == medical_text_page:
+        form = st.form(key='my-form')
+        text = form.text_input('Medical text description')
+        submit = form.form_submit_button('Submit')
+
+        st.write('Press submit to evaluate medical notes')
+
+        if submit:
+            # res = parse_gpt(text + "Organize the answers in 3 parts, first is pre-existing conditions, second is symptoms, third is prescriptions. Sample output for drugs should be the end of the answer as DRUG_NAMES: <drug 1>, <drug 2>, <drug 3>...")
+            parsed_notes = ask_gpt(f"""
+                    Please parse the following medical note in point form, without losing any important information:
+                    `{text}`
+
+                    your answer should look like: 
+                    `**Patient's medical history**:
+                    - <point 1>
+                    - <point 2>
+                    - ...
+
+                    **Patient's symptoms**:
+                    - <point 1>
+                    - <point 2>
+                    - ...
+
+                    **Prescription**:
+                    - ...
+
+                    DRUGS: <drug 1>, <drug 2>, ...
+                    `
+                    Please be reminded to give the generic names for the drugs
+                    """)
+            st.write(parsed_notes)
+            # Extract the drugs portion from the notes
+            drug_line = [line for line in parsed_notes.split("\n") if line.startswith("DRUGS:")][0]
+
+            # Strip the "DRUGS: " prefix and split the drugs by ", "
+            drugs = drug_line.replace("DRUGS: ", "").strip().split(", ")
+            
+            # Go to FDA
+            drug_info_string = get_drug_info_string(drugs)
+            # st.write(drug_info_string)
+
+            #  #
+            risk = ask_gpt(f"""I am a doctor, I would like you to check my prescription:
+                {parsed_notes}
+
+                Drug contexts:
+                {drug_info_string}
+
+                Please answer the following questions in concise point form, taking into account the provided drug context:
+                - Possible interactions between prescribed drugs?
+                - Adverse effect of given drugs, only answer those that are specifically related to patient’s pre-existing conditions and symptoms?
+
+                At the end of your answer, evaluate the level of dangerousness of this treatment, based on interactions and adverse effects that are specific to the patient. Dangerousness is categorized as: LOW, MEDIUM, HIGH
+                Your answer should look like this (you should include the * where specified):
+                `
+                * **INTERACTIONS**:
+                - <interaction 1>
+                - <interaction 2>
+                - ...
+
+                * **ADVERSE EFFECTS**:
+                - <adverse effect 1>
+                - <adverse effect 2>
+                - ...`
+
+                * **DANGEROUSNESS**: <LOW / MEDIUM / HIGH>
+
+                Note that you don't have to include any interactions or adverse effect, only those that are necessary.
+                """, model = 'gpt-3.5-turbo-16k')
+            # st.write(res)
+            st.write(risk)
+
+    elif app_mode == text_only_page:
+        form = st.form(key='my-form')
+        text = form.text_input('Image description')
+        submit = form.form_submit_button('Submit')
+
+        st.write('Press submit to generate image')
+
+        if submit:
+            app_sst_dalle2(text)
+
+        # form = st.form(key='my_form')
+        # text = form.text_input(label='Image Description')
+        # submit_button = form.form_submit_button(label='Submit')
+        # if submit_button:
+        #     app_sst_dalle2(form.text)
+
+        #text = st.text_input('Image description')
+        #if st.form_submit_button('Generate') == True:
+        #    app_sst_dalle2(text)
+
+
+def app_sst_dalle2(text):
+    dalle = Dalle2("sess-TotC46rSs5pbqdXTRy75cr81ynLJALwa2b3rdxeh")
+    #generations = dalle.generate(text)
+    file_paths = dalle.generate_and_download(text)
+    print(file_paths)
+    #generations = dalle.generate_amount(text, 8) # Every generation has batch size 4 -> amount % 4 == 0 works best
+    for file in file_paths:
+        image = Image.open(file)
+        st.image(image, caption=text)
+
+def app_sst(model_path: str, lm_path: str, lm_alpha: float, lm_beta: float, beam: int):
+    webrtc_ctx = webrtc_streamer(
+        key="speech-to-text",
+        mode=WebRtcMode.SENDONLY,
+        audio_receiver_size=1024,
+        rtc_configuration={"iceServers": [{"urls": ["stun:stun.l.google.com:19302"]}]},
+        media_stream_constraints={"video": False, "audio": True},
+    )
+
+    status_indicator = st.empty()
+
+    if not webrtc_ctx.state.playing:
+        return
+
+    status_indicator.write("Loading...")
+    text_output = st.empty()
+    stream = None
+
+    while True:
+        if webrtc_ctx.audio_receiver:
+            if stream is None:
+                from deepspeech import Model
+                # https://github.com/openai/whisper
+                # model = whisper.load_model(“large”)
+
+                model = Model(model_path)
+                model.enableExternalScorer(lm_path)
+                model.setScorerAlphaBeta(lm_alpha, lm_beta)
+                model.setBeamWidth(beam)
+
+                stream = model.createStream()
+
+                status_indicator.write("Model loaded.")
+
+            sound_chunk = pydub.AudioSegment.empty()
+            try:
+                audio_frames = webrtc_ctx.audio_receiver.get_frames(timeout=1)
+            except queue.Empty:
+                time.sleep(0.1)
+                status_indicator.write("No frame arrived.")
+                continue
+
+            status_indicator.write("Running. Say something!")
+
+            for audio_frame in audio_frames:
+                sound = pydub.AudioSegment(
+                    data=audio_frame.to_ndarray().tobytes(),
+                    sample_width=audio_frame.format.bytes,
+                    frame_rate=audio_frame.sample_rate,
+                    channels=len(audio_frame.layout.channels),
+                )
+                sound_chunk += sound
+
+            if len(sound_chunk) > 0:
+                sound_chunk = sound_chunk.set_channels(1).set_frame_rate(
+                    model.sampleRate()
+                )
+                buffer = np.array(sound_chunk.get_array_of_samples())
+                stream.feedAudioContent(buffer)
+                text = stream.intermediateDecode()
+                text_output.markdown(f"**Text:** {text}")
+        else:
+            status_indicator.write("AudioReciver is not set. Abort.")
+            break
+
+
+def app_sst_with_video(
+    model_path: str, lm_path: str, lm_alpha: float, lm_beta: float, beam: int
+):
+    frames_deque_lock = threading.Lock()
+    frames_deque: deque = deque([])
+
+    async def queued_audio_frames_callback(
+        frames: List[av.AudioFrame],
+    ) -> av.AudioFrame:
+        with frames_deque_lock:
+            frames_deque.extend(frames)
+
+        # Return empty frames to be silent.
+        new_frames = []
+        for frame in frames:
+            input_array = frame.to_ndarray()
+            new_frame = av.AudioFrame.from_ndarray(
+                np.zeros(input_array.shape, dtype=input_array.dtype),
+                layout=frame.layout.name,
+            )
+            new_frame.sample_rate = frame.sample_rate
+            new_frames.append(new_frame)
+
+        return new_frames
+
+    webrtc_ctx = webrtc_streamer(
+        key="speech-to-text-w-video",
+        mode=WebRtcMode.SENDRECV,
+        queued_audio_frames_callback=queued_audio_frames_callback,
+        rtc_configuration={"iceServers": [{"urls": ["stun:stun.l.google.com:19302"]}]},
+        media_stream_constraints={"video": True, "audio": True},
+    )
+
+    status_indicator = st.empty()
+
+    if not webrtc_ctx.state.playing:
+        return
+
+    status_indicator.write("Loading...")
+    text_output = st.empty()
+    stream = None
+
+    while True:
+        if webrtc_ctx.state.playing:
+            if stream is None:
+                from deepspeech import Model
+
+                model = Model(model_path)
+                model.enableExternalScorer(lm_path)
+                model.setScorerAlphaBeta(lm_alpha, lm_beta)
+                model.setBeamWidth(beam)
+
+                stream = model.createStream()
+
+                status_indicator.write("Model loaded.")
+
+            sound_chunk = pydub.AudioSegment.empty()
+
+            audio_frames = []
+            with frames_deque_lock:
+                while len(frames_deque) > 0:
+                    frame = frames_deque.popleft()
+                    audio_frames.append(frame)
+
+            if len(audio_frames) == 0:
+                time.sleep(0.1)
+                status_indicator.write("No frame arrived.")
+                continue
+
+            status_indicator.write("Running. Say something!")
+
+            for audio_frame in audio_frames:
+                sound = pydub.AudioSegment(
+                    data=audio_frame.to_ndarray().tobytes(),
+                    sample_width=audio_frame.format.bytes,
+                    frame_rate=audio_frame.sample_rate,
+                    channels=len(audio_frame.layout.channels),
+                )
+                sound_chunk += sound
+
+            if len(sound_chunk) > 0:
+                sound_chunk = sound_chunk.set_channels(1).set_frame_rate(
+                    model.sampleRate()
+                )
+                buffer = np.array(sound_chunk.get_array_of_samples())
+                stream.feedAudioContent(buffer)
+                text = stream.intermediateDecode()
+                text_output.markdown(f"**Text:** {text}")
+        else:
+            status_indicator.write("Stopped.")
+            break
+
+# a raccoon astronaut with the cosmos reflecting on the glass of his helmet dreaming of the stars
+def add_bg_from_url():
+    st.markdown(
+         f"""
+         <style>
+         .stApp {{
+             background-image: url("https://i.redd.it/zung2u9zryb91.png");
+             background-attachment: fixed;
+             background-size: cover
+         }}
+         </style>
+         """,
+         unsafe_allow_html=True
+     )
+
+#add_bg_from_url() 
+
+
+if __name__ == "__main__":
+    import os
+
+    DEBUG = os.environ.get("DEBUG", "false").lower() not in ["false", "no", "0"]
+
+    logging.basicConfig(
+        format="[%(asctime)s] %(levelname)7s from %(name)s in %(pathname)s:%(lineno)d: "
+        "%(message)s",
+        force=True,
+    )
+
+    logger.setLevel(level=logging.DEBUG if DEBUG else logging.INFO)
+
+    st_webrtc_logger = logging.getLogger("streamlit_webrtc")
+    st_webrtc_logger.setLevel(logging.DEBUG)
+
+    fsevents_logger = logging.getLogger("fsevents")
+    fsevents_logger.setLevel(logging.WARNING)
+
+    main()
+