add whisper STT

app.py

@@ -7,6 +7,40 @@ import openai
 from requests.models import ChunkedEncodingError
 from streamlit.components import v1
 from custom import css_code, js_code, set_context_all
+from st_audiorec import st_audiorec
+
+device = "cpu"
+
+# STT
+import whisper
+WHISPER_LANG = "en" # detecting language if None
+warnings.filterwarnings("ignore")
+WHISPER_MODEL = whisper.load_model("base")
+WHISPER_MODEL.to(device)
+
+def transcribe(aud_inp):
+    if aud_inp is None:
+        return ""
+    if isinstance(aud_inp, str):
+        aud = whisper.load_audio(aud_inp)
+    elif isinstance(aud_inp, bytes): # if st_audiorec
+        aud = np.frombuffer(wav_bytes, dtype=np.uint8).flatten().astype(np.float32) / 255.0
+    aud = whisper.pad_or_trim(aud)
+    mel = whisper.log_mel_spectrogram(aud).to(device)
+    _, probs = WHISPER_MODEL.detect_language(mel)
+
+    if device == "cpu":
+        options = whisper.DecodingOptions(fp16 = False, language=WHISPER_LANG)
+    else:
+        options = whisper.DecodingOptions(language=WHISPER_LANG)
+
+    result = whisper.decode(WHISPER_MODEL, mel, options)
+    print("result.text", result.text)
+    result_text = ""
+    if result and result.text:
+        result_text = result.text
+    return result_text
+
 
 st.set_page_config(page_title='ChatGPT Assistant', layout='wide', page_icon='🤖')
 # 自定义元素样式
@@ -153,8 +187,8 @@ with tap_set:
               key='context_level' + current_chat, help="表示每次会话中包含的历史对话次数，预设内容不计算在内。")
 
     st.markdown("模型参数：")
-    st.selectbox("Model", ["gpt-3.5-turbo", "gpt-3.5-turbo-16k", "gpt-4", "gpt-4-32k"], index=0, 
-                 help="[模型选择参考](https://platform.openai.com/docs/models)", 
+    st.selectbox("Model", ["gpt-3.5-turbo", "gpt-3.5-turbo-16k", "gpt-4", "gpt-4-32k"], index=0,
+                 help="[模型选择参考](https://platform.openai.com/docs/models)",
                  on_change=write_data, key='model' + current_chat)
     st.slider("Temperature", 0.0, 2.0, st.session_state["temperature" + current_chat + "default"], 0.1,
               help="""在0和2之间，应该使用什么样的采样温度？较高的值（如0.8）会使输出更随机，而较低的值（如0.2）则会使其更加集中和确定性。
@@ -190,11 +224,16 @@ with tap_input:
                 write_data(new_name)
 
 
-
     with st.form("input_form", clear_on_submit=True):
         user_input = st.text_area("**输入：**", key="user_input_area")
         submitted = st.form_submit_button("确认提交", use_container_width=True, on_click=input_callback)
-    if submitted:
+
+        wav_audio_data = st_audiorec()
+        if wav_audio_data is not None:
+            st.audio(wav_audio_data, format='audio/wav')
+            user_input = transcribe(wav_audio_data)
+
+    if submitted or wav_audio_data is not None:
         st.session_state['user_input_content'] = user_input
 
     if st.session_state['user_input_content'] != '':
@@ -276,4 +315,4 @@ if ("r" in st.session_state) and (current_chat == st.session_state["chat_of_r"])
         st.session_state.pop("r")
 
 # 添加事件监听
-v1.html(js_code, height=0)
+v1.html(js_code, height=0)

requirements.txt

@@ -3,3 +3,5 @@ streamlit==1.19.0
 pandas==1.5.3
 requests==2.28.2
 altair<5
+streamlit-audiorec
+git+https://github.com/openai/whisper.git