feat: 增加情感检测与标点识别

app/controllers/chat_controller.py

@@ -16,9 +16,9 @@ import aiohttp
 logger = get_logger()
 
 # --------------------初始化本地ASR-----------------------
-from utils.stt.funasr_utils import FunAutoSpeechRecognizer
+from utils.stt.modified_funasr import ModifiedRecognizer
 
-asr = FunAutoSpeechRecognizer()
+asr = ModifiedRecognizer()
 logger.info("本地ASR初始化成功")
 # -------------------------------------------------------
 
@@ -60,6 +60,10 @@ def parseChunkDelta(chunk):
             return "end"
     except KeyError:
         logger.error(f"error chunk: {chunk}")
+        return ""
+    except json.JSONDecodeError:
+        logger.error(f"error chunk: {chunk}")
+        return ""
 
 #断句函数
 def split_string_with_punctuation(current_sentence,text,is_first,is_end):
@@ -224,6 +228,7 @@ async def sct_user_input_handler(ws,user_input_q,llm_input_q,future_session_id,f
 async def sct_asr_handler(session_id,user_input_q,llm_input_q,user_input_finish_event):
     logger.debug("语音识别函数启动")
     is_signup = False
+    audio = ""
     try:
         current_message = ""
         while not (user_input_finish_event.is_set() and user_input_q.empty()):
@@ -231,10 +236,16 @@ async def sct_asr_handler(session_id,user_input_q,llm_input_q,user_input_finish_
                 asr.session_signup(session_id)
                 is_signup = True
             audio_data = await user_input_q.get()
+            audio += audio_data
             asr_result = asr.streaming_recognize(session_id,audio_data)
             current_message += ''.join(asr_result['text'])
         asr_result = asr.streaming_recognize(session_id,b'',is_end=True)
         current_message += ''.join(asr_result['text'])
+        current_message = asr.punctuation_correction(current_message)
+        emotion_dict = asr.emtion_recognition(audio)   #情感辨识
+        if not isinstance(emotion_dict, str):
+            max_index = emotion_dict['scores'].index(max(emotion_dict['scores']))
+        current_message = f"{current_message},当前说话人的情绪：{emotion_dict['labels'][max_index]}"
         await llm_input_q.put(current_message)
         asr.session_signout(session_id)
     except Exception as e:
@@ -371,6 +382,7 @@ async def scl_asr_handler(session_id,user_input_q,llm_input_q,input_finished_eve
     logger.debug("语音识别函数启动")
     is_signup = False
     current_message = ""
+    audio = ""
     while not (input_finished_event.is_set() and user_input_q.empty()):
         try:
             aduio_frame = await asyncio.wait_for(user_input_q.get(),timeout=3)
@@ -380,15 +392,24 @@ async def scl_asr_handler(session_id,user_input_q,llm_input_q,input_finished_eve
             if aduio_frame['is_end']:
                 asr_result = asr.streaming_recognize(session_id,aduio_frame['audio'], is_end=True)
                 current_message += ''.join(asr_result['text'])
+                current_message = asr.punctuation_correction(current_message)
+                audio += aduio_frame['audio']
+                emotion_dict =asr.emtion_recognition(audio)   #情感辨识
+                if not isinstance(emotion_dict, str):
+                    max_index = emotion_dict['scores'].index(max(emotion_dict['scores']))
+                current_message = f"{current_message}当前说话人的情绪：{emotion_dict['labels'][max_index]}"
+                print(current_message)
                 await llm_input_q.put(current_message)
+                current_message = ""
+                audio = ""
                 logger.debug(f"接收到用户消息: {current_message}")
             else:
                 asr_result = asr.streaming_recognize(session_id,aduio_frame['audio'])
+                audio += aduio_frame['audio']
                 current_message += ''.join(asr_result['text'])
         except asyncio.TimeoutError:
             continue
         except Exception as e:
-            asr.session_signout(session_id)
             logger.error(f"语音识别函数发生错误: {str(e)}")
             break
     asr.session_signout(session_id)
@@ -523,6 +544,7 @@ async def voice_call_audio_consumer(ws,session_id,audio_q,asr_result_q,input_fin
     current_message = ""
     vad_count = 0
     is_signup = False
+    audio = ""
     while not (input_finished_event.is_set() and audio_q.empty()):
         try:
             if not is_signup:
@@ -533,14 +555,22 @@ async def voice_call_audio_consumer(ws,session_id,audio_q,asr_result_q,input_fin
                 if vad_count > 0:
                     vad_count -= 1
                 asr_result = asr.streaming_recognize(session_id, audio_data)
+                audio += audio_data
                 current_message += ''.join(asr_result['text'])
             else:
                 vad_count += 1
                 if vad_count >= 25: #连续25帧没有语音，则认为说完了
                     asr_result = asr.streaming_recognize(session_id, audio_data, is_end=True)
                     if current_message:
+                        current_message = asr.punctuation_correction(current_message)
+                        audio += audio_data
+                        emotion_dict =asr.emtion_recognition(audio)   #情感辨识
+                        if not isinstance(emotion_dict, str):
+                            max_index = emotion_dict['scores'].index(max(emotion_dict['scores']))
+                        current_message = f"{current_message}当前说话人的情绪：{emotion_dict['labels'][max_index]}"
                         logger.debug(f"检测到静默，用户输入为:{current_message}")
                         await asr_result_q.put(current_message)
+                        audio = ""
                         text_response = {"type": "user_text", "code": 200, "msg": current_message}
                         await ws.send_text(json.dumps(text_response, ensure_ascii=False)) #返回文本数据
                     current_message = ""