killua/TakwayPlatform
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: 使全局asr实例有一定并发能力

Browse Source
This commit is contained in:
killua committed 2024-05-13 10:44:58 +08:00
1 parent 11c429befa
commit 4974570f20
2 files changed
+236 -46
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/controllers/chat_controller.py
+31 -13
View File
@@ -223,18 +223,24 @@ async def sct_user_input_handler(ws,user_input_q,llm_input_q,future_session_id,f
logger.error(f"用户输入处理函数发生错误: {str(e)}")
#语音识别
async def sct_asr_handler(user_input_q,llm_input_q,user_input_finish_event):
async def sct_asr_handler(session_id,user_input_q,llm_input_q,user_input_finish_event):
logger.debug("语音识别函数启动")
is_signup = False
try:
current_message = ""
while not (user_input_finish_event.is_set() and user_input_q.empty()):
if not is_signup:
asr.session_signup(session_id)
is_signup = True
audio_data = await user_input_q.get()
asr_result = asr.streaming_recognize(audio_data)
asr_result = asr.streaming_recognize(session_id,audio_data)
current_message += ''.join(asr_result['text'])
asr_result = asr.streaming_recognize(b'',is_end=True)
asr_result = asr.streaming_recognize(session_id,b'',is_end=True)
current_message += ''.join(asr_result['text'])
await llm_input_q.put(current_message)
asr.session_signout(session_id)
except Exception as e:
asr.session_signout(session_id)
logger.error(f"语音识别函数发生错误: {str(e)}")
logger.debug(f"接收到用户消息: {current_message}")
@@ -305,12 +311,11 @@ async def streaming_chat_temporary_handler(ws: WebSocket, db, redis):
future_session_id = asyncio.Future()
future_response_type = asyncio.Future()
asyncio.create_task(sct_user_input_handler(ws,user_input_q,llm_input_q,future_session_id,future_response_type,user_input_finish_event))
asyncio.create_task(sct_asr_handler(user_input_q,llm_input_q,user_input_finish_event))
session_id = await future_session_id #获取session_id
update_session_activity(session_id,db)
response_type = await future_response_type #获取返回类型
asyncio.create_task(sct_asr_handler(session_id,user_input_q,llm_input_q,user_input_finish_event))
tts_info = json.loads(get_session_content(session_id,redis,db)["tts_info"])
llm_info = json.loads(get_session_content(session_id,redis,db)["llm_info"])
@@ -346,6 +351,7 @@ async def scl_user_input_handler(ws,user_input_q,llm_input_q,future_session_id,f
is_future_done = True
if scl_data_json['text']:
await llm_input_q.put(scl_data_json['text'])
continue
if scl_data_json['meta_info']['is_end']:
user_input_frame = {"audio": scl_data_json['audio'], "is_end": True}
await user_input_q.put(user_input_frame)
@@ -362,25 +368,31 @@ async def scl_user_input_handler(ws,user_input_q,llm_input_q,future_session_id,f
break
#语音识别
async def scl_asr_handler(user_input_q,llm_input_q,input_finished_event,asr_finished_event):
async def scl_asr_handler(session_id,user_input_q,llm_input_q,input_finished_event,asr_finished_event):
logger.debug("语音识别函数启动")
is_signup = False
current_message = ""
while not (input_finished_event.is_set() and user_input_q.empty()):
try:
aduio_frame = await asyncio.wait_for(user_input_q.get(),timeout=3)
if not is_signup:
asr.session_signup(session_id)
is_signup = True
if aduio_frame['is_end']:
asr_result = asr.streaming_recognize(aduio_frame['audio'], is_end=True)
asr_result = asr.streaming_recognize(session_id,aduio_frame['audio'], is_end=True)
current_message += ''.join(asr_result['text'])
await llm_input_q.put(current_message)
logger.debug(f"接收到用户消息: {current_message}")
else:
asr_result = asr.streaming_recognize(aduio_frame['audio'])
asr_result = asr.streaming_recognize(session_id,aduio_frame['audio'])
current_message += ''.join(asr_result['text'])
except asyncio.TimeoutError:
continue
except Exception as e:
asr.session_signout(session_id)
logger.error(f"语音识别函数发生错误: {str(e)}")
break
asr.session_signout(session_id)
asr_finished_event.set()
#大模型调用
@@ -455,7 +467,6 @@ async def streaming_chat_lasting_handler(ws,db,redis):
future_response_type = asyncio.Future()
asyncio.create_task(scl_user_input_handler(ws,user_input_q,llm_input_q,future_session_id,future_response_type,input_finished_event))
asyncio.create_task(scl_asr_handler(user_input_q,llm_input_q,input_finished_event,asr_finished_event))
session_id = await future_session_id #获取session_id
update_session_activity(session_id,db)
@@ -463,6 +474,7 @@ async def streaming_chat_lasting_handler(ws,db,redis):
tts_info = json.loads(get_session_content(session_id,redis,db)["tts_info"])
llm_info = json.loads(get_session_content(session_id,redis,db)["llm_info"])
asyncio.create_task(scl_asr_handler(session_id,user_input_q,llm_input_q,input_finished_event,asr_finished_event))
asyncio.create_task(scl_llm_handler(ws,session_id,response_type,llm_info,tts_info,db,redis,llm_input_q,asr_finished_event,chat_finished_event))
while not chat_finished_event.is_set():
@@ -505,23 +517,27 @@ async def voice_call_audio_producer(ws,audio_q,future,input_finished_event):
#音频数据消费函数
async def voice_call_audio_consumer(ws,audio_q,asr_result_q,input_finished_event,asr_finished_event):
async def voice_call_audio_consumer(ws,session_id,audio_q,asr_result_q,input_finished_event,asr_finished_event):
logger.debug("音频数据消费者函数启动")
vad = VAD()
current_message = ""
vad_count = 0
is_signup = False
while not (input_finished_event.is_set() and audio_q.empty()):
try:
if not is_signup:
asr.session_signup(session_id)
is_signup = True
audio_data = await asyncio.wait_for(audio_q.get(),timeout=3)
if vad.is_speech(audio_data):
if vad_count > 0:
vad_count -= 1
asr_result = asr.streaming_recognize(audio_data)
asr_result = asr.streaming_recognize(session_id, audio_data)
current_message += ''.join(asr_result['text'])
else:
vad_count += 1
if vad_count >= 25: #连续25帧没有语音，则认为说完了
asr_result = asr.streaming_recognize(audio_data, is_end=True)
asr_result = asr.streaming_recognize(session_id, audio_data, is_end=True)
if current_message:
logger.debug(f"检测到静默，用户输入为:{current_message}")
await asr_result_q.put(current_message)
@@ -532,8 +548,10 @@ async def voice_call_audio_consumer(ws,audio_q,asr_result_q,input_finished_event
except asyncio.TimeoutError:
continue
except Exception as e:
asr.session_signout(session_id)
logger.error(f"音频数据消费者函数发生错误: {str(e)}")
break
asr.session_signout(session_id)
asr_finished_event.set()
#asr结果消费以及llm返回生产函数
@@ -621,7 +639,6 @@ async def voice_call_handler(ws, db, redis):
future = asyncio.Future() #用于获取传输的session_id
asyncio.create_task(voice_call_audio_producer(ws,audio_q,future,input_finished_event)) #创建音频数据生产者
asyncio.create_task(voice_call_audio_consumer(ws,audio_q,asr_result_q,input_finished_event,asr_finished_event)) #创建音频数据消费者
#获取session内容
session_id = await future #获取session_id
@@ -629,6 +646,7 @@ async def voice_call_handler(ws, db, redis):
tts_info = json.loads(get_session_content(session_id,redis,db)["tts_info"])
llm_info = json.loads(get_session_content(session_id,redis,db)["llm_info"])
asyncio.create_task(voice_call_audio_consumer(ws,session_id,audio_q,asr_result_q,input_finished_event,asr_finished_event)) #创建音频数据消费者
asyncio.create_task(voice_call_llm_handler(ws,session_id,llm_info,tts_info,db,redis,asr_result_q,asr_finished_event,voice_call_end_event)) #创建llm处理者
while not voice_call_end_event.is_set():
await asyncio.sleep(3)