feat: openvoice重构，添加语音克隆功能

app/controllers/chat_controller.py

@@ -4,12 +4,13 @@ from ..dependencies.summarizer import get_summarizer
 from ..dependencies.asr import get_asr
 from ..dependencies.tts import get_tts
 from .controller_enum import *
-from ..models import UserCharacter, Session, Character, User
+from ..models import UserCharacter, Session, Character, User, Audio
 from utils.audio_utils import VAD
 from fastapi import WebSocket, HTTPException, status
 from datetime import datetime
 from utils.xf_asr_utils import generate_xf_asr_url
 from config import get_config
+import numpy as np
 import uuid
 import json
 import asyncio
@@ -100,6 +101,19 @@ def update_session_activity(session_id,db):
     except Exception as e:
         db.roolback()
         raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(e))
+
+#获取target_se
+def get_emb(session_id,db):
+    try:
+        session_record = db.query(Session).filter(Session.id == session_id).first()
+        user_character_record = db.query(UserCharacter).filter(UserCharacter.id == session_record.user_character_id).first()
+        audio_record = db.query(Audio).filter(Audio.user_id == user_character_record.user_id).first()
+        emb_npy = np.frombuffer(audio_record.emb_data,dtype=np.int32)
+        emb_npy_3d = np.reshape(emb_npy,(1,256,1))
+        return emb_npy_3d
+    except Exception as e:
+        logger.error("未找到音频:"+str(e))
+        return np.array([])
 #--------------------------------------------------------
 
 # 创建新聊天
@@ -283,6 +297,7 @@ async def sct_llm_handler(ws,session_id,response_type,llm_info,tts_info,db,redis
             'Authorization': f"Bearer {Config.MINIMAX_LLM.API_KEY}",
             'Content-Type': 'application/json'
         }
+        target_se = get_emb(session_id,db)
     except Exception as e:
         logger.error(f"编辑http请求时发生错误: {str(e)}")
     try:
@@ -299,7 +314,17 @@ async def sct_llm_handler(ws,session_id,response_type,llm_info,tts_info,db,redis
                             response_message = {"type": "text", "code":200, "msg": sentence}
                             await ws.send_text(json.dumps(response_message, ensure_ascii=False)) #返回文本信息
                         elif response_type == RESPONSE_AUDIO:
-                            sr,audio = tts.synthesize(sentence, tts_info["speaker_id"], tts_info["language"], tts_info["noise_scale"], tts_info["noise_scale_w"], tts_info["length_scale"],return_bytes=True)
+                            if target_se.size == 0:
+                                audio,sr = tts._base_tts(text=sentence,
+                                                        noise_scale=tts_info["noise_scale"],
+                                                        noise_scale_w=tts_info["noise_scale_w"],
+                                                        speed=tts_info["length_scale"])
+                            else:
+                                audio,sr = tts.synthesize(text=sentence,
+                                                        noise_scale=tts_info["noise_scale"],
+                                                        noise_scale_w=tts_info["noise_scale_w"],
+                                                        speed=tts_info["length_scale"],
+                                                        target_se=target_se)
                             response_message = {"type": "text", "code":200, "msg": sentence}
                             await ws.send_bytes(audio)  #返回音频数据
                             await ws.send_text(json.dumps(response_message, ensure_ascii=False))    #返回文本信息
@@ -455,6 +480,7 @@ async def scl_llm_handler(ws,session_id,response_type,llm_info,tts_info,db,redis
                 'Authorization': f"Bearer {Config.MINIMAX_LLM.API_KEY}",
                 'Content-Type': 'application/json'
             }
+            target_se = get_emb(session_id,db)
             async with aiohttp.ClientSession() as client:
                 async with client.post(Config.MINIMAX_LLM.URL, headers=headers, data=payload) as response:  #发送大模型请求
                     async for chunk in response.content.iter_any():
@@ -469,7 +495,17 @@ async def scl_llm_handler(ws,session_id,response_type,llm_info,tts_info,db,redis
                                 response_message = {"type": "text", "code":200, "msg": sentence}
                                 await ws.send_text(json.dumps(response_message, ensure_ascii=False))
                             elif response_type == RESPONSE_AUDIO:
-                                sr,audio = tts.synthesize(sentence, tts_info["speaker_id"], tts_info["language"], tts_info["noise_scale"], tts_info["noise_scale_w"], tts_info["length_scale"],return_bytes=True)
+                                if target_se.size == 0:
+                                    audio,sr = tts._base_tts(text=sentence,
+                                                            noise_scale=tts_info["noise_scale"],
+                                                            noise_scale_w=tts_info["noise_scale_w"],
+                                                            speed=tts_info["length_scale"])
+                                else:
+                                    audio,sr = tts.synthesize(text=sentence,
+                                                            noise_scale=tts_info["noise_scale"],
+                                                            noise_scale_w=tts_info["noise_scale_w"],
+                                                            speed=tts_info["length_scale"],
+                                                            target_se=target_se)
                                 response_message = {"type": "text", "code":200, "msg": sentence}
                                 await ws.send_bytes(audio)
                                 await ws.send_text(json.dumps(response_message, ensure_ascii=False))
@@ -629,11 +665,11 @@ async def voice_call_llm_handler(ws,session_id,llm_info,tts_info,db,redis,asr_re
                 "temperature": llm_info["temperature"],
                 "top_p": llm_info["top_p"]
             })
-            
             headers = {
                 'Authorization': f"Bearer {Config.MINIMAX_LLM.API_KEY}",
                 'Content-Type': 'application/json'
             }
+            target_se = get_emb(session_id,db)
             async with aiohttp.ClientSession() as client:
                 async with client.post(Config.MINIMAX_LLM.URL, headers=headers, data=payload) as response:  #发送大模型请求
                     async for chunk in response.content.iter_any():
@@ -643,7 +679,17 @@ async def voice_call_llm_handler(ws,session_id,llm_info,tts_info,db,redis,asr_re
                             llm_response += chunk_data
                         sentences,current_sentence,is_first = split_string_with_punctuation(current_sentence,chunk_data,is_first,is_end)
                         for sentence in sentences:
-                            sr,audio = tts.synthesize(sentence, tts_info["language"], tts_info["speaker_id"], tts_info["noise_scale"], tts_info["noise_scale_w"], tts_info["length_scale"], return_bytes=True)
+                            if target_se.size == 0:
+                                audio,sr = tts._base_tts(text=sentence,
+                                                        noise_scale=tts_info["noise_scale"],
+                                                        noise_scale_w=tts_info["noise_scale_w"],
+                                                        speed=tts_info["length_scale"])
+                            else:
+                                audio,sr = tts.synthesize(text=sentence,
+                                                        noise_scale=tts_info["noise_scale"],
+                                                        noise_scale_w=tts_info["noise_scale_w"],
+                                                        speed=tts_info["length_scale"],
+                                                        target_se=target_se)
                             text_response = {"type": "llm_text", "code": 200, "msg": sentence}
                             await ws.send_bytes(audio) #返回音频二进制流数据
                             await ws.send_text(json.dumps(text_response, ensure_ascii=False)) #返回文本数据
@@ -673,23 +719,6 @@ async def voice_call_llm_handler(ws,session_id,llm_info,tts_info,db,redis,asr_re
             break
     voice_call_end_event.set()
 
-        
-#语音合成及返回函数
-async def voice_call_tts_handler(ws,tts_info,split_result_q,split_finished_event,voice_call_end_event):
-    logger.debug("语音合成及返回函数启动")
-    while not (split_finished_event.is_set() and split_result_q.empty()):
-        try:
-            sentence = await asyncio.wait_for(split_result_q.get(),timeout=3)
-            sr,audio = tts.synthesize(sentence, tts_info["language"], tts_info["speaker_id"], tts_info["noise_scale"], tts_info["noise_scale_w"], tts_info["length_scale"], return_bytes=True)
-            text_response = {"type": "llm_text", "code": 200, "msg": sentence}
-            await ws.send_bytes(audio) #返回音频二进制流数据
-            await ws.send_text(json.dumps(text_response, ensure_ascii=False)) #返回文本数据
-            logger.debug(f"websocket返回:{sentence}")
-        except asyncio.TimeoutError:
-            continue
-    voice_call_end_event.set()
-
-
 async def voice_call_handler(ws, db, redis):
     logger.debug("voice_call websocket 连接建立")
     audio_q = asyncio.Queue()	#音频队列