init

app/__init__.py

@@ -0,0 +1 @@
+from . import *

app/abstract.py

@@ -0,0 +1,41 @@
+from abc import ABC, abstractmethod
+
+#------ 抽象 ASR, LLM, TTS 类 ------ #
+class ASR(ABC):
+    @abstractmethod
+    async def stream_recognize(self, chunk):
+        pass
+
+class LLM(ABC):
+    @abstractmethod
+    def chat(self, assistant, prompt, db):
+        pass
+
+class TTS(ABC):
+    @abstractmethod
+    def synthetize(self, assistant, text):
+        pass
+# --------------------------------- #
+
+
+# ----------- 抽象服务类 ----------- #
+class UserAudioService(ABC):
+    @abstractmethod
+    def user_audio_process(self, audio:str, **kwargs) -> str:
+        pass
+    
+class PromptService(ABC):
+    @abstractmethod
+    def prompt_process(self, prompt:str, **kwargs) -> str:
+        pass
+
+class LLMMsgService(ABC):
+    @abstractmethod
+    def llm_msg_process(self, llm_chunk:list, **kwargs) -> list:
+        pass
+    
+class TTSAudioService(ABC):
+    @abstractmethod
+    def tts_audio_process(self, audio:bytes, **kwargs) -> bytes:
+        pass
+# --------------------------------- #

app/concrete.py

@@ -0,0 +1,285 @@
+from utils.xf_asr_utils import xf_asr_websocket_factory, make_first_frame, make_continue_frame, make_last_frame, parse_xfasr_recv
+from .model import Assistant
+from .abstract import *
+from .public import *
+from utils.vits_utils import TextToSpeech
+from config import Config
+import aiohttp
+import asyncio
+import struct
+import base64
+import json
+
+# ----------- 初始化vits ----------- #
+vits = TextToSpeech()
+# ---------------------------------- #
+
+
+
+#------ 具体 ASR, LLM, TTS 类 ------ #
+FIRST_FRAME = 1
+CONTINUE_FRAME =2
+LAST_FRAME =3
+
+class XF_ASR(ASR):
+    def __init__(self):
+        self.websocket = None
+        self.current_message = ""
+        self.audio = ""
+        self.status = FIRST_FRAME
+        self.segment_duration_threshold = 25 #超时时间为25秒
+        self.segment_start_time = None
+    
+    async def stream_recognize(self, chunk):
+        if self.websocket is None: #如果websocket未建立，则建立一个新的连接
+            self.websocket = await xf_asr_websocket_factory()
+        if self.segment_start_time is None: #如果是第一段，则记录开始时间
+            self.segment_start_time = asyncio.get_event_loop().time()
+        if chunk['meta_info']['is_end']:
+            self.status = LAST_FRAME
+        audio_data = chunk['audio']
+        self.audio += audio_data
+        
+        if self.status == FIRST_FRAME: #发送第一帧
+            await self.websocket.send(make_first_frame(audio_data))
+            self.status = CONTINUE_FRAME
+        elif self.status == CONTINUE_FRAME: #发送中间帧
+            await self.websocket.send(make_continue_frame(audio_data))
+        elif self.status == LAST_FRAME: #发送最后一帧
+            await self.websocket.send(make_last_frame(audio_data))
+            self.current_message += parse_xfasr_recv(json.loads(await self.websocket.recv()))
+            if self.current_message == "":
+                raise AsrResultNoneError()
+            await self.websocket.close()
+            print("语音识别结束，用户消息：", self.current_message)
+            return [{"text":self.current_message, "audio":self.audio}]
+        
+        current_time = asyncio.get_event_loop().time()
+        if current_time - self.segment_start_time > self.segment_duration_threshold: #超时，发送最后一帧并重置状态
+            await self.websocket.send(make_last_frame())
+            self.current_message += parse_xfasr_recv(await self.websocket.recv())
+            await self.websocket.close()
+            self.websocket = await xf_asr_websocket_factory()
+            self.status = FIRST_FRAME
+            self.segment_start_time = current_time
+        return []
+        
+class MINIMAX_LLM(LLM):
+    def __init__(self):
+        self.token = 0
+    
+    async def chat(self, assistant, prompt, db):
+        llm_info = json.loads(assistant.llm_info)
+        messages = json.loads(assistant.messages)
+        messages.append({"role":"user","content":prompt})
+        assistant.messages = json.dumps(messages)
+        payload = json.dumps({
+            "model": llm_info['model'],
+            "stream": True,
+            "messages": messages,
+            "max_tokens": llm_info['max_tokens'],
+            "temperature": llm_info['temperature'],
+            "top_p": llm_info['top_p'],
+        })
+        headers = {
+            'Authorization': f"Bearer {Config.MINIMAX_LLM.API_KEY}",
+            'Content-Type': 'application/json'
+        }
+        async with aiohttp.ClientSession() as client:
+            async with client.post(Config.MINIMAX_LLM.URL, headers=headers, data=payload) as response:
+                async for chunk in response.content.iter_any():
+                    try:
+                        chunk_msg = self.__parseChunk(chunk)
+                        msg_frame = {"is_end":False,"code":200,"msg":chunk_msg}
+                        yield msg_frame
+                    except LLMResponseEnd:
+                        msg_frame = {"is_end":True,"msg":""}
+                        if self.token > llm_info['max_tokens'] * 0.8:
+                            msg_frame['msg'] = 'max_token reached'
+                            as_query = db.query(Assistant).filter(Assistant.id == assistant.id).first()
+                            as_query.messages = json.dumps([{'role':'system','content':assistant.system_prompt},{'role':'user','content':prompt}])
+                            db.commit()
+                            assistant.messages = as_query.messages
+                        yield msg_frame
+
+                       
+    def __parseChunk(self, llm_chunk):
+        result = ""
+        data=json.loads(llm_chunk.decode('utf-8')[6:])
+        if data["object"] == "chat.completion": #如果是结束帧
+            self.token = data['usage']['total_tokens']
+            raise LLMResponseEnd()
+        elif data['object'] == 'chat.completion.chunk':
+            for choice in data['choices']:
+                result += choice['delta']['content']
+        else:
+            raise UnkownLLMFrame()
+        return result
+
+class VITS_TTS(TTS):
+    def __init__(self):
+        pass
+    
+    def synthetize(self, assistant, text):
+        tts_info = json.loads(assistant.tts_info)
+        return vits.synthesize(text, tts_info)
+# --------------------------------- #
+
+
+
+# ------ ASR, LLM, TTS 工厂类 ------ #
+class ASRFactory:
+    def create_asr(self,asr_type:str) -> ASR:
+        if asr_type == 'XF':
+            return XF_ASR()
+
+class LLMFactory:
+    def create_llm(self,llm_type:str) -> LLM:
+        if llm_type == 'MINIMAX':
+            return MINIMAX_LLM()
+
+class TTSFactory:
+    def create_tts(self,tts_type:str) -> TTS:
+        if tts_type == 'VITS':
+            return VITS_TTS()
+# --------------------------------- #
+
+
+
+# ----------- 具体服务类 ----------- #
+# 从单说话人的asr_results中取出第一个结果作为prompt
+class BasicPromptService(PromptService):
+    def prompt_process(self, prompt:str, **kwargs):
+        if 'asr_result' in kwargs:
+            return kwargs['asr_result'][0]['text'], kwargs
+        else:
+            raise NoAsrResultsError()
+
+class UserAudioRecordService(UserAudioService):
+    def user_audio_process(self, audio:str, **kwargs):
+        audio_decoded = base64.b64decode(audio)
+        kwargs['recorder'].user_audio += audio_decoded
+        return audio,kwargs
+    
+class TTSAudioRecordService(TTSAudioService):
+    def tts_audio_process(self, audio:bytes, **kwargs):
+        kwargs['recorder'].tts_audio += audio
+        return audio,kwargs
+# --------------------------------- #
+
+
+
+# ------------ 服务链类 ----------- #
+class UserAudioServiceChain:
+    def __init__(self):
+        self.services = []
+    def add_service(self, service:UserAudioService):
+        self.services.append(service)
+    def user_audio_process(self, audio, **kwargs):
+        for service in self.services:
+            audio, kwargs = service.user_audio_process(audio, **kwargs)
+        return audio
+
+class PromptServiceChain:
+    def __init__(self):
+        self.services = []
+    def add_service(self, service:PromptService):
+        self.services.append(service)
+    def prompt_process(self, asr_results):
+        kwargs = {"asr_result":asr_results}
+        prompt = ""
+        for service in self.services:
+            prompt, kwargs = service.prompt_process(prompt, **kwargs)
+        return prompt
+
+class LLMMsgServiceChain:
+    def __init__(self, ):
+        self.services = []
+        self.spliter = SentenceSegmentation()
+    def add_service(self, service:LLMMsgService):
+        self.services.append(service)
+    def llm_msg_process(self, llm_msg):
+        kwargs = {}
+        llm_chunks = self.spliter.split(llm_msg)
+        for service in self.services:
+            llm_chunks , kwargs = service.llm_msg_process(llm_chunks, **kwargs)
+        return llm_chunks
+
+class TTSAudioServiceChain:
+    def __init__(self):
+        self.services = []
+    def add_service(self, service:TTSAudioService):
+        self.services.append(service)
+    def tts_audio_process(self, audio, **kwargs):
+        for service in self.services:
+            audio, kwargs = service.tts_audio_process(audio, **kwargs)
+        return audio
+# --------------------------------- #
+
+class Agent():
+    def __init__(self, asr_type:str, llm_type:str, tts_type:str):
+        asrFactory = ASRFactory()
+        llmFactory = LLMFactory()
+        ttsFactory = TTSFactory()
+        self.recorder = None
+        
+        self.user_audio_service_chain = UserAudioServiceChain() #创建用户音频处理服务链
+        self.user_audio_service_chain.add_service(UserAudioRecordService()) #添加用户音频记录服务
+        
+        self.asr = asrFactory.create_asr(asr_type)
+        
+        self.prompt_service_chain = PromptServiceChain()
+        self.prompt_service_chain.add_service(BasicPromptService())
+        
+        self.llm = llmFactory.create_llm(llm_type)
+        
+        self.llm_chunk_service_chain = LLMMsgServiceChain()
+        
+        self.tts = ttsFactory.create_tts(tts_type)
+        
+        self.tts_audio_service_chain = TTSAudioServiceChain()
+        self.tts_audio_service_chain.add_service(TTSAudioRecordService())
+    
+    def init_recorder(self,user_id):
+        self.recorder = Recorder(user_id)
+    
+    # 对用户输入的音频进行预处理
+    def user_audio_process(self, audio, db):
+        return self.user_audio_service_chain.user_audio_process(audio, recorder=self.recorder)
+    
+    # 进行流式语音识别
+    async def stream_recognize(self, chunk, db):
+        return await self.asr.stream_recognize(chunk)
+    
+    # 进行Prompt加工
+    def prompt_process(self, asr_results, db):
+        return self.prompt_service_chain.prompt_process(asr_results)
+    
+    # 进行大模型调用
+    async def chat(self, assistant ,prompt, db):
+        return self.llm.chat(assistant, prompt, db)
+    
+    # 对大模型的返回进行处理
+    def llm_msg_process(self, llm_chunk, db):
+        return self.llm_chunk_service_chain.llm_msg_process(llm_chunk)
+    
+    # 进行TTS合成
+    def synthetize(self, assistant, text, db):
+        return self.tts.synthetize(assistant, text)
+    
+    # 对合成后的音频进行处理
+    def tts_audio_process(self, audio, db):
+        return self.tts_audio_service_chain.tts_audio_process(audio, recorder=self.recorder)
+    
+    # 编码
+    def encode(self, text, audio):
+        text_resp = {"type":"text","code":200,"msg":text}
+        text_resp_bytes = json.dumps(text_resp,ensure_ascii=False).encode('utf-8')
+        header = struct.pack('!II',len(text_resp_bytes),len(audio))
+        final_resp = header + text_resp_bytes + audio
+        return final_resp
+    
+    # 保存音频
+    def save(self):
+        self.recorder.write()
+    

app/model.py

@@ -0,0 +1,39 @@
+from sqlalchemy import create_engine, Column, Integer, String, CHAR
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker, Session
+from config import Config
+
+engine = create_engine(Config.SQLITE_URL, connect_args={"check_same_thread": False})
+SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+Base = declarative_base()
+
+MESSAGE_LENGTH_LIMIT = 2**31-1
+
+class User(Base):
+    __tablename__ = "user"
+    id = Column(CHAR(36), primary_key=True)
+    name = Column(String(32))
+    email = Column(String(64))
+    password = Column(String(128))
+
+class Assistant(Base):
+    __tablename__ = "assistant"
+    id = Column(CHAR(36), primary_key=True)
+    user_id = Column(CHAR(36))
+    name = Column(String(32))
+    system_prompt = Column(String(512))
+    messages = Column(String(MESSAGE_LENGTH_LIMIT))
+    user_info = Column(String(256))
+    llm_info = Column(String(256))
+    tts_info = Column(String(256))
+    token = Column(Integer)
+
+Base.metadata.create_all(bind=engine)
+
+def get_db():
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()
+    

app/public.py

@@ -0,0 +1,79 @@
+from datetime import datetime
+import wave
+import json
+
+# -------------- 公共类 ------------ #
+class AsrResultNoneError(Exception):
+    def __init__(self, message="Asr Result is None!"):
+        super().__init__(message)
+        self.message = message
+
+class NoAsrResultsError(Exception):
+    def __init__(self, message="No Asr Results!"):
+        super().__init__(message)
+        self.message = message
+
+class LLMResponseEnd(Exception):
+    def __init__(self, message="LLM Response End!"):
+        super().__init__(message)
+        self.message = message
+
+class UnkownLLMFrame(Exception):
+    def __init__(self, message="Unkown LLM Frame!"):
+        super().__init__(message)
+        self.message = message
+        
+class TokenOutofRangeError(Exception):
+    def __init__(self, message="Token Out of Range!"):
+        super().__init__(message)
+        self.message = message
+        
+class SentenceSegmentation():
+    def __init__(self,):
+        self.is_first_sentence = True
+        self.cache = ""
+        
+    def __sentenceSegmentation(self, llm_frame):
+        results = []
+        if llm_frame['is_end']:
+            if self.cache:
+                results.append(self.cache)
+            self.cache = ""
+            return results
+        for char in llm_frame['msg']:
+            self.cache += char
+            if self.is_first_sentence and char in ',.?!，。？！':
+                results.append(self.cache)
+                self.cache = ""
+                self.is_first_sentence = False
+            elif char in '。？！':
+                results.append(self.cache)
+                self.cache = ""
+        return results
+    
+    def split(self, llm_chunk):
+        return self.__sentenceSegmentation(llm_chunk)
+
+class Recorder:
+    def __init__(self, user_id):
+        self.input_wav_path = 'storage/wav/'+ datetime.now().strftime('%Y%m%d%H%M%S') + 'U' + user_id + 'i.wav'
+        self.output_wav_path = 'storage/wav/'+ datetime.now().strftime('%Y%m%d%H%M%S') + 'U' + user_id + 'o.wav'
+        self.out_put_text_path = 'storage/record/'+ datetime.now().strftime('%Y%m%d%H%M%S') + 'U' + user_id + 'o.txt'
+        self.input_sr = 16000
+        self.output_sr = 22050
+        self.user_audio = b''
+        self.tts_audio = b''
+        self.input_text = ""
+        self.output_text = ""
+        
+    def write(self):
+        record = {"input_wav":self.input_wav_path,"input_text":self.input_text,"input_sr":self.input_sr,"output_wav":self.output_wav_path,"output_text":self.output_text,"output_sr":self.output_sr}
+        with wave.open(self.input_wav_path, 'wb') as wav_file:
+            wav_file.setparams((1, 2, self.input_sr, 0, 'NONE', 'not compressed'))
+            wav_file.writeframes(self.user_audio)
+        with wave.open(self.output_wav_path, 'wb') as wav_file:
+            wav_file.setparams((1, 2, self.output_sr, 0, 'NONE', 'not compressed'))
+            wav_file.writeframes(self.tts_audio)
+        with open(self.out_put_text_path, 'w', encoding='utf-8') as file:
+            file.write(json.dumps(record, ensure_ascii=False))
+# ---------------------------------- #

app/schemas.py

@@ -0,0 +1,36 @@
+from pydantic import BaseModel
+
+class create_assistant_request(BaseModel):
+    user_id: str
+    name: str
+    system_prompt : str
+    user_info : str
+    llm_info : str
+    tts_info : str
+
+class update_assistant_request(BaseModel):
+    name: str
+    system_prompt : str
+    messages: str
+    user_info : str
+    llm_info : str
+    tts_info : str
+    
+class create_user_request(BaseModel):
+    name: str
+    email: str
+    password: str
+    
+class update_user_request(BaseModel):
+    name: str
+    email: str
+    password: str
+
+class update_assistant_system_prompt_request(BaseModel):
+    system_prompt:str
+
+class update_assistant_deatil_params_request(BaseModel):
+    model :str
+    temperature :float
+    speaker_id:int
+    length_scale:float