update
This commit is contained in:
IrvingGao committed
commit
8255cc8c68
36 files changed
+4800
No files matched your search
Whitespace-only changes.
@@ -0,0 +1,160 @@
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import datetime
|
||||
import requests
|
||||
|
||||
from takway.common_utils import encode_bytes2str, decode_str2bytes
|
||||
|
||||
'''
|
||||
{
|
||||
"RESPONSE_INFO": {
|
||||
"status": "success/error", # string
|
||||
"message": "xxxxx", # string
|
||||
},
|
||||
"DATA": {
|
||||
"Audio": {
|
||||
"data": "xxxxx", # base64 encoded data
|
||||
"metadata": {
|
||||
"rate": ; # int
|
||||
"channels": ; # int
|
||||
"format": ; # int
|
||||
}
|
||||
},
|
||||
"Text": {
|
||||
"data": "xxxxx", # base64 encoded data
|
||||
"metadata": {
|
||||
"is_end": True/False, # bool
|
||||
}
|
||||
},
|
||||
"Image": {
|
||||
"data": "xxxxx", # base64 encoded data
|
||||
"metadata": {
|
||||
"width": ; # int
|
||||
"height": ; # int
|
||||
"format": ; # string
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
'''
|
||||
|
||||
class Client:
|
||||
def __init__(self, server_url):
|
||||
self.server_url = server_url
|
||||
|
||||
def gen_request_data(self, **kwargs):
|
||||
# print("kwargs:", kwargs)
|
||||
audio_data = kwargs.get("audio_data", None)
|
||||
text_data = kwargs.get("text_data", dict())
|
||||
return json.dumps(
|
||||
{
|
||||
"is_end": audio_data.get("is_end"), # bool
|
||||
"is_bgn": audio_data.get("is_bgn"), # bool
|
||||
"DATA": {
|
||||
"Audio": {
|
||||
"data": encode_bytes2str(audio_data['frames']), # base64 encoded data
|
||||
"metadata": {
|
||||
"frames_size": audio_data.get("frames_size"), # string
|
||||
"chunk_size": audio_data.get("chunk_size"), # int
|
||||
"is_end": audio_data.get("is_end"), # bool
|
||||
}
|
||||
},
|
||||
"Text": {
|
||||
"data": text_data.get("text"), # base64 encoded data
|
||||
"metadata": {
|
||||
"chat_status": text_data.get("chat_status"), # string
|
||||
"chat_history": text_data.get("chat_history"), # list of dict
|
||||
}
|
||||
},
|
||||
},
|
||||
"META_INFO": {
|
||||
# "model_version": kwargs.get("model_version", ""), # string
|
||||
# "model_url": kwargs.get("model_url", ""), # string
|
||||
"character": {
|
||||
"name": kwargs.get("character", "Klee"), # string
|
||||
"speaker_id": kwargs.get("speaker_id", 113), # int
|
||||
"wakeup_words": kwargs.get("wakeup_words", "可莉来啦"), # list of string
|
||||
}
|
||||
}
|
||||
}
|
||||
) + '\n'
|
||||
|
||||
def send_data_to_server(self, **kwargs):
|
||||
return requests.post(self.server_url,
|
||||
data=self.gen_request_data(**kwargs), stream=True)
|
||||
|
||||
|
||||
|
||||
|
||||
# ############################################ #
|
||||
# ############ WebSocket Client ############# #
|
||||
def check_audio_type(data, return_type='base64'):
|
||||
'''
|
||||
Check if the data type is valid.
|
||||
'''
|
||||
assert return_type in ['bytes', 'base64']
|
||||
if return_type == 'base64':
|
||||
if isinstance(data, bytes):
|
||||
return encode_bytes2str(data)
|
||||
elif return_type == 'bytes':
|
||||
if isinstance(data, str):
|
||||
return decode_str2bytes(data)
|
||||
else:
|
||||
raise ValueError('Invalid data type: {}.'.format(type(data)))
|
||||
|
||||
import websocket
|
||||
from websocket import create_connection
|
||||
|
||||
class BaseWebSocketClient:
|
||||
def __init__(self, server_url, session_id):
|
||||
self.server_url = server_url
|
||||
self.session_id = session_id
|
||||
|
||||
def wakeup_client(self):
|
||||
'''
|
||||
Start the client.
|
||||
'''
|
||||
self.websocket = create_connection(self.server_url)
|
||||
|
||||
def send_per_data(self,
|
||||
text: str = '',
|
||||
audio: bytes = b'',
|
||||
stream: bool = True,
|
||||
voice_synthesize: bool = False,
|
||||
is_end: bool = False,
|
||||
encoding: str = 'base64',
|
||||
):
|
||||
'''
|
||||
Send data to server.
|
||||
|
||||
Args:
|
||||
data: bytes, data to be sent to server.
|
||||
'''
|
||||
|
||||
self.websocket.send(json.dumps({
|
||||
"text": text,
|
||||
"audio": check_audio_type(audio, return_type=encoding),
|
||||
"meta_info": {
|
||||
"session_id": self.session_id,
|
||||
"stream": stream,
|
||||
"voice_synthesize": voice_synthesize,
|
||||
"is_end": is_end,
|
||||
"encoding": encoding,
|
||||
}}))
|
||||
|
||||
def receive_per_data(self):
|
||||
try:
|
||||
recv_data = self.websocket.recv()
|
||||
except websocket._exceptions.WebSocketConnectionClosedException:
|
||||
return None, None
|
||||
try:
|
||||
recv_data = json.loads(recv_data)
|
||||
except json.JSONDecodeError as e:
|
||||
# print(f"JSONDecodeError: {e}")
|
||||
# is_end = True
|
||||
pass
|
||||
except Exception as e:
|
||||
# print(f"receive_per_data error: {e}")
|
||||
assert isinstance(recv_data, bytes), ValueError(f"Received data is not bytes, got {type(recv_data)}.")
|
||||
return recv_data, type(recv_data)
|
||||
@@ -0,0 +1,513 @@
|
||||
# basic
|
||||
import io
|
||||
import time
|
||||
import json
|
||||
import random
|
||||
from collections import deque
|
||||
# log
|
||||
import logging
|
||||
import warnings
|
||||
# multiprocessing
|
||||
import queue
|
||||
import threading
|
||||
import multiprocessing
|
||||
# web request
|
||||
import requests
|
||||
import pyaudio
|
||||
# hot words detection
|
||||
import pvporcupine
|
||||
|
||||
from takway.apps.data_struct import QueueIterator
|
||||
from takway.common_utils import *
|
||||
from takway.audio_utils import PicovoiceRecorder
|
||||
from takway.clients.client_utils import CharacterClient
|
||||
from takway.audio_utils import AudioPlayer
|
||||
from takway.emo_utils import EmoVideoPlayer
|
||||
|
||||
|
||||
class LocalClinet:
|
||||
def __init__(self,
|
||||
server_args,
|
||||
recorder_args,
|
||||
video_args,
|
||||
emo_args,
|
||||
log_args):
|
||||
# server_args
|
||||
self.server_args = server_args
|
||||
# recorder_args
|
||||
self.recorder_args = recorder_args
|
||||
# video_args
|
||||
self.video_args = video_args
|
||||
# emo_args
|
||||
self.emo_args = emo_args
|
||||
# log_args
|
||||
self.log_args = log_args
|
||||
|
||||
# TODO: 设计多进程log queue
|
||||
self.logger_init()
|
||||
|
||||
|
||||
def logger_init(self):
|
||||
# log_args
|
||||
log_level = self.log_args['log_level']
|
||||
log_file = self.log_args['log_file']
|
||||
|
||||
if log_level == 'debug':
|
||||
log_level = logging.DEBUG
|
||||
elif log_level == 'info':
|
||||
log_level = logging.INFO
|
||||
|
||||
# logger
|
||||
self.logger = logging.getLogger('mylogger')
|
||||
self.logger.setLevel(log_level)
|
||||
# handler 创建一个handler，用于写入日志文件
|
||||
handler = logging.FileHandler(log_file)
|
||||
handler.setLevel(log_level)
|
||||
# stream handler 创建一个handler，用于输出到控制台
|
||||
console = logging.StreamHandler()
|
||||
console.setLevel(logging.INFO)
|
||||
|
||||
# 定义handler的输出格式（formatter）
|
||||
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
handler.setFormatter(formatter)
|
||||
console.setFormatter(formatter)
|
||||
|
||||
# 添加handler
|
||||
self.logger.addHandler(handler)
|
||||
self.logger.addHandler(console)
|
||||
self.logger.info("Logger started.")
|
||||
|
||||
def process_init(self):
|
||||
# multiprocessing
|
||||
manager = multiprocessing.Manager()
|
||||
self.trigger_queue = manager.Queue()
|
||||
self.client_queue = manager.Queue()
|
||||
self.audio_play_queue = manager.Queue()
|
||||
self.emo_display_queue = manager.Queue()
|
||||
|
||||
self.share_time_dict = manager.dict()
|
||||
|
||||
self.speaking_emo_event = manager.Event()
|
||||
|
||||
processes = [
|
||||
multiprocessing.Process(target=self.audio_process,
|
||||
args=(self.trigger_queue,self.client_queue)),
|
||||
# multiprocessing.Process(target=self.camera_process, args=(self.trigger_queue,self.client_queue)),
|
||||
multiprocessing.Process(target=self.local_client_process,
|
||||
args=(self.client_queue,self.audio_play_queue,self.emo_display_queue, self.share_time_dict)),
|
||||
multiprocessing.Process(target=self.audio_play_process,
|
||||
args=(self.audio_play_queue,self.share_time_dict)),
|
||||
]
|
||||
if self.emo_args.pop('enable'):
|
||||
processes.append(
|
||||
multiprocessing.Process(target=self.emo_display_process, args=(self.emo_display_queue,)),
|
||||
)
|
||||
|
||||
for process in processes:
|
||||
process.start()
|
||||
for process in processes:
|
||||
process.join()
|
||||
|
||||
def audio_process(self,
|
||||
trigger_queue,
|
||||
client_queue):
|
||||
"""audio_process
|
||||
|
||||
Args:
|
||||
trigger_queue: multiprocessing.Queue, trigger queue
|
||||
client_queue: multiprocessing.Queue, client queue
|
||||
"""
|
||||
|
||||
self.frame_chunk_size = self.recorder_args.pop('frame_chunk_size')
|
||||
self.min_stream_record_time = self.recorder_args.pop('min_stream_record_time')
|
||||
voice_trigger = self.recorder_args.pop('voice_trigger')
|
||||
self.RATE = self.recorder_args['RATE']
|
||||
|
||||
recorder = PicovoiceRecorder(**self.recorder_args)
|
||||
|
||||
# shared data struct:
|
||||
self.shared_waiting = False
|
||||
self.shared_lock = threading.Lock()
|
||||
self.shared_data_lock = threading.Lock()
|
||||
|
||||
# create threads
|
||||
threads = [threading.Thread(target=self.hardware_trigger_thread, args=(recorder,))]
|
||||
if voice_trigger:
|
||||
vioce_threads = [
|
||||
threading.Thread(target=self.voice_trigger_thread, args=(recorder,)),
|
||||
]
|
||||
threads.extend(vioce_threads)
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
self.logger.info("Audio Process started.")
|
||||
|
||||
while True:
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
print(f"audio process exit") ; exit()
|
||||
|
||||
|
||||
def hardware_trigger_thread(self, recorder):
|
||||
"""hardware_trigger_thread
|
||||
|
||||
Args:
|
||||
recorder: takway.audio_utils.Recorder, recorder object
|
||||
"""
|
||||
self.logger.info("Hardware trigger thread started.")
|
||||
|
||||
trgrigger_status = False
|
||||
while True:
|
||||
if self.shared_waiting:
|
||||
continue
|
||||
|
||||
# init status buffer
|
||||
is_bgn = True
|
||||
_frames = 0
|
||||
_total_frames = 0
|
||||
frames = []
|
||||
full_frames = []
|
||||
|
||||
print("Waiting for button press...")
|
||||
recorder.wait_for_hardware_pressed()
|
||||
print("Button pressed.")
|
||||
# stop voice trigger thread
|
||||
with self.shared_data_lock:
|
||||
self.shared_waiting = True # shared_waiting 控制所有线程的待机状态，True表示待机，False表示工作
|
||||
|
||||
print("Start recording...")
|
||||
bg_t = time.time()
|
||||
record_chunk_size = recorder.hd_chunk_size
|
||||
while True:
|
||||
|
||||
data = recorder.record_chunk_voice(
|
||||
CHUNK=record_chunk_size,
|
||||
return_type=None,
|
||||
exception_on_overflow=False)
|
||||
|
||||
frames.append(data)
|
||||
full_frames.append(data)
|
||||
_total_frames += 1
|
||||
|
||||
if not recorder.is_hardware_pressed:
|
||||
break
|
||||
|
||||
stream_reset_status = self.stream_record(
|
||||
bytes_frames=recorder.write_wave_bytes(full_frames),
|
||||
frames_size=len(full_frames),
|
||||
record_chunk_size=record_chunk_size,
|
||||
is_bgn=is_bgn,
|
||||
is_end=False)
|
||||
if stream_reset_status:
|
||||
full_frames.clear()
|
||||
is_bgn = False
|
||||
|
||||
self.stream_record(
|
||||
bytes_frames=recorder.write_wave_bytes(full_frames),
|
||||
frames_size=len(full_frames),
|
||||
record_chunk_size=record_chunk_size,
|
||||
is_bgn=is_bgn,
|
||||
is_end=True)
|
||||
|
||||
print(f"Tatal frames: {_total_frames*record_chunk_size}, {_total_frames*record_chunk_size/recorder.RATE} sec.")
|
||||
# recorder.write_wave_io(f"record_{int(bg_t)}.wav", frames); print(f"write record_{int(bg_t)}.wav")
|
||||
|
||||
with self.shared_data_lock:
|
||||
self.shared_waiting = False # 恢复voice trigger线程工作
|
||||
|
||||
def voice_trigger_thread(self, recorder):
|
||||
"""voice_trigger_thread
|
||||
|
||||
Args:
|
||||
recorder: takway.audio_utils.Recorder, recorder object
|
||||
"""
|
||||
self.logger.info("voice record thread started.")
|
||||
|
||||
print("Waiting for wake up...")
|
||||
while True:
|
||||
if self.shared_waiting:
|
||||
continue
|
||||
data = recorder.record_chunk_voice(
|
||||
CHUNK=recorder.porcupine.frame_length,
|
||||
return_type=None,
|
||||
exception_on_overflow=False,
|
||||
queue=None)
|
||||
|
||||
record_chunk_size = recorder.vad_chunk_size
|
||||
|
||||
if not recorder.is_wakeup(data):
|
||||
continue
|
||||
# wake up
|
||||
is_bgn = True
|
||||
_frames = 0
|
||||
_total_frames = 0
|
||||
frames = []
|
||||
full_frames = []
|
||||
# status buffer
|
||||
buffer_size = recorder.vad_buffer_size
|
||||
active_buffer = deque(maxlen=buffer_size)
|
||||
bg_t = time.time()
|
||||
print("Start recording...")
|
||||
while True:
|
||||
data = recorder.record_chunk_voice(
|
||||
CHUNK=record_chunk_size,
|
||||
return_type=None,
|
||||
exception_on_overflow=False)
|
||||
if data is None:
|
||||
continue
|
||||
|
||||
is_speech = recorder.is_speech(data)
|
||||
if is_speech:
|
||||
_frames += 1
|
||||
frames.append(data)
|
||||
print("add vad frame")
|
||||
_total_frames += 1
|
||||
full_frames.append(data)
|
||||
|
||||
# send chunk data to client
|
||||
stream_reset_status = self.stream_record(
|
||||
bytes_frames=recorder.write_wave_bytes(full_frames),
|
||||
frames_size=len(full_frames),
|
||||
record_chunk_size=record_chunk_size,
|
||||
is_bgn=is_bgn,
|
||||
is_end=False)
|
||||
if stream_reset_status:
|
||||
full_frames.clear()
|
||||
is_bgn = False
|
||||
|
||||
if is_speech:
|
||||
if active_buffer.__len__() == buffer_size:
|
||||
active_buffer.popleft()
|
||||
active_buffer.append(True)
|
||||
else:
|
||||
if active_buffer.__len__() == buffer_size:
|
||||
active_buffer.popleft()
|
||||
active_buffer.append(False)
|
||||
if active_buffer.count(False) != active_buffer.maxlen:
|
||||
continue
|
||||
if time.time() - bg_t > recorder.min_act_time:
|
||||
# end recording
|
||||
self.stream_record(
|
||||
bytes_frames=recorder.write_wave_bytes(full_frames),
|
||||
frames_size=len(full_frames),
|
||||
record_chunk_size=record_chunk_size,
|
||||
is_bgn=is_bgn,
|
||||
is_end=True)
|
||||
print(f"Tatal frames: {_total_frames*record_chunk_size}, valid frame: {_frames*record_chunk_size}, valid rate: {_frames/_total_frames*100:.2f}%, {_frames*record_chunk_size/recorder.RATE} sec.")
|
||||
print("End recording.")
|
||||
break
|
||||
|
||||
|
||||
|
||||
def stream_record(self,
|
||||
bytes_frames: bytes,
|
||||
frames_size: int,
|
||||
record_chunk_size: int,
|
||||
is_bgn: bool,
|
||||
is_end: bool):
|
||||
'''
|
||||
Args:
|
||||
bytes_frames: bytes, audio data
|
||||
frames_size: int, audio data size
|
||||
record_chunk_size: int, audio data chunk size
|
||||
is_bgn: bool, is begin of stream
|
||||
is_end: bool, is end of stream
|
||||
|
||||
Returns:
|
||||
bool, if stream reset status
|
||||
'''
|
||||
if len(bytes_frames) == 0:
|
||||
return False
|
||||
if frames_size*record_chunk_size >= self.min_stream_record_time*self.RATE or is_end:
|
||||
if is_bgn and is_end:
|
||||
return False
|
||||
stream_data = dict(
|
||||
frames=bytes_frames,
|
||||
frames_size=frames_size,
|
||||
chunk_size=record_chunk_size,
|
||||
is_bgn=is_bgn,
|
||||
is_end=is_end)
|
||||
self.client_queue.put(('audio', stream_data))
|
||||
if is_end:
|
||||
print("put None to client queue.")
|
||||
self.client_queue.put(None)
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def camera_process(self, logger, trigger_queue, client_queue):
|
||||
from takway.cam_utils import Camera
|
||||
cam = Camera(self.video_args)
|
||||
while True:
|
||||
if trigger_queue.empty():
|
||||
time.sleep(0.5)
|
||||
else:
|
||||
item = trigger_queue.get()
|
||||
if item[0] == 'trgrigger_status' and item[1]:
|
||||
_, frame = cap.read()
|
||||
client_queue.put(('image', frame))
|
||||
|
||||
|
||||
def local_client_process(self,
|
||||
client_queue,
|
||||
audio_play_queue,
|
||||
emo_display_queue,
|
||||
share_time_dict):
|
||||
'''
|
||||
Args:
|
||||
client_queue: multiprocessing.Queue, client queue
|
||||
audio_play_queue: multiprocessing.Queue, audio play queue
|
||||
emo_display_queue: multiprocessing.Queue, emo display queue
|
||||
share_time_dict: multiprocessing.Manager.dict, shared time dict
|
||||
'''
|
||||
character = self.server_args.pop('character')
|
||||
client = CharacterClient(**self.server_args)
|
||||
|
||||
# print(f"-------------------{character}-------------------")
|
||||
# print(f"client.chat_status: {client.chat_status}")
|
||||
if client.chat_status == 'init':
|
||||
client.set_character(character)
|
||||
client.chat_status = 'chating'
|
||||
# print(f"client.chat_history: {client.chat_history}")
|
||||
|
||||
self.logger.info("Local client process started.")
|
||||
|
||||
while True:
|
||||
response = None
|
||||
if self.client_queue.empty():
|
||||
continue
|
||||
try:
|
||||
response = requests.post(client.server_url, stream=True,
|
||||
data=self.generate_stream_queue_data(client, client_queue))
|
||||
print("get response from server.")
|
||||
self.get_stream_response(client, response, audio_play_queue, emo_display_queue)
|
||||
except requests.exceptions.ConnectionError or ConnectionError as e:
|
||||
print(f"Wait for Server connection...")
|
||||
except requests.exceptions.Timeout or Timeout as e:
|
||||
print(f"Timeout: {e}")
|
||||
except requests.exceptions.ChunkedEncodingError:
|
||||
print("ChunkedEncodingError")
|
||||
|
||||
def generate_stream_queue_data(self, client, client_queue, **kwargs):
|
||||
_i = 0
|
||||
for queue_data in QueueIterator(client_queue):
|
||||
if queue_data[0] == 'audio':
|
||||
_i += 1
|
||||
if _i == 1:
|
||||
self.share_time_dict['client_time'] = [time.time()]
|
||||
else:
|
||||
self.share_time_dict['client_time'].append(time.time())
|
||||
audio_data = queue_data[1]
|
||||
print("send audio data to server...")
|
||||
# print(f"local chat history: {client.chat_history}")
|
||||
yield client.gen_request_data(
|
||||
audio_data=audio_data,
|
||||
chat_data=dict(
|
||||
chat_history=client.chat_history,
|
||||
chat_status=client.chat_status),
|
||||
character_data=client.character_info)
|
||||
|
||||
def get_stream_response(self,
|
||||
client,
|
||||
response,
|
||||
audio_play_queue=None,
|
||||
emo_display_queue=None,
|
||||
chunk_size=1024):
|
||||
'''
|
||||
Args:
|
||||
client: takway.client_utils.CharacterClient, client object
|
||||
response: requests.Response, response object
|
||||
audio_play_queue: multiprocessing.Queue, audio play queue
|
||||
emo_display_queue: multiprocessing.Queue, emo display queue
|
||||
chunk_size: int, chunk size
|
||||
'''
|
||||
assert isinstance(response, requests.Response), \
|
||||
f"response is not requests.Response, but {type(response)}"
|
||||
|
||||
temp_data = '' # init temp_data
|
||||
|
||||
if response.status_code == 200:
|
||||
print("get response from server successfully.")
|
||||
else:
|
||||
print(f"response error, status code: {response.status_code}")
|
||||
|
||||
chat_llm_response = ''
|
||||
|
||||
_i = 0
|
||||
# for chunk in response.iter_lines():
|
||||
# if chunk:
|
||||
for chunk in response.iter_content(chunk_size=chunk_size):
|
||||
temp_data += chunk.decode('utf-8')
|
||||
if temp_data.endswith('\n'):
|
||||
_i += 1
|
||||
try:
|
||||
temp_json = json.loads(temp_data.rstrip('\n'))
|
||||
# phase 1: get audio data
|
||||
audio_play_queue.put(('server_data', temp_json['audio_output']['tts_stream_data']))
|
||||
# phase 2: get chat data
|
||||
chat_llm_response += temp_json['chat_output']['llm_stream_data']
|
||||
|
||||
if temp_json['is_end']:
|
||||
client.update_chat_history(question=temp_json['chat_output']['question'],
|
||||
response=chat_llm_response, asw_prompt_id=1)
|
||||
# print(f"chat_history: {client.chat_history}")
|
||||
if _i == 1:
|
||||
emo_display_queue.put(('emo_data', '高兴'))
|
||||
except json.JSONDecodeError:
|
||||
print(f"json decode error: {temp_data}")
|
||||
temp_data = ''
|
||||
# print("get response.")
|
||||
print("End get response.")
|
||||
|
||||
def audio_play_process(self,
|
||||
audio_play_queue,
|
||||
share_time_dict):
|
||||
'''
|
||||
Args:
|
||||
audio_play_queue: multiprocessing.Queue, audio play queue
|
||||
share_time_dict: multiprocessing.Manager.dict, shared time dict
|
||||
'''
|
||||
audio_player = AudioPlayer()
|
||||
self.logger.info("Audio play process started.")
|
||||
while True:
|
||||
self.speaking_emo_event.clear()
|
||||
item = audio_play_queue.get()
|
||||
self.speaking_emo_event.set() # stop emo random display
|
||||
if item[0] == 'server_data':
|
||||
# 播放音频
|
||||
print("Playing audio...")
|
||||
tts_audio = item[1]
|
||||
print(f"wait time: {(time.time() - self.share_time_dict['client_time'][0])*1000:.2f} ms")
|
||||
try:
|
||||
audio_player.play(tts_audio)
|
||||
except TypeError as e:
|
||||
# print(f"audio play error: {e}")
|
||||
# print(f"tts_audio: {tts_audio}")
|
||||
# print(f"type tts_audio: {type(tts_audio)}")
|
||||
# tts_audio: <class 'NoneType'>
|
||||
continue
|
||||
|
||||
|
||||
def emo_display_process(self, emo_display_queue):
|
||||
'''
|
||||
Args:
|
||||
emo_display_queue: multiprocessing.Queue, emo display queue
|
||||
'''
|
||||
emo_player = EmoVideoPlayer(**self.emo_args)
|
||||
self.logger.info("Emo display process started.")
|
||||
while True:
|
||||
if emo_display_queue.empty():
|
||||
time.sleep(0.1)
|
||||
if self.speaking_emo_event.is_set():
|
||||
continue
|
||||
emo_player.random_wink()
|
||||
else:
|
||||
item = emo_display_queue.get()
|
||||
print(f"Emo display process Get item: {item[0]}")
|
||||
if item[0] == 'emo_data':
|
||||
server_data = item[1]
|
||||
print("Displaying emo...")
|
||||
emo_player.display_emo(emo_name='兴奋', stage='start')
|
||||
emo_player.display_emo(emo_name='兴奋', stage='loop')
|
||||
emo_player.display_emo(emo_name='兴奋', stage='end')
|
||||
print("Display done.")
|
||||
time.sleep(15)
|
||||
@@ -0,0 +1,330 @@
|
||||
# basic
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import json
|
||||
import random
|
||||
from collections import deque
|
||||
from datetime import datetime
|
||||
# log
|
||||
import logging
|
||||
import warnings
|
||||
# multiprocessing
|
||||
import queue
|
||||
import threading
|
||||
import multiprocessing
|
||||
# web request
|
||||
import requests
|
||||
import pyaudio
|
||||
# hot words detection
|
||||
import pvporcupine
|
||||
|
||||
from takway.apps.data_struct import QueueIterator
|
||||
from takway.common_utils import *
|
||||
from takway.audio_utils import PicovoiceRecorder, HDRecorder
|
||||
from takway.clients.client_utils import BaseWebSocketClient
|
||||
from takway.audio_utils import AudioPlayer
|
||||
|
||||
|
||||
class WebSocketClinet:
|
||||
def __init__(self,
|
||||
board,
|
||||
server_args,
|
||||
recorder_args,
|
||||
player_args,
|
||||
log_args,
|
||||
excute_args=None,
|
||||
):
|
||||
self.board = board
|
||||
# server_args
|
||||
self.server_args = server_args
|
||||
# recorder_args
|
||||
self.recorder_args = recorder_args
|
||||
# player_args
|
||||
self.player_args = player_args
|
||||
# excute_args
|
||||
self.excute_args = excute_args
|
||||
# log_args
|
||||
self.log_args = log_args
|
||||
|
||||
|
||||
def process_init(self):
|
||||
# multiprocessing
|
||||
manager = multiprocessing.Manager()
|
||||
self.trigger_queue = manager.Queue()
|
||||
self.client_queue = manager.Queue()
|
||||
self.audio_play_queue = manager.Queue()
|
||||
self.excute_queue = manager.Queue()
|
||||
|
||||
# 多进程标志为
|
||||
self.mircophone_active_set = manager.Event()
|
||||
self.speaker_active_set = manager.Event()
|
||||
|
||||
processes = [
|
||||
multiprocessing.Process(target=self.audio_process),
|
||||
multiprocessing.Process(target=self.web_socket_client_process),
|
||||
multiprocessing.Process(target=self.audio_play_process),
|
||||
]
|
||||
if self.excute_args.get('enable', False):
|
||||
processes.append(
|
||||
multiprocessing.Process(target=self.excute_process),
|
||||
)
|
||||
|
||||
for process in processes:
|
||||
time.sleep(0.5)
|
||||
process.start()
|
||||
for process in processes:
|
||||
process.join()
|
||||
|
||||
def audio_process(self):
|
||||
"""audio_process
|
||||
|
||||
Args:
|
||||
trigger_queue: multiprocessing.Queue, trigger queue
|
||||
client_queue: multiprocessing.Queue, client queue
|
||||
"""
|
||||
min_stream_record_time = self.recorder_args.pop('min_stream_record_time')
|
||||
voice_trigger = self.recorder_args.pop('voice_trigger')
|
||||
# TODO:
|
||||
press_type = self.recorder_args.pop('press_type')
|
||||
max_slience_time = self.recorder_args.pop('max_slience_time')
|
||||
if voice_trigger:
|
||||
recorder = PicovoiceRecorder(**self.recorder_args)
|
||||
else:
|
||||
voice_keys = ['access_key', 'keywords', 'keyword_paths', 'model_path','sensitivities', 'library_path']
|
||||
for key in voice_keys:
|
||||
self.recorder_args.pop(key)
|
||||
recorder = HDRecorder(**self.recorder_args)
|
||||
recorder.min_stream_record_time = min_stream_record_time
|
||||
# TODO:
|
||||
recorder.press_type = press_type
|
||||
recorder.max_slience_time = max_slience_time
|
||||
|
||||
print("Audio Process started.")
|
||||
|
||||
print("Waiting for wake up...")
|
||||
# recorder.hardware.set_led_on("green")
|
||||
while True:
|
||||
if self.shared_waiting:
|
||||
continue
|
||||
data = recorder.record_chunk_voice(
|
||||
CHUNK=recorder.porcupine.frame_length,
|
||||
return_type=None,
|
||||
exception_on_overflow=False,
|
||||
queue=None)
|
||||
|
||||
record_chunk_size = recorder.vad_chunk_size
|
||||
|
||||
# 开关按键被按下或被关键词唤醒
|
||||
if recorder.hardware.power_status or recorder.is_wakeup(data):
|
||||
# recorder.hardware.set_led_on("blue")
|
||||
pass
|
||||
else:
|
||||
continue
|
||||
# wake up
|
||||
is_bgn = True
|
||||
is_end = False
|
||||
frames = []
|
||||
# status buffer
|
||||
slience_bgn_t = time.time()
|
||||
slience_time = 0
|
||||
print("Start recording...")
|
||||
# 准备对话状态
|
||||
while True:
|
||||
# 语音活动检测
|
||||
data = recorder.record_chunk_voice(
|
||||
CHUNK=record_chunk_size,
|
||||
return_type=None,
|
||||
exception_on_overflow=False)
|
||||
is_speech = recorder.is_speech(data)
|
||||
|
||||
# 判断状态
|
||||
if is_speech:
|
||||
print("valid voice")
|
||||
slience_bgn_t = time.time()
|
||||
frames.append(data)
|
||||
else:
|
||||
slience_time = time.time() - slience_bgn_t
|
||||
|
||||
# 长时沉默关闭唤醒状态：如果唤醒后超过一定时间没有说话/关闭按键被按下，则认为是结束
|
||||
if slience_time > recorder.max_slience_time or not recorder.hardware.power_status:
|
||||
break
|
||||
|
||||
# 短时沉默结束单次对话：沉默时间超过一定时间段（0.5s左右），则发送数据
|
||||
if slience_time > recorder.min_act_time:
|
||||
is_end = True
|
||||
is_bgn = False
|
||||
|
||||
if not is_speech:
|
||||
continue
|
||||
|
||||
# 流式发送数据
|
||||
stream_reset_status = self.stream_record_process(
|
||||
bytes_frames=recorder.write_wave_bytes(frames),
|
||||
frames_size=len(frames),
|
||||
record_chunk_size=record_chunk_size,
|
||||
sample_rate=recorder.RATE,
|
||||
min_stream_record_time=recorder.min_stream_record_time,
|
||||
is_bgn=is_bgn,
|
||||
is_end=is_end)
|
||||
|
||||
if stream_reset_status:
|
||||
frames.clear()
|
||||
is_bgn = False
|
||||
|
||||
# print(f"Tatal frames: {_total_frames*record_chunk_size}, valid frame: {_frames*record_chunk_size}, valid RATE: {_frames/_total_frames*100:.2f}%, {_frames*record_chunk_size/recorder.RATE} sec.")
|
||||
# print("End recording.")
|
||||
# recorder.hardware.set_led_off("blue")
|
||||
|
||||
|
||||
def stream_record_process(self,
|
||||
bytes_frames: bytes,
|
||||
frames_size: int,
|
||||
record_chunk_size: int,
|
||||
sample_rate: int,
|
||||
min_stream_record_time: int,
|
||||
is_bgn: bool,
|
||||
is_end: bool):
|
||||
'''
|
||||
Args:
|
||||
bytes_frames: bytes, audio data
|
||||
frames_size: int, audio data size
|
||||
record_chunk_size: int, audio data chunk size
|
||||
is_bgn: bool, is begin of stream
|
||||
is_end: bool, is end of stream
|
||||
|
||||
Returns:
|
||||
bool, if stream reset status
|
||||
'''
|
||||
if len(bytes_frames) == 0:
|
||||
return False
|
||||
if frames_size*record_chunk_size >= min_stream_record_time*sample_rate or is_end:
|
||||
if is_bgn and is_end:
|
||||
return False
|
||||
stream_data = dict(
|
||||
frames=bytes_frames,
|
||||
frames_size=frames_size,
|
||||
chunk_size=record_chunk_size,
|
||||
is_bgn=is_bgn,
|
||||
is_end=is_end)
|
||||
self.client_queue.put(('audio', stream_data))
|
||||
if is_end:
|
||||
# print("put None to client queue.")
|
||||
self.client_queue.put(None)
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def web_socket_client_process(self):
|
||||
|
||||
client = BaseWebSocketClient(self.server_args['server_url'], self.server_args['session_id'])
|
||||
print("Web socket client process started.")
|
||||
# print("Web socket client process started.")
|
||||
|
||||
while True:
|
||||
if self.client_queue.empty():
|
||||
continue
|
||||
|
||||
# print(f"init skt time: {datetime.now()}")
|
||||
# 唤醒
|
||||
client.wakeup_client()
|
||||
|
||||
# 发送数据
|
||||
for queue_data in QueueIterator(self.client_queue):
|
||||
if queue_data[0] == 'audio':
|
||||
audio_dict = queue_data[1]
|
||||
|
||||
client.send_per_data(
|
||||
audio=audio_dict['frames'],
|
||||
stream=True,
|
||||
voice_synthesize=True,
|
||||
is_end=audio_dict['is_end'],
|
||||
encoding='base64',
|
||||
)
|
||||
# print(f"send skt time: {datetime.now()}")
|
||||
# print(f"fnsh skt time: {datetime.now()}")
|
||||
|
||||
# 接收数据
|
||||
while True:
|
||||
response, data_type = client.receive_per_data()
|
||||
if data_type == dict:
|
||||
print(response) # 打印接收到的消息
|
||||
'''
|
||||
try:
|
||||
response = json.loads(response['msg'])
|
||||
if 'content' in response.keys():
|
||||
self.excute_queue.put((response['instruct'], response['content']))
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"json decode error: {e}")
|
||||
continue
|
||||
# print(f"recv json time: {datetime.now()}")
|
||||
'''
|
||||
elif data_type == bytes:
|
||||
# print(f"recv bytes time: {datetime.now()}")
|
||||
self.audio_play_queue.put(('audio_bytes', response))
|
||||
elif data_type == None:
|
||||
break # 如果没有接收到消息，则退出循环
|
||||
# print("接收完毕:", datetime.now())
|
||||
|
||||
|
||||
def audio_play_process(self):
|
||||
'''
|
||||
Args:
|
||||
audio_play_queue: multiprocessing.Queue, audio play queue
|
||||
share_time_dict: multiprocessing.Manager.dict, shared time dict
|
||||
'''
|
||||
audio_player = AudioPlayer(**self.player_args)
|
||||
print("Audio play process started.")
|
||||
while True:
|
||||
item = self.audio_play_queue.get()
|
||||
# 播放音频
|
||||
print("Playing audio...")
|
||||
tts_audio = item[1]
|
||||
print(f"tts_audio len: {len(tts_audio)}")
|
||||
print(f"play audio time: {datetime.now()}")
|
||||
try:
|
||||
# 播放
|
||||
self.speaker_active_set.set()
|
||||
tts_audio = audio_player.check_audio_type(tts_audio, return_type=None)
|
||||
for i in range(0, len(tts_audio), audio_player.CHUNK):
|
||||
audio_player.stream.write(tts_audio[i:i+audio_player.CHUNK])
|
||||
# print("Playing {} data...{}/{}".format(item[0], i, len(tts_audio)))
|
||||
if self.mircophone_active_set.is_set():
|
||||
print("mirophone is active.")
|
||||
self.mircophone_active_set.wait()
|
||||
break
|
||||
audio_player.stream.write(tts_audio[i+audio_player.CHUNK:])
|
||||
print(f"audio data played.")
|
||||
except TypeError as e:
|
||||
print(f"audio play error: {e}")
|
||||
continue
|
||||
|
||||
# audio_player.stream.write(audio_data[i+audio_player.CHUNK:])
|
||||
# print(f"{item[0]} data played.")
|
||||
|
||||
|
||||
|
||||
|
||||
def excute_process(self):
|
||||
'''
|
||||
Args:
|
||||
excute_queue: multiprocessing.Queue, excute display queue
|
||||
'''
|
||||
print("Excute process started.")
|
||||
|
||||
while True:
|
||||
if self.excute_queue.empty():
|
||||
continue
|
||||
|
||||
if self.speaker_active_set.is_set():
|
||||
instruct, content = self.excute_queue.get()
|
||||
|
||||
print(f"Got speaker info: {instruct, content}")
|
||||
|
||||
print(f"Playing {instruct} {content}...")
|
||||
print(f"play {instruct} time: {datetime.now()}")
|
||||
self.audio_play_queue.put((instruct, content))
|
||||
|
||||
self.speaker_active_set.clear()
|
||||
|
||||
@@ -0,0 +1,464 @@
|
||||
# basic
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import json
|
||||
import random
|
||||
from collections import deque
|
||||
from datetime import datetime
|
||||
# log
|
||||
import logging
|
||||
import warnings
|
||||
# multiprocessing
|
||||
import queue
|
||||
import threading
|
||||
import multiprocessing
|
||||
# web request
|
||||
import requests
|
||||
import pyaudio
|
||||
# hot words detection
|
||||
import pvporcupine
|
||||
|
||||
from takway.apps.data_struct import QueueIterator
|
||||
from takway.common_utils import *
|
||||
from takway.audio_utils import PicovoiceRecorder, HDRecorder
|
||||
from takway.clients.client_utils import BaseWebSocketClient
|
||||
from takway.audio_utils import AudioPlayer
|
||||
|
||||
|
||||
class WebSocketClinet:
|
||||
def __init__(self,
|
||||
board,
|
||||
server_args,
|
||||
recorder_args,
|
||||
player_args,
|
||||
log_args,
|
||||
excute_args=None,
|
||||
):
|
||||
self.board = board
|
||||
# server_args
|
||||
self.server_args = server_args
|
||||
# recorder_args
|
||||
self.recorder_args = recorder_args
|
||||
# player_args
|
||||
self.player_args = player_args
|
||||
# excute_args
|
||||
self.excute_args = excute_args
|
||||
# log_args
|
||||
self.log_args = log_args
|
||||
|
||||
|
||||
def process_init(self):
|
||||
# multiprocessing
|
||||
manager = multiprocessing.Manager()
|
||||
self.trigger_queue = manager.Queue()
|
||||
self.client_queue = manager.Queue()
|
||||
self.audio_play_queue = manager.Queue()
|
||||
self.excute_queue = manager.Queue()
|
||||
|
||||
# 多进程标志为
|
||||
self.mircophone_active_set = manager.Event()
|
||||
self.speaker_active_set = manager.Event()
|
||||
|
||||
processes = [
|
||||
multiprocessing.Process(target=self.audio_process),
|
||||
multiprocessing.Process(target=self.web_socket_client_process),
|
||||
multiprocessing.Process(target=self.audio_play_process),
|
||||
]
|
||||
if self.excute_args.get('enable', False):
|
||||
processes.append(
|
||||
multiprocessing.Process(target=self.excute_process),
|
||||
)
|
||||
|
||||
for process in processes:
|
||||
time.sleep(0.5)
|
||||
process.start()
|
||||
for process in processes:
|
||||
process.join()
|
||||
|
||||
def audio_process(self):
|
||||
"""audio_process
|
||||
|
||||
Args:
|
||||
trigger_queue: multiprocessing.Queue, trigger queue
|
||||
client_queue: multiprocessing.Queue, client queue
|
||||
"""
|
||||
min_stream_record_time = self.recorder_args.pop('min_stream_record_time')
|
||||
voice_trigger = self.recorder_args.pop('voice_trigger')
|
||||
press_type = self.recorder_args.pop('press_type')
|
||||
if voice_trigger:
|
||||
recorder = PicovoiceRecorder(**self.recorder_args)
|
||||
else:
|
||||
voice_keys = ['access_key', 'keywords', 'keyword_paths', 'model_path','sensitivities', 'library_path']
|
||||
for key in voice_keys:
|
||||
self.recorder_args.pop(key)
|
||||
recorder = HDRecorder(**self.recorder_args)
|
||||
recorder.min_stream_record_time = min_stream_record_time
|
||||
recorder.press_type = press_type
|
||||
|
||||
# shared data struct:
|
||||
self.shared_waiting = False
|
||||
self.shared_lock = threading.Lock()
|
||||
self.shared_data_lock = threading.Lock()
|
||||
|
||||
# create threads
|
||||
threads = [threading.Thread(target=self.hardware_trigger_thread, args=(recorder,))]
|
||||
if voice_trigger:
|
||||
vioce_threads = [
|
||||
threading.Thread(target=self.voice_trigger_thread, args=(recorder,)),
|
||||
]
|
||||
threads.extend(vioce_threads)
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
print("Audio Process started.")
|
||||
|
||||
while True:
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
print(f"audio process exit") ; exit()
|
||||
|
||||
|
||||
def hardware_trigger_thread(self, recorder):
|
||||
"""hardware_trigger_thread
|
||||
|
||||
Args:
|
||||
recorder: takway.audio_utils.Recorder, recorder object
|
||||
"""
|
||||
print("Hardware trigger thread started.")
|
||||
|
||||
trgrigger_status = False
|
||||
record_chunk_size = recorder.hd_chunk_size
|
||||
|
||||
while True:
|
||||
if self.shared_waiting:
|
||||
continue
|
||||
|
||||
# init status buffer
|
||||
is_bgn = True
|
||||
frames = []
|
||||
_total_frames = 0
|
||||
|
||||
self.mircophone_active_set.clear()
|
||||
print("Waiting for button press...")
|
||||
recorder.wait_for_hardware_pressed()
|
||||
print("Button pressed.")
|
||||
self.mircophone_active_set.set()
|
||||
# stop voice trigger thread
|
||||
with self.shared_data_lock:
|
||||
self.shared_waiting = True # shared_waiting 控制所有线程的待机状态，True表示待机，False表示工作
|
||||
|
||||
print("Start recording...")
|
||||
bg_t = time.time()
|
||||
while True:
|
||||
|
||||
data = recorder.record_chunk_voice(
|
||||
CHUNK=record_chunk_size,
|
||||
return_type=None,
|
||||
exception_on_overflow=False)
|
||||
|
||||
frames.append(data)
|
||||
_total_frames += 1
|
||||
|
||||
if not recorder.is_hardware_pressed:
|
||||
# print("Button released.")
|
||||
print(f"button rlse time: {datetime.now()}")
|
||||
break
|
||||
|
||||
stream_reset_status = self.stream_record_process(
|
||||
bytes_frames=recorder.write_wave_bytes(frames),
|
||||
frames_size=len(frames),
|
||||
record_chunk_size=record_chunk_size,
|
||||
sample_rate=recorder.RATE,
|
||||
min_stream_record_time=recorder.min_stream_record_time,
|
||||
is_bgn=is_bgn,
|
||||
is_end=False)
|
||||
if stream_reset_status:
|
||||
frames.clear()
|
||||
is_bgn = False
|
||||
|
||||
self.stream_record_process(
|
||||
bytes_frames=recorder.write_wave_bytes(frames),
|
||||
frames_size=len(frames),
|
||||
record_chunk_size=record_chunk_size,
|
||||
sample_rate=recorder.RATE,
|
||||
min_stream_record_time=recorder.min_stream_record_time,
|
||||
is_bgn=is_bgn,
|
||||
is_end=True)
|
||||
|
||||
# print(f"Tatal frames: {_total_frames*record_chunk_size}, {_total_frames*record_chunk_size/recorder.RATE} sec.")
|
||||
# print(f"rcrd time: {datetime.now()}")
|
||||
|
||||
with self.shared_data_lock:
|
||||
self.shared_waiting = False # 恢复voice trigger线程工作
|
||||
|
||||
def voice_trigger_thread(self, recorder):
|
||||
"""voice_trigger_thread
|
||||
|
||||
Args:
|
||||
recorder: takway.audio_utils.Recorder, recorder object
|
||||
"""
|
||||
print("voice record thread started.")
|
||||
|
||||
print("Waiting for wake up...")
|
||||
while True:
|
||||
if self.shared_waiting:
|
||||
continue
|
||||
data = recorder.record_chunk_voice(
|
||||
CHUNK=recorder.porcupine.frame_length,
|
||||
return_type=None,
|
||||
exception_on_overflow=False,
|
||||
queue=None)
|
||||
|
||||
record_chunk_size = recorder.vad_chunk_size
|
||||
|
||||
self.mircophone_active_set.clear()
|
||||
if not recorder.is_wakeup(data):
|
||||
continue
|
||||
|
||||
if self.board == 'orangepi':
|
||||
recorder.hardware.set_led2_on()
|
||||
self.mircophone_active_set.set()
|
||||
# wake up
|
||||
is_bgn = True
|
||||
_frames = 0
|
||||
_total_frames = 0
|
||||
frames = []
|
||||
full_frames = []
|
||||
# status buffer
|
||||
buffer_size = recorder.vad_buffer_size
|
||||
active_buffer = deque(maxlen=buffer_size)
|
||||
bg_t = time.time()
|
||||
print("Start recording...")
|
||||
while True:
|
||||
data = recorder.record_chunk_voice(
|
||||
CHUNK=record_chunk_size,
|
||||
return_type=None,
|
||||
exception_on_overflow=False)
|
||||
if data is None:
|
||||
continue
|
||||
|
||||
is_speech = recorder.is_speech(data)
|
||||
if is_speech:
|
||||
_frames += 1
|
||||
frames.append(data)
|
||||
# print("add vad frame")
|
||||
_total_frames += 1
|
||||
full_frames.append(data)
|
||||
|
||||
# send chunk data to client
|
||||
stream_reset_status = self.stream_record_process(
|
||||
bytes_frames=recorder.write_wave_bytes(full_frames),
|
||||
frames_size=len(full_frames),
|
||||
record_chunk_size=record_chunk_size,
|
||||
sample_rate=recorder.RATE,
|
||||
min_stream_record_time=recorder.min_stream_record_time,
|
||||
is_bgn=is_bgn,
|
||||
is_end=False)
|
||||
if stream_reset_status:
|
||||
full_frames.clear()
|
||||
is_bgn = False
|
||||
|
||||
if is_speech:
|
||||
if active_buffer.__len__() == buffer_size:
|
||||
active_buffer.popleft()
|
||||
active_buffer.append(True)
|
||||
else:
|
||||
if active_buffer.__len__() == buffer_size:
|
||||
active_buffer.popleft()
|
||||
active_buffer.append(False)
|
||||
if active_buffer.count(False) != active_buffer.maxlen:
|
||||
continue
|
||||
if time.time() - bg_t > recorder.min_act_time:
|
||||
# end recording
|
||||
self.stream_record_process(
|
||||
bytes_frames=recorder.write_wave_bytes(full_frames),
|
||||
frames_size=len(full_frames),
|
||||
record_chunk_size=record_chunk_size,
|
||||
sample_rate=recorder.RATE,
|
||||
min_stream_record_time=recorder.min_stream_record_time,
|
||||
is_bgn=is_bgn,
|
||||
is_end=True)
|
||||
# print(f"Tatal frames: {_total_frames*record_chunk_size}, valid frame: {_frames*record_chunk_size}, valid RATE: {_frames/_total_frames*100:.2f}%, {_frames*record_chunk_size/recorder.RATE} sec.")
|
||||
# print("End recording.")
|
||||
break
|
||||
if self.board == 'orangepi':
|
||||
recorder.hardware.set_led2_off()
|
||||
|
||||
|
||||
def stream_record_process(self,
|
||||
bytes_frames: bytes,
|
||||
frames_size: int,
|
||||
record_chunk_size: int,
|
||||
sample_rate: int,
|
||||
min_stream_record_time: int,
|
||||
is_bgn: bool,
|
||||
is_end: bool):
|
||||
'''
|
||||
Args:
|
||||
bytes_frames: bytes, audio data
|
||||
frames_size: int, audio data size
|
||||
record_chunk_size: int, audio data chunk size
|
||||
is_bgn: bool, is begin of stream
|
||||
is_end: bool, is end of stream
|
||||
|
||||
Returns:
|
||||
bool, if stream reset status
|
||||
'''
|
||||
if len(bytes_frames) == 0:
|
||||
return False
|
||||
if frames_size*record_chunk_size >= min_stream_record_time*sample_rate or is_end:
|
||||
if is_bgn and is_end:
|
||||
return False
|
||||
stream_data = dict(
|
||||
frames=bytes_frames,
|
||||
frames_size=frames_size,
|
||||
chunk_size=record_chunk_size,
|
||||
is_bgn=is_bgn,
|
||||
is_end=is_end)
|
||||
self.client_queue.put(('audio', stream_data))
|
||||
if is_end:
|
||||
# print("put None to client queue.")
|
||||
self.client_queue.put(None)
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def web_socket_client_process(self):
|
||||
|
||||
client = BaseWebSocketClient(self.server_args['server_url'], self.server_args['session_id'])
|
||||
print("Web socket client process started.")
|
||||
# print("Web socket client process started.")
|
||||
|
||||
while True:
|
||||
if self.client_queue.empty():
|
||||
continue
|
||||
|
||||
# print(f"init skt time: {datetime.now()}")
|
||||
# 唤醒
|
||||
client.wakeup_client()
|
||||
|
||||
# 发送数据
|
||||
for queue_data in QueueIterator(self.client_queue):
|
||||
if queue_data[0] == 'audio':
|
||||
audio_dict = queue_data[1]
|
||||
|
||||
client.send_per_data(
|
||||
audio=audio_dict['frames'],
|
||||
stream=True,
|
||||
voice_synthesize=True,
|
||||
is_end=audio_dict['is_end'],
|
||||
encoding='base64',
|
||||
)
|
||||
# print(f"send skt time: {datetime.now()}")
|
||||
# print(f"fnsh skt time: {datetime.now()}")
|
||||
|
||||
# 接收数据
|
||||
while True:
|
||||
response, data_type = client.receive_per_data()
|
||||
if data_type == dict:
|
||||
print(response) # 打印接收到的消息
|
||||
'''
|
||||
try:
|
||||
response = json.loads(response['msg'])
|
||||
if 'content' in response.keys():
|
||||
self.excute_queue.put((response['instruct'], response['content']))
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"json decode error: {e}")
|
||||
continue
|
||||
# print(f"recv json time: {datetime.now()}")
|
||||
'''
|
||||
elif data_type == bytes:
|
||||
# print(f"recv bytes time: {datetime.now()}")
|
||||
self.audio_play_queue.put(('audio_bytes', response))
|
||||
elif data_type == None:
|
||||
break # 如果没有接收到消息，则退出循环
|
||||
# print("接收完毕:", datetime.now())
|
||||
|
||||
def audio_play_process(self):
|
||||
'''
|
||||
Args:
|
||||
audio_play_queue: multiprocessing.Queue, audio play queue
|
||||
share_time_dict: multiprocessing.Manager.dict, shared time dict
|
||||
'''
|
||||
audio_player = AudioPlayer(**self.player_args)
|
||||
print("Audio play process started.")
|
||||
while True:
|
||||
item = self.audio_play_queue.get()
|
||||
if item[0] == 'audio_bytes':
|
||||
# 播放音频
|
||||
print("Playing audio...")
|
||||
tts_audio = item[1]
|
||||
print(f"tts_audio len: {len(tts_audio)}")
|
||||
print(f"play audio time: {datetime.now()}")
|
||||
try:
|
||||
# 播放
|
||||
self.speaker_active_set.set()
|
||||
tts_audio = audio_player.check_audio_type(tts_audio, return_type=None)
|
||||
for i in range(0, len(tts_audio), audio_player.CHUNK):
|
||||
audio_player.stream.write(tts_audio[i:i+audio_player.CHUNK])
|
||||
print("Playing {} data...{}/{}".format(item[0], i, len(tts_audio)))
|
||||
if self.mircophone_active_set.is_set():
|
||||
print("mirophone is active.")
|
||||
self.mircophone_active_set.wait()
|
||||
break
|
||||
|
||||
audio_player.stream.write(tts_audio[i+audio_player.CHUNK:])
|
||||
|
||||
# svae bytes to file, 追加写
|
||||
with open("chat_audio.txt", 'ab') as f:
|
||||
f.write(tts_audio)
|
||||
print("Audio saved.")
|
||||
print(f"audio data played.")
|
||||
except TypeError as e:
|
||||
print(f"audio play error: {e}")
|
||||
continue
|
||||
else:
|
||||
if item[0] == 'story':
|
||||
audio_data = audio_player.load_audio_file(f"/home/orangepi/story_22050/{item[1]}.wav")
|
||||
elif item[0] == 'music':
|
||||
audio_data = audio_player.load_audio_file("/home/orangepi/music_22050/1.wav")
|
||||
# 播放
|
||||
self.speaker_active_set.set()
|
||||
audio_data = audio_player.check_audio_type(audio_data, return_type=None)
|
||||
time.sleep(0.5)
|
||||
for i in range(0, len(audio_data), audio_player.CHUNK):
|
||||
audio_player.stream.write(audio_data[i:i+audio_player.CHUNK])
|
||||
print("Playing {} data...{}/{}".format(item[0], i, len(audio_data)))
|
||||
if self.mircophone_active_set.is_set():
|
||||
audio_player.close()
|
||||
print("Reinit audio player.")
|
||||
print("mirophone is active.")
|
||||
self.mircophone_active_set.wait()
|
||||
time.sleep(0.5)
|
||||
audio_player = AudioPlayer(**self.player_args)
|
||||
break
|
||||
|
||||
# audio_player.stream.write(audio_data[i+audio_player.CHUNK:])
|
||||
# print(f"{item[0]} data played.")
|
||||
|
||||
|
||||
|
||||
|
||||
def excute_process(self):
|
||||
'''
|
||||
Args:
|
||||
excute_queue: multiprocessing.Queue, excute display queue
|
||||
'''
|
||||
print("Excute process started.")
|
||||
|
||||
while True:
|
||||
if self.excute_queue.empty():
|
||||
continue
|
||||
|
||||
if self.speaker_active_set.is_set():
|
||||
instruct, content = self.excute_queue.get()
|
||||
|
||||
print(f"Got speaker info: {instruct, content}")
|
||||
|
||||
print(f"Playing {instruct} {content}...")
|
||||
print(f"play {instruct} time: {datetime.now()}")
|
||||
self.audio_play_queue.put((instruct, content))
|
||||
|
||||
self.speaker_active_set.clear()
|
||||
|
||||
Reference in new issue
Block a user