update

takway/apps/data_struct.py

@@ -0,0 +1,133 @@
+import multiprocessing
+import queue
+from typing import Iterator
+import time
+import json
+import requests
+
+class QueueIterator:
+    def __init__(self, 
+                 queue, 
+                 timeout: int = 10):
+        self.queue = queue
+        self.timeout = timeout
+
+    def __iter__(self) -> Iterator:
+        return self
+
+    def __next__(self):
+        try:
+            data = self.queue.get(block=True, timeout=self.timeout)
+            if data is None:  # 使用None作为结束信号
+                # print("QueueIterator: End of data")
+                raise StopIteration
+            else:
+                # print("QueueIterator: Get data")
+                return data
+        except queue.Empty:
+            # print("QueueIterator: Queue is empty")
+            raise StopIteration
+
+
+
+
+def producer(queue: multiprocessing.Queue):
+    for i in range(5):  # 假设生产5个数据项
+        data = {'data': i, 'is_end': False}
+        queue.put(data)
+        time.sleep(1)
+    queue.put(None)  # 发送结束信号
+
+def get_stream_data_from_queue(queue: multiprocessing.Queue):
+    for data in QueueIterator(queue):
+        print(data)
+        yield json.dumps({'line': data, 'is_end': False})
+        # 模拟数据处理时间
+
+if __name__ == "__main__":
+    queue = multiprocessing.Queue()
+    # 创建并启动生产者进程
+    p = multiprocessing.Process(target=producer, args=(queue,))
+    p.start()
+
+    # 使用迭代器来消费Queue中的数据
+    for data in QueueIterator(queue):
+        print(data)
+
+    # 等待生产者进程结束
+    p.join()
+
+
+'''
+# request body
+{
+    "AUTH_INFO": {
+        "user": "", # string
+        "authid": "",   # string
+        "api_keys": "",  # string
+        "timestamp": "",  # string
+    },
+    "DATA": {
+        "Audio": {
+            "data": "xxxxx",    # base64 encoded data
+            "metadata": {
+                "rate": ;   # int
+                "channels": ;   # int
+                "format": ;  # int
+            }
+        },
+        "Text": {
+            "data": "xxxxx",    # base64 encoded data
+            "metadata": {
+                # TODO
+            }
+        },
+        "Image": {
+            "data": "xxxxx",    # base64 encoded data
+            "metadata": {
+                "width": ;  # int
+                "height": ; # int
+                "format": ; # string
+            }
+        }
+    }
+    "META_INFO": {
+        "model_type": "",     # string
+        "model_version": "",    # string
+        "model_url": "",    # string
+        "vits": {
+            "speaker_id": ;  # int
+    }
+}
+
+# response body
+{
+    RESPONSE_INFO: {
+        "status": "success/error",   # string
+        "message": "xxxxx",  # string
+    }
+    "DATA": {
+        "Audio": {
+            "data": "xxxxx",    # base64 encoded data
+            "metadata": {
+                "rate": ;   # int
+                "channels": ;   # int
+                "format": ;  # int
+            }
+        },
+        "Text": {
+            "data": "xxxxx",    # base64 encoded data
+            "metadata": {
+                "is_end": True/False,  # bool
+            }
+        }
+        "Image": {
+            "data": "xxxxx",    # base64 encoded data
+            "metadata": {
+                "width": ;  # int
+                "height": ; # int
+                "format": ; # string
+            }
+        }
+}
+'''

takway/audio_utils.py

@@ -0,0 +1,578 @@
+import io
+import os
+import time
+import pyaudio
+import wave
+import json
+import warnings
+import threading
+import numpy as np
+from collections import deque
+
+from .common_utils import encode_bytes2str, decode_str2bytes
+   
+from takway.board import *
+try:
+    import keyboard
+except:
+    pass
+                        
+def play_audio(audio_data, type='base64'):
+    '''
+    读取base64编码的音频流并播放
+    '''
+    # PyAudio配置
+    p = pyaudio.PyAudio()
+    stream = p.open(format=pyaudio.paInt16, channels=1, rate=22050, output=True)
+
+    # 播放音频
+    stream.write(audio_data)
+    stream.stop_stream()
+    stream.close()
+    p.terminate()
+
+'''
+import librosa
+def reshape_sample_rate(audio, sr_original=None, sr_target=16000):
+    # 获取原始采样率和音频数据
+    if isinstance(audio, tuple):
+        sr_original, audio_data = audio
+    elif isinstance(audio, bytes):
+        audio_data = np.frombuffer(audio, dtype=np.int16)
+    assert sr_original is not None, f"sr_original should be provided if audio is a \
+        numpy.ndarray, but got sr_original `{sr_original}`."
+    
+    if isinstance(audio_data, np.ndarray):
+        if audio_data.dtype == np.dtype('int16'):
+            audio_data = audio_data.astype(np.float32) / np.iinfo(np.int16).max
+        assert audio_data.dtype == np.dtype('float32'), f"audio_data should be float32, \
+            but got {audio_data.dtype}."
+    else:
+        raise TypeError(f"audio_data should be numpy.ndarray, but got {type(audio_data)}.")
+    
+    # 重新采样音频数据
+    audio_data_resampled = librosa.resample(audio_data, orig_sr=sr_original, target_sr=sr_target)
+    
+    if audio_data_resampled.dtype == np.dtype('float32'):
+        audio_data_resampled = np.int16(audio_data_resampled * np.iinfo(np.int16).max)
+    
+    # If the input was bytes, return the resampled data as bytes
+    if isinstance(audio, bytes):
+        audio_data_resampled = audio_data_resampled.tobytes()
+        
+    return audio_data_resampled
+
+# Example usage:
+# If your audio data is in bytes:
+# audio_bytes = b'...'  # Your audio data as bytes
+# audio_data_resampled = reshape_sample_rate(audio_bytes)
+
+# If your audio data is in numpy int16:
+# audio_int16 = np.array([...], dtype=np.int16)  # Your audio data as numpy int16
+# audio_data_resampled = reshape_sample_rate(audio_int16)
+'''
+
+
+
+# ####################################################### #
+# base audio class
+# ####################################################### #
+
+class BaseAudio:
+    def __init__(self, 
+                 filename=None, 
+                 input=False, 
+                 output=False, 
+                 CHUNK=1024, 
+                 FORMAT=pyaudio.paInt16, 
+                 CHANNELS=1, 
+                 RATE=16000,
+                 input_device_index=None,
+                 output_device_index=None,
+                 **kwargs):
+        self.CHUNK = CHUNK
+        self.FORMAT = FORMAT
+        self.CHANNELS = CHANNELS
+        self.RATE = RATE
+        self.filename = filename
+        assert input!= output, "input and output cannot be the same, \
+            but got input={} and output={}.".format(input, output)
+        print("------------------------------------------")
+        print(f"{'Input' if input else 'Output'} Audio Initialization: ")
+        print(f"CHUNK: {self.CHUNK} \nFORMAT: {self.FORMAT} \nCHANNELS: {self.CHANNELS} \nRATE: {self.RATE} \ninput_device_index: {input_device_index} \noutput_device_index: {output_device_index}")
+        print("------------------------------------------")
+        self.p = pyaudio.PyAudio()
+        self.stream = self.p.open(format=FORMAT,
+                                  channels=CHANNELS,
+                                  rate=RATE,
+                                  input=input,
+                                  output=output,
+                                  input_device_index=input_device_index,
+                                  output_device_index=output_device_index,
+                                  **kwargs)
+    
+    def load_audio_file(self, wav_file):
+        with wave.open(wav_file, 'rb') as wf:
+            params = wf.getparams()
+            frames = wf.readframes(params.nframes)
+            print("Audio file loaded.")
+            # Audio Parameters
+            # print("Channels:", params.nchannels)
+            # print("Sample width:", params.sampwidth)
+            # print("Frame rate:", params.framerate)
+            # print("Number of frames:", params.nframes)
+            # print("Compression type:", params.comptype)
+        return frames
+    
+    def check_audio_type(self, audio_data, return_type=None):
+        assert return_type in ['bytes', 'io', None], \
+            "return_type should be 'bytes', 'io' or None."
+        if isinstance(audio_data, str):
+            if len(audio_data) > 50:
+                audio_data = decode_str2bytes(audio_data)
+            else:
+                assert os.path.isfile(audio_data), \
+                    "audio_data should be a file path or a bytes object."
+                wf = wave.open(audio_data, 'rb')
+                audio_data = wf.readframes(wf.getnframes())
+        elif isinstance(audio_data, np.ndarray):
+            if audio_data.dtype == np.dtype('float32'):
+                audio_data = np.int16(audio_data * np.iinfo(np.int16).max)
+            audio_data = audio_data.tobytes()
+        elif isinstance(audio_data, bytes):
+            pass
+        else:
+            raise TypeError(f"audio_data must be bytes, numpy.ndarray or str, \
+                but got {type(audio_data)}")
+        
+        if return_type == None:
+            return audio_data
+        return self.write_wave(None, [audio_data], return_type)
+    
+    def write_wave(self, filename, frames, return_type='io'):
+        """Write audio data to a file."""
+        if isinstance(frames, bytes):
+            frames = [frames]
+        if not isinstance(frames, list): 
+            raise TypeError("frames should be \
+            a list of bytes or a bytes object, \
+            but got {}.".format(type(frames)))
+        
+        if return_type == 'io':
+            if filename is None:
+                filename = io.BytesIO()
+            if self.filename:
+                filename = self.filename
+            return self.write_wave_io(filename, frames) 
+        elif return_type == 'bytes':
+            return self.write_wave_bytes(frames)
+
+    
+    def write_wave_io(self, filename, frames):
+        """
+        Write audio data to a file-like object.
+        
+        Args:
+            filename: [string or file-like object], file path or file-like object to write
+            frames: list of bytes, audio data to write
+        """
+        wf = wave.open(filename, 'wb')
+        
+        # 设置WAV文件的参数
+        wf.setnchannels(self.CHANNELS)
+        wf.setsampwidth(self.p.get_sample_size(self.FORMAT))
+        wf.setframerate(self.RATE)
+        wf.writeframes(b''.join(frames))
+        wf.close()
+        if isinstance(filename, io.BytesIO):
+            filename.seek(0)   # reset file pointer to beginning
+        return filename
+    
+    def write_wave_bytes(self, frames):
+        """Write audio data to a bytes object."""
+        return b''.join(frames)
+
+
+# ####################################################### #
+# play audio data from Speaker
+# ####################################################### #
+
+class AudioPlayer(BaseAudio):
+    def __init__(self, 
+                 RATE=22050, 
+                 **kwargs):
+        super().__init__(output=True, RATE=RATE, **kwargs)
+
+    def play(self, audio_data):
+        # print("Playing audio data...")
+        audio_data = self.check_audio_type(audio_data, return_type=None)
+        
+        for i in range(0, len(audio_data), self.CHUNK):
+            self.stream.write(audio_data[i:i+self.CHUNK])
+            # print("Playing audio data...{}/{}".format(i, len(audio_data)))
+        self.stream.write(audio_data[i+self.CHUNK:])
+        # print("Audio data played.")
+        
+
+    def close(self):
+        self.stream.stop_stream()
+        self.stream.close()
+        self.p.terminate()
+
+# ####################################################### #
+# record audio data from microphone
+# ####################################################### #
+class BaseRecorder(BaseAudio):
+    def __init__(self, 
+                 input=True, 
+                 base_chunk_size=None, 
+                 RATE=16000, 
+                 **kwargs):
+        super().__init__(input=input, RATE=RATE, **kwargs)
+        self.base_chunk_size = base_chunk_size
+        if base_chunk_size is None:
+            self.base_chunk_size = self.CHUNK
+
+    def record(self, 
+               filename,
+               duration=5, 
+               return_type='io',
+               logger=None):
+        if logger is not None:
+            logger.info("Recording started.")
+        else:
+            print("Recording started.")
+        frames = []
+        for i in range(0, int(self.RATE / self.CHUNK * duration)):
+            data = self.stream.read(self.CHUNK, exception_on_overflow=False)
+            frames.append(data)
+        if logger is not None:
+            logger.info("Recording stopped.")
+        else:
+            print("Recording stopped.")
+        return self.write_wave(filename, frames, return_type)
+
+    def record_chunk_voice(self, 
+                           return_type='bytes', 
+                           CHUNK=None, 
+                           exception_on_overflow=True, 
+                           queue=None):
+        data = self.stream.read(self.CHUNK if CHUNK is None else CHUNK, 
+                                exception_on_overflow=exception_on_overflow)
+        if return_type is not None:
+            return self.write_wave(None, [data], return_type)
+        return data
+
+
+class HDRecorder(BaseRecorder):
+    def __init__(self, 
+                 board=None,
+                 hd_trigger='keyboard', 
+                 keyboard_key='space',
+                 voice_trigger=True,
+                 hd_chunk_size=None,
+                 hd_detect_threshold=50,
+                 **kwargs):
+        super().__init__(**kwargs)
+        assert hd_trigger in ['keyboard', 'button']
+        
+        self.hd_trigger = hd_trigger
+        self.voice_trigger = voice_trigger
+        
+        self.hd_chunk_size = hd_chunk_size
+        if hd_chunk_size is None:
+            self.hd_chunk_size = self.base_chunk_size
+        
+        if board == None:
+            assert hd_trigger == 'keyboard', "board should be `None` if hd_trigger is `keyboard`."
+            self.keyboard_key = keyboard_key
+            self.hardware = Keyboard(hd_trigger, keyboard_key, hd_detect_threshold)
+        else:
+            assert hd_trigger == 'button', f"hd_trigger should be `button` if board is `v329` or `orangepi`, but got `{hd_trigger}`."
+            if board == 'v329':
+                self.hardware = V329(hd_trigger, hd_detect_threshold)
+            elif board == 'orangepi':
+                self.hardware = OrangePi(hd_trigger, hd_detect_threshold)
+        print(f"Using {hd_trigger} as hardware trigger.")
+        
+    def wait_for_hardware_pressed(self):
+        return self.hardware.wait_for_hardware_pressed()
+    
+    @property
+    def is_hardware_pressed(self):
+        return self.hardware.is_hardware_pressed
+    
+    def record_hardware(self, return_type='bytes'):
+        """record audio when hardware trigger"""
+        print("Recording started for hardware trigger.")
+        frames = []
+        self.wait_for_hardware_pressed()
+        while True:
+            if self.hd_trigger == 'keyboard':
+                if keyboard.is_pressed(self.keyboard_key):
+                    print("recording...")
+                    data = self.record_chunk_voice(
+                        CHUNK=self.CHUNK, 
+                        return_type=None, 
+                        exception_on_overflow=False)
+                    frames.append(data)
+                else:
+                    break
+                    print("Recording stopped.")
+            elif self.hd_trigger == 'button':
+                if self.get_button_status():
+                    data = self.stream.read(self.CHUNK)
+                    frames.append(data)
+                else:
+                    break
+            else:
+                recording = False
+                raise ValueError("hd_trigger should be 'keyboard' or 'button'.")
+        return self.write_wave(self.filename, frames, return_type)
+    
+    '''
+    def record(self, return_type='bytes', queue=None):
+        if self.hd_trigger == 'all':
+            value_list = []  # 用于记录value的状态
+            if keyboard.is_pressed(self.keyboard_key):
+                audio_data = self.record_keyboard(return_type, queue)
+            elif self.button.get_value() == 0:
+                if self.get_button_status():
+                    audio_data = self.record_button(return_type, queue)
+            else:
+                audio_data = self.record_voice(return_type, queue)
+        elif self.hd_trigger == 'keyboard':
+            print("Press SPACE to start recording.")
+            keyboard.wait("space")
+            audio_data = self.record_keyboard(return_type, queue)
+        elif self.hd_trigger == 'button':
+            print("Touch to start recording...")
+            if self.button.get_value() == 0:
+                if self.get_button_status():
+                    audio_data = self.record_button(return_type, queue)
+        else:
+            audio_data = self.record_voice(return_type, queue)
+            
+        return audio_data
+    
+    def record_keyboard(self, return_type='bytes', queue=None):
+        """record audio when keyboard pressing"""
+        print("Recording started.")
+        frames = []
+        recording = True
+        while recording:
+            if keyboard.is_pressed(self.keyboard_key):
+                data = self.stream.read(self.CHUNK)
+                frames.append(data)
+            else:
+                recording = False
+                print("Recording stopped.")
+        return self.write_wave(self.filename, frames, return_type)
+        
+    def record_button(self, return_type='bytes', queue=None):
+        """record audio when button pressing"""
+        print("Recording started.")
+        frames = []
+        recording = True
+        while recording:
+            value = self.button.get_value()
+            if value == 0:
+                data = self.stream.read(CHUNK)
+                frames.append(data)
+            else:
+                recording = False
+                print("Recording stopped.")
+        return self.write_wave(self.filename, frames, return_type)
+    '''
+        
+# ####################################################### #
+# record audio data from microphone with VAD
+# ####################################################### #
+try:
+    import webrtcvad
+    webrtcvad_available = True
+except:
+    warnings.warn("webrtcvad module not found, please install it if use `vad` hd_trigger.")
+    webrtcvad_available = False
+
+class VADRecorder(HDRecorder):
+    def __init__(self, vad_sensitivity=0, frame_duration=30, vad_buffer_size=7, min_act_time=1,**kwargs):
+        super().__init__(**kwargs)
+        if webrtcvad_available:
+            self.vad = webrtcvad.Vad(vad_sensitivity)
+        self.vad_buffer_size = vad_buffer_size
+        self.vad_chunk_size = int(self.RATE * frame_duration / 1000)
+            
+        self.min_act_time = min_act_time    # 最小活动时间，单位秒
+            
+        self.is_currently_speaking = False
+        self.frames = []
+
+    def is_speech(self, data):
+        return self.vad.is_speech(data, self.RATE)
+    
+    def vad_filter(self, data):
+        pass
+        
+
+    def vad_record(self, return_type='io', CHUNK=None, queue=None, save_file=False):
+        """录音并进行语音活动检测人声并返回分割后的音频数据"""
+        all_frames = []
+        
+        buffer_size = self.vad_buffer_size
+        active_buffer = deque([False for i in range(buffer_size)], maxlen=buffer_size)
+        audio_buffer = deque(maxlen=buffer_size)
+        silence_buffer = deque([True for i in range(buffer_size)], maxlen=buffer_size)
+        
+        print("vad_recorded_audio VAD started. Press Ctrl+C to stop.")
+        try:
+            while True:
+                data = self.stream.read(self.vad_chunk_size)
+                all_frames.append(data)
+                print(f"VAD processing..., is_speech: {self.is_speech(data)}")
+                if self.is_speech(data):
+                    # 标志位buffer
+                    active_buffer.append(True); active_buffer.popleft()
+                    silence_buffer.append(False); silence_buffer.popleft()
+                    # 暂时增加到buffer中
+                    audio_buffer.append(data)
+                    # 如果满足检测要求
+                    if all(active_buffer):
+                        if not self.is_currently_speaking:
+                            print("Speech start detected")
+                            self.is_currently_speaking = True
+                            self.frames.extend(audio_buffer)   # 把说话的buffer也加上
+                    if self.is_currently_speaking:
+                        self.frames.append(data)
+                else:
+                    # 标志位buffer
+                    active_buffer.append(False); active_buffer.popleft()
+                    silence_buffer.append(True); silence_buffer.popleft()
+                    # 检测到人声并持续录音
+                    if self.is_currently_speaking:
+                        # 结束标志位
+                        if all(silence_buffer):
+                            print("Speech end detected")
+                            break
+        except KeyboardInterrupt:
+            print("KeyboardInterrupt")
+            
+        finally:
+            print("Stopping...")
+            if len(all_frames) > 0:
+                print(f"ALL frame: {len(all_frames)}")
+                print(f"ASR frame: {len(self.frames)}")
+                if save_file:
+                    self.write_wave(f"output_{time.time()}_all.wav", all_frames)
+                    self.write_wave(f"output_{time.time()}.wav", self.frames)
+            return self.write_wave(None, self.frames, return_type='bytes')
+        
+
+# ####################################################### #
+# record audio data from microphone with PicoVoice hot words detection
+# ####################################################### #
+
+import struct
+from datetime import datetime
+import pvporcupine
+
+class PicovoiceRecorder(VADRecorder):
+    def __init__(self, 
+                 access_key, 
+                 keywords=None, 
+                 keyword_paths=None, 
+                 model_path=None, 
+                 sensitivities=0.5, 
+                 library_path=None,
+                 **kwargs):
+        
+        super().__init__(**kwargs)
+        
+        pico_cfg = dict(
+            access_key=access_key,
+            keywords=keywords,
+            keyword_paths=keyword_paths,
+            model_path=model_path,
+            sensitivities=sensitivities,
+            library_path=library_path,
+        )
+        
+        self.pico_detector_init(pico_cfg)
+        
+        self.keywords = self.pico_cfg['keywords']
+        print(f"PicovoiceRecorder initialized with keywords: {self.keywords}")
+
+    def pico_detector_init(self, pico_cfg):
+        if pico_cfg['keyword_paths'] is None:
+            if pico_cfg['keywords'] is None:
+                raise ValueError(f"Either `--keywords` or `--keyword_paths` must be set. \
+                    Available keywords: {list(pvporcupine.KEYWORDS)}")
+
+            keyword_paths = [pvporcupine.KEYWORD_PATHS[x] for x in pico_cfg['keywords']]
+        else:
+            keyword_paths = pico_cfg['keyword_paths']
+
+        if pico_cfg['sensitivities'] is None:
+            pico_cfg['sensitivities'] = [0.5] * len(keyword_paths)
+        elif isinstance(pico_cfg['sensitivities'], float):
+            pico_cfg['sensitivities'] = [pico_cfg['sensitivities']] * len(keyword_paths)
+            
+        if len(keyword_paths) != len(pico_cfg['sensitivities']):
+            raise ValueError('Number of keywords does not match the number of sensitivities.')
+        
+        try:
+            self.porcupine = pvporcupine.create(
+                access_key=pico_cfg['access_key'],
+                keywords=pico_cfg['keywords'],
+                keyword_paths=keyword_paths,
+                model_path=pico_cfg['model_path'],
+                sensitivities=pico_cfg['sensitivities'],
+                library_path=pico_cfg['library_path'])
+        except pvporcupine.PorcupineInvalidArgumentError as e:
+            print("One or more arguments provided to Porcupine is invalid: ", pico_cfg.keys())
+            print(e)
+            raise e
+        except pvporcupine.PorcupineActivationError as e:
+            print("AccessKey activation error")
+            raise e
+        except pvporcupine.PorcupineActivationLimitError as e:
+            print("AccessKey '%s' has reached it's temporary device limit" % pico_cfg['access_key'])
+            raise e
+        except pvporcupine.PorcupineActivationRefusedError as e:
+            print("AccessKey '%s' refused" % pico_cfg['access_key'])
+            raise e
+        except pvporcupine.PorcupineActivationThrottledError as e:
+            print("AccessKey '%s' has been throttled" % pico_cfg['access_key'])
+            raise e
+        except pvporcupine.PorcupineError as e:
+            print("Failed to initialize Porcupine")
+            raise e
+
+        self.pico_cfg = pico_cfg
+        
+    def is_wakeup(self, data):
+        pcm = struct.unpack_from("h" * self.porcupine.frame_length, data)
+        result = self.porcupine.process(pcm)
+        # print(f"picovoice result: {result}")
+        if result >= 0:
+            print('[%s] Detected %s' % (str(datetime.now()), self.keywords[result]))
+            return True
+        # self.write_wave(f"output_{time.time()}.wav", [data])
+        # print(f"write to: output_{time.time()}.wav")
+        return False
+        
+
+    def record_picovoice(self, return_type=None, exception_on_overflow=False, queue=None):
+        
+        print("Recording started. Press Ctrl+C to stop.")
+        while True:
+            data = self.record_chunk_voice(
+                return_type=None, 
+                CHUNK=self.porcupine.frame_length, 
+                exception_on_overflow=exception_on_overflow, 
+                queue=queue)
+            
+            wake_up = self.is_wakeup(data)
+            if wake_up:
+                break
+        return True

takway/board/__init__.py

@@ -0,0 +1,4 @@
+from .base_hd import BaseHardware
+from .keyboard import Keyboard
+from .sipeed import V329
+from .orangepi import OrangePi

takway/board/base_hd.py

@@ -0,0 +1,32 @@
+import threading
+import time
+
+class BaseHardware:
+    def __init__(self, hd_trigger=None, hd_detect_threshold=50):
+        self.hd_trigger = hd_trigger
+        self.hd_detect_threshold = hd_detect_threshold
+        
+        self.hd_lock = threading.Lock()
+        self.shared_hd_status = False
+        
+        
+    def init_hd_thread(self):
+        hd_thread = threading.Thread(target=self.hd_detection_loop)
+        hd_thread.start()
+        # hd_thread.join()
+        print("HD detection thread started.")
+        
+    def hd_detection_loop(self):
+        pass
+    
+    @property
+    def is_hardware_pressed(self):
+        return self.shared_hd_status
+    
+    def wait_for_hardware_pressed(self):
+        print("Waiting for hardware trigger.")
+        while True:
+            if self.is_hardware_pressed:
+                time.sleep(0.01)
+                break
+        return True

takway/board/keyboard.py

@@ -0,0 +1,36 @@
+import keyboard
+import time
+
+from takway.board.base_hd import BaseHardware
+
+import datetime
+# 
+
+class Keyboard(BaseHardware):
+    def __init__(self, hd_trigger='keyboard', keyboard_key='space', hd_detect_threshold=50):
+        super().__init__(hd_trigger, hd_detect_threshold)
+        
+        self.keyboard_key = keyboard_key
+        self.init_hd_thread()
+        self.power_status = False # 单次触发按键状态
+        
+    def hd_detection_loop(self):
+        keyboard_status = False
+        last_status = False
+        while True:
+            '''
+            keyboard_status = keyboard.is_pressed(self.keyboard_key)
+            with self.hd_lock:
+                self.shared_hd_status = keyboard_status
+            '''
+            self.shared_hd_status = keyboard.is_pressed(self.keyboard_key)
+            time.sleep(0.001)
+            
+            if not self.shared_hd_status and last_status:
+                self.power_status = ~self.power_status
+                if self.power_status:
+                    print("Chating mode.")
+                else:
+                    print("Slience mode.")
+                print(f"pres time: {datetime.datetime.now()}")
+            last_status = self.shared_hd_status

takway/board/orangepi.py

@@ -0,0 +1,94 @@
+from takway.board.base_hd import BaseHardware
+import threading
+import datetime
+try:
+    import wiringpi
+    from wiringpi import GPIO
+except:
+    pass
+
+
+'''
+| GPIO | LED |
+|  --  | - - |
+| 0 | 红色 |
+| 1 | 黄色 |
+| 2 | 绿色 |
+| 3 | 蓝色 |
+| 4 | 白色 |
+
+| GPIO | BUTTOM |
+|  --  |  ----  |
+| 6 | 按键1 |
+| 8 | 按键2 |
+'''
+
+class OrangePi(BaseHardware):
+    def __init__(self, hd_trigger='button', hd_detect_threshold=50):
+        super().__init__(hd_trigger, hd_detect_threshold)
+        
+        self.LED_PIN_red = 0
+        self.LED_PIN_yellow = 1
+        self.LED_PIN_green = 2
+        self.LED_PIN_blue = 3
+        self.LED_PIN_white = 4
+        
+        self.BUTTON_PIN_1 = 6
+        self.BUTTON_PIN_2 = 8
+        
+        self.button_status_2 = False
+        self.led_set_status_2 = False
+        
+        self.power_status = False # 单次触发按键状态
+        
+        self.button_init()
+        self.init_hd_thread()
+    
+    def button_init(self):
+        wiringpi.wiringPiSetup()
+        # GPIO 输出模式
+        wiringpi.pinMode(self.LED_PIN_red,GPIO.OUTPUT)
+        wiringpi.pinMode(self.LED_PIN_yellow,GPIO.OUTPUT)
+        wiringpi.pinMode(self.LED_PIN_green,GPIO.OUTPUT)
+        wiringpi.pinMode(self.LED_PIN_blue,GPIO.OUTPUT)
+        wiringpi.pinMode(self.LED_PIN_white,GPIO.OUTPUT)
+        # GPIO 输入模式
+        wiringpi.pinMode(self.BUTTON_PIN_1,GPIO.INPUT)
+        wiringpi.pinMode(self.BUTTON_PIN_2,GPIO.INPUT)
+        
+    def init_hd_thread(self):
+        # hd_threads = [threading.Thread(target=self.hd_detection_loop), 
+        #               threading.Thread(target=self.hd_detection_loop_2)]
+        hd_threads = [threading.Thread(target=self.hd_detection_loop)]
+        for hd_thread in hd_threads:
+            hd_thread.start()
+    
+    def hd_detection_loop(self):
+        keyboard_status = False
+        last_status = False
+        while True:
+            self.button_status = wiringpi.digitalRead(self.BUTTON_PIN_1)
+            if self.button_status:
+                wiringpi.digitalWrite(self.LED_PIN_red, GPIO.LOW)
+            else:
+                wiringpi.digitalWrite(self.LED_PIN_red,GPIO.HIGH)
+            
+            if not self.button_status and last_status:
+                self.power_status = ~self.power_status
+                if self.power_status:
+                    print("Chating mode.")
+                else:
+                    print("Slience mode.")
+                print(f"pres time: {datetime.datetime.now()}")
+            last_status = self.button_status
+    
+    def set_led_on(self, color='red'):
+        wiringpi.digitalWrite(getattr(self, f'LED_PIN_{color}'), GPIO.HIGH)
+    
+    def set_led_off(self, color='red'):
+        wiringpi.digitalWrite(getattr(self, f'LED_PIN_{color}'), GPIO.LOW)
+
+if __name__ == '__main__':
+    orangepi = OrangePi()
+    while True:
+        pass

takway/board/sipeed.py

@@ -0,0 +1,58 @@
+import sys
+import warnings
+import threading
+import time
+from collections import deque
+
+from takway.board.base_hd import BaseHardware
+
+if "gpiod" in sys.modules:
+    # sipeed MaixSense V329
+    import gpiod as gpio
+else:
+    # 如果所有库都不存在，执行默认操作或抛出异常
+    # raise ImportError("gpiod package is not available.")
+    pass
+
+class V329(BaseHardware):
+    def __init__(self, hd_trigger='button', hd_detect_threshold=50):
+        super().__init__(hd_trigger, hd_detect_threshold)
+        self.button = self.button_init()
+        
+        self.init_hd_thread()
+    
+    def button_init(self):
+        PH_BASE = (8-1)*32 #PH
+
+        gpiochip1 = gpio.chip("gpiochip1")
+        button = gpiochip1.get_line((PH_BASE+5))
+        config = gpio.line_request()
+        config.request_type = gpio.line_request.DIRECTION_INPUT
+        config.flags = gpio.line_request.FLAG_BIAS_PULL_UP
+        button.request(config)
+        return button
+
+    @property
+    def button_status(self):
+        return True if self.button.get_value() == 1 else False
+        
+    def hd_detection_loop(self):
+        self.shared_hd_status = False
+        button_value_list = deque(maxlen=self.hd_detect_threshold)
+        
+        while True:
+            if len(button_value_list) > button_value_list.maxlen:
+                button_value_list.popleft()
+            button_value_list.append(self.button_status)
+            # 记录50个值，如果连续50个值都是True，则认为按钮被按下
+            if button_value_list.count(True) == button_value_list.maxlen:
+                with self.hd_lock:
+                    self.shared_hd_status = True
+            # 记录50个值，如果连续50个值都是False，则认为按钮被松开
+            if button_value_list.count(False) == button_value_list.maxlen:
+                with self.hd_lock:
+                    self.shared_hd_status = False
+    
+    
+
+

takway/cam_utils.py

@@ -0,0 +1,11 @@
+try:
+    import cv2
+except:
+    warnings.warn("OpenCV is not installed, please check the module if you need.")
+
+class Camera:
+    def __init__(self, 
+                 device='pc',
+                 width=1280, 
+                 height=720):
+        pass

takway/clients/client_utils.py

@@ -0,0 +1,160 @@
+import os
+import json
+import time
+import datetime
+import requests
+
+from takway.common_utils import encode_bytes2str, decode_str2bytes
+
+'''
+{
+    "RESPONSE_INFO": {
+        "status": "success/error",   # string
+        "message": "xxxxx",  # string
+    },
+    "DATA": {
+        "Audio": {
+            "data": "xxxxx",    # base64 encoded data
+            "metadata": {
+                "rate": ;   # int
+                "channels": ;   # int
+                "format": ;  # int
+            }
+        },
+        "Text": {
+            "data": "xxxxx",    # base64 encoded data
+            "metadata": {
+                "is_end": True/False,  # bool
+            }
+        },
+        "Image": {
+            "data": "xxxxx",    # base64 encoded data
+            "metadata": {
+                "width": ;  # int
+                "height": ; # int
+                "format": ; # string
+            }
+        }
+    }
+}
+'''
+
+class Client:
+    def __init__(self, server_url):
+        self.server_url = server_url
+
+    def gen_request_data(self, **kwargs):
+        # print("kwargs:", kwargs)
+        audio_data = kwargs.get("audio_data", None)
+        text_data = kwargs.get("text_data", dict())
+        return json.dumps(
+            {
+            "is_end": audio_data.get("is_end"),  # bool
+            "is_bgn": audio_data.get("is_bgn"),  # bool
+            "DATA": {
+                "Audio": {
+                    "data": encode_bytes2str(audio_data['frames']),    # base64 encoded data
+                    "metadata": {
+                        "frames_size": audio_data.get("frames_size"),  # string
+                        "chunk_size": audio_data.get("chunk_size"),  # int
+                        "is_end": audio_data.get("is_end"),  # bool
+                        }
+                    },
+                "Text": {
+                    "data": text_data.get("text"),    # base64 encoded data
+                    "metadata": {
+                        "chat_status": text_data.get("chat_status"),  # string
+                        "chat_history": text_data.get("chat_history"),  # list of dict
+                        }
+                    },
+                },
+            "META_INFO": {
+                # "model_version": kwargs.get("model_version", ""),    # string
+                # "model_url": kwargs.get("model_url", ""),    # string
+                "character": {
+                    "name": kwargs.get("character", "Klee"),  # string
+                    "speaker_id": kwargs.get("speaker_id", 113),  # int
+                    "wakeup_words": kwargs.get("wakeup_words", "可莉来啦"),  # list of string
+                    }
+                }
+            }
+        ) + '\n'
+    
+    def send_data_to_server(self, **kwargs):
+        return requests.post(self.server_url, 
+            data=self.gen_request_data(**kwargs), stream=True)
+
+
+
+
+# ############################################ #
+# ############  WebSocket Client  ############# #
+def check_audio_type(data, return_type='base64'):
+    '''
+    Check if the data type is valid.
+    '''
+    assert return_type in ['bytes', 'base64']
+    if return_type == 'base64':
+        if isinstance(data, bytes):
+            return encode_bytes2str(data)
+    elif return_type == 'bytes':
+        if isinstance(data, str):
+            return decode_str2bytes(data)
+    else:
+        raise ValueError('Invalid data type: {}.'.format(type(data)))
+
+import websocket
+from websocket import create_connection
+
+class BaseWebSocketClient:
+    def __init__(self, server_url, session_id):
+        self.server_url = server_url
+        self.session_id = session_id
+        
+    def wakeup_client(self):
+        '''
+        Start the client.
+        '''
+        self.websocket = create_connection(self.server_url)
+        
+    def send_per_data(self, 
+        text: str = '',
+        audio: bytes = b'',
+        stream: bool = True,
+        voice_synthesize: bool = False,
+        is_end: bool = False,
+        encoding: str = 'base64',
+        ):
+        '''
+        Send data to server.
+        
+        Args:
+            data: bytes, data to be sent to server.
+        '''
+                
+        self.websocket.send(json.dumps({
+            "text": text,
+            "audio": check_audio_type(audio, return_type=encoding),
+            "meta_info": {
+                "session_id": self.session_id,
+                "stream": stream,
+                "voice_synthesize": voice_synthesize,
+                "is_end": is_end,
+                "encoding": encoding,
+            }}))
+        
+    def receive_per_data(self):
+        try:
+            recv_data = self.websocket.recv()
+        except websocket._exceptions.WebSocketConnectionClosedException:
+            return None, None
+        try:
+            recv_data = json.loads(recv_data)
+        except json.JSONDecodeError as e:
+            # print(f"JSONDecodeError: {e}")
+            # is_end = True
+            pass
+        except Exception as e:
+            # print(f"receive_per_data error: {e}")
+            assert isinstance(recv_data, bytes), ValueError(f"Received data is not bytes, got {type(recv_data)}.")
+        return recv_data, type(recv_data)

takway/clients/local_client_utils.py

@@ -0,0 +1,513 @@
+# basic
+import io
+import time
+import json
+import random
+from collections import deque
+# log
+import logging
+import warnings
+# multiprocessing
+import queue
+import threading
+import multiprocessing
+# web request
+import requests
+import pyaudio
+# hot words detection
+import pvporcupine
+
+from takway.apps.data_struct import QueueIterator
+from takway.common_utils import *
+from takway.audio_utils import PicovoiceRecorder
+from takway.clients.client_utils import CharacterClient
+from takway.audio_utils import AudioPlayer
+from takway.emo_utils import EmoVideoPlayer
+
+
+class LocalClinet:
+    def __init__(self, 
+                 server_args, 
+                 recorder_args, 
+                 video_args, 
+                 emo_args, 
+                 log_args):
+        # server_args
+        self.server_args = server_args
+        # recorder_args
+        self.recorder_args = recorder_args
+        # video_args
+        self.video_args = video_args
+        # emo_args
+        self.emo_args = emo_args
+        # log_args
+        self.log_args = log_args
+        
+        # TODO: 设计多进程log queue
+        self.logger_init()
+        
+        
+    def logger_init(self):
+        # log_args
+        log_level = self.log_args['log_level']
+        log_file = self.log_args['log_file']
+        
+        if log_level == 'debug':
+            log_level = logging.DEBUG
+        elif log_level == 'info':
+            log_level = logging.INFO
+        
+        # logger
+        self.logger = logging.getLogger('mylogger')
+        self.logger.setLevel(log_level)
+        # handler 创建一个handler，用于写入日志文件
+        handler = logging.FileHandler(log_file)
+        handler.setLevel(log_level)
+        # stream handler 创建一个handler，用于输出到控制台
+        console = logging.StreamHandler()
+        console.setLevel(logging.INFO)
+        
+        # 定义handler的输出格式（formatter）
+        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+        handler.setFormatter(formatter)
+        console.setFormatter(formatter)
+        
+        # 添加handler        
+        self.logger.addHandler(handler)
+        self.logger.addHandler(console)
+        self.logger.info("Logger started.")
+        
+    def process_init(self):
+        # multiprocessing
+        manager = multiprocessing.Manager()
+        self.trigger_queue = manager.Queue()
+        self.client_queue = manager.Queue()
+        self.audio_play_queue = manager.Queue()
+        self.emo_display_queue = manager.Queue()
+        
+        self.share_time_dict = manager.dict()
+        
+        self.speaking_emo_event = manager.Event()
+        
+        processes = [
+            multiprocessing.Process(target=self.audio_process, 
+                                    args=(self.trigger_queue,self.client_queue)),
+            # multiprocessing.Process(target=self.camera_process, args=(self.trigger_queue,self.client_queue)),
+            multiprocessing.Process(target=self.local_client_process, 
+                                    args=(self.client_queue,self.audio_play_queue,self.emo_display_queue, self.share_time_dict)),
+            multiprocessing.Process(target=self.audio_play_process, 
+                                    args=(self.audio_play_queue,self.share_time_dict)),
+        ]
+        if self.emo_args.pop('enable'):
+            processes.append(
+                multiprocessing.Process(target=self.emo_display_process, args=(self.emo_display_queue,)),
+            )
+        
+        for process in processes:
+            process.start()
+        for process in processes:
+            process.join()
+
+    def audio_process(self, 
+                      trigger_queue, 
+                      client_queue):
+        """audio_process
+        
+        Args:    
+            trigger_queue: multiprocessing.Queue, trigger queue
+            client_queue: multiprocessing.Queue, client queue
+        """
+        
+        self.frame_chunk_size = self.recorder_args.pop('frame_chunk_size')
+        self.min_stream_record_time = self.recorder_args.pop('min_stream_record_time')
+        voice_trigger = self.recorder_args.pop('voice_trigger')
+        self.RATE = self.recorder_args['RATE']
+        
+        recorder = PicovoiceRecorder(**self.recorder_args)
+        
+        # shared data struct:
+        self.shared_waiting = False
+        self.shared_lock = threading.Lock()
+        self.shared_data_lock = threading.Lock()
+        
+        # create threads
+        threads = [threading.Thread(target=self.hardware_trigger_thread, args=(recorder,))]
+        if voice_trigger:
+            vioce_threads = [
+                threading.Thread(target=self.voice_trigger_thread, args=(recorder,)),
+            ]
+            threads.extend(vioce_threads)
+        for thread in threads:
+            thread.start()
+        self.logger.info("Audio Process started.")
+        
+        while True:
+            for thread in threads:
+                thread.join()
+            print(f"audio process exit") ; exit()
+            
+            
+    def hardware_trigger_thread(self, recorder):
+        """hardware_trigger_thread
+        
+        Args: 
+            recorder: takway.audio_utils.Recorder, recorder object
+        """
+        self.logger.info("Hardware trigger thread started.")
+        
+        trgrigger_status = False
+        while True:
+            if self.shared_waiting:
+                continue
+            
+            # init status buffer
+            is_bgn = True
+            _frames = 0
+            _total_frames = 0
+            frames = []
+            full_frames = []
+            
+            print("Waiting for button press...")
+            recorder.wait_for_hardware_pressed()
+            print("Button pressed.")
+            # stop voice trigger thread
+            with self.shared_data_lock:
+                self.shared_waiting = True  # shared_waiting 控制所有线程的待机状态，True表示待机，False表示工作
+            
+            print("Start recording...")
+            bg_t = time.time()
+            record_chunk_size = recorder.hd_chunk_size
+            while True:
+                
+                data = recorder.record_chunk_voice(
+                    CHUNK=record_chunk_size, 
+                    return_type=None, 
+                    exception_on_overflow=False)
+                
+                frames.append(data)
+                full_frames.append(data)
+                _total_frames += 1
+                
+                if not recorder.is_hardware_pressed:
+                    break
+                
+                stream_reset_status = self.stream_record(
+                    bytes_frames=recorder.write_wave_bytes(full_frames), 
+                    frames_size=len(full_frames),
+                    record_chunk_size=record_chunk_size,
+                    is_bgn=is_bgn,
+                    is_end=False)
+                if stream_reset_status:
+                    full_frames.clear()
+                    is_bgn = False
+            
+            self.stream_record(
+                bytes_frames=recorder.write_wave_bytes(full_frames), 
+                frames_size=len(full_frames),
+                record_chunk_size=record_chunk_size,
+                is_bgn=is_bgn,
+                is_end=True)
+            
+            print(f"Tatal frames: {_total_frames*record_chunk_size}, {_total_frames*record_chunk_size/recorder.RATE} sec.")
+            # recorder.write_wave_io(f"record_{int(bg_t)}.wav", frames); print(f"write record_{int(bg_t)}.wav")
+
+            with self.shared_data_lock:
+                self.shared_waiting = False # 恢复voice trigger线程工作
+                
+    def voice_trigger_thread(self, recorder):
+        """voice_trigger_thread
+        
+        Args: 
+            recorder: takway.audio_utils.Recorder, recorder object
+        """
+        self.logger.info("voice record thread started.")
+        
+        print("Waiting for wake up...")
+        while True:
+            if self.shared_waiting:
+                continue
+            data = recorder.record_chunk_voice(
+                CHUNK=recorder.porcupine.frame_length, 
+                return_type=None, 
+                exception_on_overflow=False, 
+                queue=None)
+            
+            record_chunk_size = recorder.vad_chunk_size
+            
+            if not recorder.is_wakeup(data):
+                continue
+            # wake up
+            is_bgn = True
+            _frames = 0
+            _total_frames = 0
+            frames = []
+            full_frames = []
+            # status buffer
+            buffer_size = recorder.vad_buffer_size
+            active_buffer = deque(maxlen=buffer_size)
+            bg_t = time.time()
+            print("Start recording...")
+            while True:
+                data = recorder.record_chunk_voice(
+                    CHUNK=record_chunk_size, 
+                    return_type=None, 
+                    exception_on_overflow=False)
+                if data is None:
+                    continue
+                
+                is_speech = recorder.is_speech(data)
+                if is_speech:
+                    _frames += 1
+                    frames.append(data)
+                    print("add vad frame")
+                _total_frames += 1
+                full_frames.append(data)
+                
+                # send chunk data to client
+                stream_reset_status = self.stream_record(
+                    bytes_frames=recorder.write_wave_bytes(full_frames), 
+                    frames_size=len(full_frames),
+                    record_chunk_size=record_chunk_size,
+                    is_bgn=is_bgn,
+                    is_end=False)
+                if stream_reset_status:
+                    full_frames.clear()
+                    is_bgn = False
+                
+                if is_speech:
+                    if active_buffer.__len__() == buffer_size:
+                        active_buffer.popleft()
+                    active_buffer.append(True)
+                else:
+                    if active_buffer.__len__() == buffer_size:
+                        active_buffer.popleft()
+                    active_buffer.append(False)
+                    if active_buffer.count(False) != active_buffer.maxlen:
+                        continue
+                    if time.time() - bg_t > recorder.min_act_time:
+                        # end recording
+                        self.stream_record(
+                            bytes_frames=recorder.write_wave_bytes(full_frames), 
+                            frames_size=len(full_frames),
+                            record_chunk_size=record_chunk_size,
+                            is_bgn=is_bgn,
+                            is_end=True)
+                        print(f"Tatal frames: {_total_frames*record_chunk_size}, valid frame: {_frames*record_chunk_size}, valid rate: {_frames/_total_frames*100:.2f}%, {_frames*record_chunk_size/recorder.RATE} sec.")
+                        print("End recording.")
+                        break
+                
+    
+    
+    def stream_record(self, 
+                      bytes_frames: bytes, 
+                      frames_size: int,
+                      record_chunk_size: int,
+                      is_bgn: bool,
+                      is_end: bool):
+        '''
+        Args:
+            bytes_frames: bytes, audio data
+            frames_size: int, audio data size
+            record_chunk_size: int, audio data chunk size
+            is_bgn: bool, is begin of stream
+            is_end: bool, is end of stream
+        
+        Returns:
+            bool, if stream reset status
+        '''
+        if len(bytes_frames) == 0:
+            return False
+        if frames_size*record_chunk_size >= self.min_stream_record_time*self.RATE or is_end:
+            if is_bgn and is_end:
+                return False
+            stream_data = dict(
+                frames=bytes_frames, 
+                frames_size=frames_size,
+                chunk_size=record_chunk_size,
+                is_bgn=is_bgn,
+                is_end=is_end)
+            self.client_queue.put(('audio', stream_data))
+            if is_end:
+                print("put None to client queue.")
+                self.client_queue.put(None)
+            return True
+        else:
+            return False
+    
+    def camera_process(self, logger, trigger_queue, client_queue):
+        from takway.cam_utils import Camera
+        cam = Camera(self.video_args)
+        while True:
+            if trigger_queue.empty():
+                time.sleep(0.5)
+            else:
+                item = trigger_queue.get()
+                if item[0] == 'trgrigger_status' and item[1]:
+                    _, frame = cap.read()
+                    client_queue.put(('image', frame))
+
+
+    def local_client_process(self, 
+                             client_queue,
+                             audio_play_queue,
+                             emo_display_queue, 
+                             share_time_dict):
+        '''
+        Args:
+            client_queue: multiprocessing.Queue, client queue
+            audio_play_queue: multiprocessing.Queue, audio play queue
+            emo_display_queue: multiprocessing.Queue, emo display queue
+            share_time_dict: multiprocessing.Manager.dict, shared time dict
+        '''
+        character = self.server_args.pop('character')
+        client = CharacterClient(**self.server_args)
+        
+        # print(f"-------------------{character}-------------------")
+        # print(f"client.chat_status: {client.chat_status}")
+        if client.chat_status == 'init':
+            client.set_character(character)
+            client.chat_status = 'chating'
+        # print(f"client.chat_history: {client.chat_history}")
+                
+        self.logger.info("Local client process started.")
+        
+        while True:
+            response = None
+            if self.client_queue.empty():
+                continue
+            try:
+                response = requests.post(client.server_url, stream=True, 
+                    data=self.generate_stream_queue_data(client, client_queue))
+                print("get response from server.")
+                self.get_stream_response(client, response, audio_play_queue, emo_display_queue)
+            except requests.exceptions.ConnectionError or ConnectionError as e:
+                print(f"Wait for Server connection...")
+            except requests.exceptions.Timeout or Timeout as e:
+                print(f"Timeout: {e}")
+            except requests.exceptions.ChunkedEncodingError:
+                print("ChunkedEncodingError")
+                
+    def generate_stream_queue_data(self, client, client_queue, **kwargs):
+        _i = 0
+        for queue_data in QueueIterator(client_queue):
+            if queue_data[0] == 'audio':
+                _i += 1
+                if _i == 1:
+                    self.share_time_dict['client_time'] = [time.time()]
+                else:
+                    self.share_time_dict['client_time'].append(time.time())
+                audio_data = queue_data[1]
+                print("send audio data to server...")
+                # print(f"local chat history: {client.chat_history}")
+                yield client.gen_request_data(
+                        audio_data=audio_data, 
+                        chat_data=dict(
+                            chat_history=client.chat_history,
+                            chat_status=client.chat_status),
+                        character_data=client.character_info)
+                
+    def get_stream_response(self, 
+                            client, 
+                            response, 
+                            audio_play_queue=None,
+                            emo_display_queue=None, 
+                            chunk_size=1024):
+        '''
+        Args:
+            client: takway.client_utils.CharacterClient, client object
+            response: requests.Response, response object
+            audio_play_queue: multiprocessing.Queue, audio play queue
+            emo_display_queue: multiprocessing.Queue, emo display queue
+            chunk_size: int, chunk size
+        '''
+        assert isinstance(response, requests.Response), \
+            f"response is not requests.Response, but {type(response)}"
+        
+        temp_data = ''  # init temp_data
+        
+        if response.status_code == 200:
+            print("get response from server successfully.")
+        else:
+            print(f"response error, status code: {response.status_code}")
+        
+        chat_llm_response = ''
+        
+        _i = 0
+        # for chunk in response.iter_lines():
+            # if chunk:
+        for chunk in response.iter_content(chunk_size=chunk_size):
+            temp_data += chunk.decode('utf-8')
+            if temp_data.endswith('\n'):
+                _i += 1
+                try:
+                    temp_json = json.loads(temp_data.rstrip('\n'))
+                    # phase 1: get audio data
+                    audio_play_queue.put(('server_data', temp_json['audio_output']['tts_stream_data']))
+                    # phase 2: get chat data
+                    chat_llm_response += temp_json['chat_output']['llm_stream_data']
+                    
+                    if temp_json['is_end']:
+                        client.update_chat_history(question=temp_json['chat_output']['question'],
+                            response=chat_llm_response, asw_prompt_id=1)
+                    # print(f"chat_history: {client.chat_history}")
+                    if _i == 1:
+                        emo_display_queue.put(('emo_data', '高兴'))
+                except json.JSONDecodeError:
+                    print(f"json decode error: {temp_data}")
+                temp_data = ''
+                # print("get response.")
+        print("End get response.")
+            
+    def audio_play_process(self, 
+                           audio_play_queue, 
+                           share_time_dict):
+        '''
+        Args:
+            audio_play_queue: multiprocessing.Queue, audio play queue
+            share_time_dict: multiprocessing.Manager.dict, shared time dict
+        '''
+        audio_player = AudioPlayer()
+        self.logger.info("Audio play process started.")
+        while True:
+            self.speaking_emo_event.clear()
+            item = audio_play_queue.get()
+            self.speaking_emo_event.set()   # stop emo random display
+            if item[0] == 'server_data':
+                # 播放音频
+                print("Playing audio...")
+                tts_audio = item[1]
+                print(f"wait time: {(time.time() - self.share_time_dict['client_time'][0])*1000:.2f} ms")
+                try:
+                    audio_player.play(tts_audio)
+                except TypeError as e:
+                    # print(f"audio play error: {e}")
+                    # print(f"tts_audio: {tts_audio}")
+                    # print(f"type tts_audio: {type(tts_audio)}")
+                    # tts_audio: <class 'NoneType'>
+                    continue
+
+
+    def emo_display_process(self, emo_display_queue):
+        '''
+        Args:
+            emo_display_queue: multiprocessing.Queue, emo display queue
+        '''
+        emo_player = EmoVideoPlayer(**self.emo_args)
+        self.logger.info("Emo display process started.")
+        while True:
+            if emo_display_queue.empty():
+                time.sleep(0.1)
+                if self.speaking_emo_event.is_set():
+                    continue
+                emo_player.random_wink()
+            else:
+                item = emo_display_queue.get()
+                print(f"Emo display process Get item: {item[0]}")
+                if item[0] == 'emo_data':
+                    server_data = item[1]
+                    print("Displaying emo...")
+                    emo_player.display_emo(emo_name='兴奋', stage='start')
+                    emo_player.display_emo(emo_name='兴奋', stage='loop')
+                    emo_player.display_emo(emo_name='兴奋', stage='end')
+                    print("Display done.")
+                    time.sleep(15)

takway/clients/web_socket_client_utils.py

@@ -0,0 +1,330 @@
+# basic
+import io
+import os
+import sys
+import time
+import json
+import random
+from collections import deque
+from datetime import datetime
+# log
+import logging
+import warnings
+# multiprocessing
+import queue
+import threading
+import multiprocessing
+# web request
+import requests
+import pyaudio
+# hot words detection
+import pvporcupine
+
+from takway.apps.data_struct import QueueIterator
+from takway.common_utils import *
+from takway.audio_utils import PicovoiceRecorder, HDRecorder
+from takway.clients.client_utils import BaseWebSocketClient
+from takway.audio_utils import AudioPlayer
+
+
+class WebSocketClinet:
+    def __init__(self, 
+                 board, 
+                 server_args, 
+                 recorder_args, 
+                 player_args, 
+                 log_args, 
+                 excute_args=None, 
+                 ):
+        self.board = board
+        # server_args
+        self.server_args = server_args
+        # recorder_args
+        self.recorder_args = recorder_args
+        # player_args
+        self.player_args = player_args
+        # excute_args
+        self.excute_args = excute_args
+        # log_args
+        self.log_args = log_args
+        
+        
+    def process_init(self):
+        # multiprocessing
+        manager = multiprocessing.Manager()
+        self.trigger_queue = manager.Queue()
+        self.client_queue = manager.Queue()
+        self.audio_play_queue = manager.Queue()
+        self.excute_queue = manager.Queue()
+        
+        # 多进程标志为
+        self.mircophone_active_set = manager.Event()
+        self.speaker_active_set = manager.Event()
+                
+        processes = [
+            multiprocessing.Process(target=self.audio_process),
+            multiprocessing.Process(target=self.web_socket_client_process),
+            multiprocessing.Process(target=self.audio_play_process),
+        ]
+        if self.excute_args.get('enable', False):
+            processes.append(
+                multiprocessing.Process(target=self.excute_process),
+            )
+        
+        for process in processes:
+            time.sleep(0.5)
+            process.start()
+        for process in processes:
+            process.join()
+
+    def audio_process(self):
+        """audio_process
+        
+        Args:
+            trigger_queue: multiprocessing.Queue, trigger queue
+            client_queue: multiprocessing.Queue, client queue
+        """
+        min_stream_record_time = self.recorder_args.pop('min_stream_record_time')
+        voice_trigger = self.recorder_args.pop('voice_trigger')
+        # TODO: 
+        press_type = self.recorder_args.pop('press_type')
+        max_slience_time = self.recorder_args.pop('max_slience_time')
+        if voice_trigger:
+            recorder = PicovoiceRecorder(**self.recorder_args)
+        else:
+            voice_keys = ['access_key', 'keywords', 'keyword_paths', 'model_path','sensitivities', 'library_path']
+            for key in voice_keys:
+                self.recorder_args.pop(key)
+            recorder = HDRecorder(**self.recorder_args)
+        recorder.min_stream_record_time = min_stream_record_time
+        # TODO:
+        recorder.press_type = press_type
+        recorder.max_slience_time = max_slience_time
+        
+        print("Audio Process started.")
+        
+        print("Waiting for wake up...")
+        # recorder.hardware.set_led_on("green")
+        while True:
+            if self.shared_waiting:
+                continue
+            data = recorder.record_chunk_voice(
+                CHUNK=recorder.porcupine.frame_length, 
+                return_type=None, 
+                exception_on_overflow=False, 
+                queue=None)
+            
+            record_chunk_size = recorder.vad_chunk_size
+            
+            # 开关按键被按下或被关键词唤醒
+            if recorder.hardware.power_status or recorder.is_wakeup(data):
+                # recorder.hardware.set_led_on("blue")
+                pass
+            else:
+                continue
+            # wake up
+            is_bgn = True
+            is_end = False
+            frames = []
+            # status buffer
+            slience_bgn_t = time.time()
+            slience_time = 0
+            print("Start recording...")
+            # 准备对话状态
+            while True:
+                # 语音活动检测
+                data = recorder.record_chunk_voice(
+                    CHUNK=record_chunk_size, 
+                    return_type=None, 
+                    exception_on_overflow=False)
+                is_speech = recorder.is_speech(data)
+                
+                # 判断状态
+                if is_speech:
+                    print("valid voice")
+                    slience_bgn_t = time.time()
+                    frames.append(data)
+                else:
+                    slience_time = time.time() - slience_bgn_t
+
+                # 长时沉默关闭唤醒状态：如果唤醒后超过一定时间没有说话/关闭按键被按下，则认为是结束
+                if slience_time > recorder.max_slience_time or not recorder.hardware.power_status:
+                    break
+                
+                # 短时沉默结束单次对话：沉默时间超过一定时间段（0.5s左右），则发送数据
+                if slience_time > recorder.min_act_time:
+                    is_end = True
+                    is_bgn = False
+                
+                if not is_speech:
+                    continue
+                
+                # 流式发送数据
+                stream_reset_status = self.stream_record_process(
+                    bytes_frames=recorder.write_wave_bytes(frames), 
+                    frames_size=len(frames),
+                    record_chunk_size=record_chunk_size,
+                    sample_rate=recorder.RATE,
+                    min_stream_record_time=recorder.min_stream_record_time,
+                    is_bgn=is_bgn,
+                    is_end=is_end)
+                
+                if stream_reset_status:
+                    frames.clear()
+                    is_bgn = False
+                
+                        # print(f"Tatal frames: {_total_frames*record_chunk_size}, valid frame: {_frames*record_chunk_size}, valid RATE: {_frames/_total_frames*100:.2f}%, {_frames*record_chunk_size/recorder.RATE} sec.")
+                        # print("End recording.")
+                # recorder.hardware.set_led_off("blue")
+    
+    
+    def stream_record_process(self, 
+                            bytes_frames: bytes, 
+                            frames_size: int,
+                            record_chunk_size: int,
+                            sample_rate: int,
+                            min_stream_record_time: int,
+                            is_bgn: bool,
+                            is_end: bool):
+        '''
+        Args:
+            bytes_frames: bytes, audio data
+            frames_size: int, audio data size
+            record_chunk_size: int, audio data chunk size
+            is_bgn: bool, is begin of stream
+            is_end: bool, is end of stream
+        
+        Returns:
+            bool, if stream reset status
+        '''
+        if len(bytes_frames) == 0:
+            return False
+        if frames_size*record_chunk_size >= min_stream_record_time*sample_rate or is_end:
+            if is_bgn and is_end:
+                return False
+            stream_data = dict(
+                frames=bytes_frames, 
+                frames_size=frames_size,
+                chunk_size=record_chunk_size,
+                is_bgn=is_bgn,
+                is_end=is_end)
+            self.client_queue.put(('audio', stream_data))
+            if is_end:
+                # print("put None to client queue.")
+                self.client_queue.put(None)
+            return True
+        else:
+            return False
+    
+    def web_socket_client_process(self):
+        
+        client = BaseWebSocketClient(self.server_args['server_url'], self.server_args['session_id'])
+        print("Web socket client process started.")
+        # print("Web socket client process started.")
+        
+        while True:
+            if self.client_queue.empty():
+                continue
+            
+            # print(f"init skt time: {datetime.now()}")
+            # 唤醒
+            client.wakeup_client()
+            
+            # 发送数据
+            for queue_data in QueueIterator(self.client_queue):
+                if queue_data[0] == 'audio':
+                    audio_dict = queue_data[1]
+                
+                    client.send_per_data(
+                        audio=audio_dict['frames'],
+                        stream=True,
+                        voice_synthesize=True,
+                        is_end=audio_dict['is_end'],
+                        encoding='base64',
+                    )
+                    # print(f"send skt time: {datetime.now()}")
+            # print(f"fnsh skt time: {datetime.now()}")
+                    
+            # 接收数据
+            while True:
+                response, data_type = client.receive_per_data()
+                if data_type == dict:
+                    print(response)  # 打印接收到的消息
+                    '''
+                    try:
+                        response = json.loads(response['msg'])
+                        if 'content' in response.keys():
+                            self.excute_queue.put((response['instruct'], response['content']))
+                    except json.JSONDecodeError as e:
+                        print(f"json decode error: {e}")
+                        continue
+                    # print(f"recv json time: {datetime.now()}")
+                    '''
+                elif data_type == bytes:
+                    # print(f"recv bytes time: {datetime.now()}")
+                    self.audio_play_queue.put(('audio_bytes', response))
+                elif data_type == None:
+                    break  # 如果没有接收到消息，则退出循环
+            # print("接收完毕:", datetime.now())
+    
+    
+    def audio_play_process(self):
+        '''
+        Args:
+            audio_play_queue: multiprocessing.Queue, audio play queue
+            share_time_dict: multiprocessing.Manager.dict, shared time dict
+        '''
+        audio_player = AudioPlayer(**self.player_args)
+        print("Audio play process started.")
+        while True:
+            item = self.audio_play_queue.get()
+            # 播放音频
+            print("Playing audio...")
+            tts_audio = item[1]
+            print(f"tts_audio len: {len(tts_audio)}")
+            print(f"play audio time: {datetime.now()}")
+            try:
+                # 播放
+                self.speaker_active_set.set()
+                tts_audio = audio_player.check_audio_type(tts_audio, return_type=None)
+                for i in range(0, len(tts_audio), audio_player.CHUNK):
+                    audio_player.stream.write(tts_audio[i:i+audio_player.CHUNK])
+                    # print("Playing {} data...{}/{}".format(item[0], i, len(tts_audio)))
+                    if self.mircophone_active_set.is_set():
+                        print("mirophone is active.")
+                        self.mircophone_active_set.wait()
+                        break
+                audio_player.stream.write(tts_audio[i+audio_player.CHUNK:])
+                print(f"audio data played.")
+            except TypeError as e:
+                print(f"audio play error: {e}")
+                continue
+
+            # audio_player.stream.write(audio_data[i+audio_player.CHUNK:])
+            # print(f"{item[0]} data played.")
+                
+                
+        
+
+    def excute_process(self):
+        '''
+        Args:
+            excute_queue: multiprocessing.Queue, excute display queue
+        '''
+        print("Excute process started.")
+        
+        while True:
+            if self.excute_queue.empty():
+                continue
+            
+            if self.speaker_active_set.is_set():
+                instruct, content = self.excute_queue.get()
+                
+                print(f"Got speaker info: {instruct, content}")
+                
+                print(f"Playing {instruct} {content}...")
+                print(f"play {instruct} time: {datetime.now()}")
+                self.audio_play_queue.put((instruct, content))
+
+                self.speaker_active_set.clear()
+                

takway/clients/web_socket_client_utils_v1.py

@@ -0,0 +1,464 @@
+# basic
+import io
+import os
+import sys
+import time
+import json
+import random
+from collections import deque
+from datetime import datetime
+# log
+import logging
+import warnings
+# multiprocessing
+import queue
+import threading
+import multiprocessing
+# web request
+import requests
+import pyaudio
+# hot words detection
+import pvporcupine
+
+from takway.apps.data_struct import QueueIterator
+from takway.common_utils import *
+from takway.audio_utils import PicovoiceRecorder, HDRecorder
+from takway.clients.client_utils import BaseWebSocketClient
+from takway.audio_utils import AudioPlayer
+
+
+class WebSocketClinet:
+    def __init__(self, 
+                 board, 
+                 server_args, 
+                 recorder_args, 
+                 player_args, 
+                 log_args, 
+                 excute_args=None, 
+                 ):
+        self.board = board
+        # server_args
+        self.server_args = server_args
+        # recorder_args
+        self.recorder_args = recorder_args
+        # player_args
+        self.player_args = player_args
+        # excute_args
+        self.excute_args = excute_args
+        # log_args
+        self.log_args = log_args
+        
+        
+    def process_init(self):
+        # multiprocessing
+        manager = multiprocessing.Manager()
+        self.trigger_queue = manager.Queue()
+        self.client_queue = manager.Queue()
+        self.audio_play_queue = manager.Queue()
+        self.excute_queue = manager.Queue()
+        
+        # 多进程标志为
+        self.mircophone_active_set = manager.Event()
+        self.speaker_active_set = manager.Event()
+                
+        processes = [
+            multiprocessing.Process(target=self.audio_process),
+            multiprocessing.Process(target=self.web_socket_client_process),
+            multiprocessing.Process(target=self.audio_play_process),
+        ]
+        if self.excute_args.get('enable', False):
+            processes.append(
+                multiprocessing.Process(target=self.excute_process),
+            )
+        
+        for process in processes:
+            time.sleep(0.5)
+            process.start()
+        for process in processes:
+            process.join()
+
+    def audio_process(self):
+        """audio_process
+        
+        Args:    
+            trigger_queue: multiprocessing.Queue, trigger queue
+            client_queue: multiprocessing.Queue, client queue
+        """
+        min_stream_record_time = self.recorder_args.pop('min_stream_record_time')
+        voice_trigger = self.recorder_args.pop('voice_trigger')
+        press_type = self.recorder_args.pop('press_type')
+        if voice_trigger:
+            recorder = PicovoiceRecorder(**self.recorder_args)
+        else:
+            voice_keys = ['access_key', 'keywords', 'keyword_paths', 'model_path','sensitivities', 'library_path']
+            for key in voice_keys:
+                self.recorder_args.pop(key)
+            recorder = HDRecorder(**self.recorder_args)
+        recorder.min_stream_record_time = min_stream_record_time
+        recorder.press_type = press_type
+        
+        # shared data struct:
+        self.shared_waiting = False
+        self.shared_lock = threading.Lock()
+        self.shared_data_lock = threading.Lock()
+        
+        # create threads
+        threads = [threading.Thread(target=self.hardware_trigger_thread, args=(recorder,))]
+        if voice_trigger:
+            vioce_threads = [
+                threading.Thread(target=self.voice_trigger_thread, args=(recorder,)),
+            ]
+            threads.extend(vioce_threads)
+        for thread in threads:
+            thread.start()
+        print("Audio Process started.")
+        
+        while True:
+            for thread in threads:
+                thread.join()
+            print(f"audio process exit") ; exit()
+            
+            
+    def hardware_trigger_thread(self, recorder):
+        """hardware_trigger_thread
+        
+        Args: 
+            recorder: takway.audio_utils.Recorder, recorder object
+        """
+        print("Hardware trigger thread started.")
+        
+        trgrigger_status = False
+        record_chunk_size = recorder.hd_chunk_size
+        
+        while True:
+            if self.shared_waiting:
+                continue
+            
+            # init status buffer
+            is_bgn = True
+            frames = []
+            _total_frames = 0
+            
+            self.mircophone_active_set.clear()
+            print("Waiting for button press...")
+            recorder.wait_for_hardware_pressed()
+            print("Button pressed.")
+            self.mircophone_active_set.set()
+            # stop voice trigger thread
+            with self.shared_data_lock:
+                self.shared_waiting = True  # shared_waiting 控制所有线程的待机状态，True表示待机，False表示工作
+            
+            print("Start recording...")
+            bg_t = time.time()
+            while True:
+                
+                data = recorder.record_chunk_voice(
+                    CHUNK=record_chunk_size, 
+                    return_type=None, 
+                    exception_on_overflow=False)
+                
+                frames.append(data)
+                _total_frames += 1
+                
+                if not recorder.is_hardware_pressed:
+                    # print("Button released.")
+                    print(f"button rlse time: {datetime.now()}")
+                    break
+
+                stream_reset_status = self.stream_record_process(
+                    bytes_frames=recorder.write_wave_bytes(frames), 
+                    frames_size=len(frames),
+                    record_chunk_size=record_chunk_size,
+                    sample_rate=recorder.RATE,
+                    min_stream_record_time=recorder.min_stream_record_time,
+                    is_bgn=is_bgn,
+                    is_end=False)
+                if stream_reset_status:
+                    frames.clear()
+                    is_bgn = False
+            
+            self.stream_record_process(
+                bytes_frames=recorder.write_wave_bytes(frames), 
+                frames_size=len(frames),
+                record_chunk_size=record_chunk_size,
+                sample_rate=recorder.RATE,
+                min_stream_record_time=recorder.min_stream_record_time,
+                is_bgn=is_bgn,
+                is_end=True)
+            
+            # print(f"Tatal frames: {_total_frames*record_chunk_size}, {_total_frames*record_chunk_size/recorder.RATE} sec.")
+            # print(f"rcrd time: {datetime.now()}")
+            
+            with self.shared_data_lock:
+                self.shared_waiting = False # 恢复voice trigger线程工作
+                
+    def voice_trigger_thread(self, recorder):
+        """voice_trigger_thread
+        
+        Args: 
+            recorder: takway.audio_utils.Recorder, recorder object
+        """
+        print("voice record thread started.")
+        
+        print("Waiting for wake up...")
+        while True:
+            if self.shared_waiting:
+                continue
+            data = recorder.record_chunk_voice(
+                CHUNK=recorder.porcupine.frame_length, 
+                return_type=None, 
+                exception_on_overflow=False, 
+                queue=None)
+            
+            record_chunk_size = recorder.vad_chunk_size
+            
+            self.mircophone_active_set.clear()
+            if not recorder.is_wakeup(data):
+                continue
+            
+            if self.board == 'orangepi':
+                recorder.hardware.set_led2_on()
+            self.mircophone_active_set.set()
+            # wake up
+            is_bgn = True
+            _frames = 0
+            _total_frames = 0
+            frames = []
+            full_frames = []
+            # status buffer
+            buffer_size = recorder.vad_buffer_size
+            active_buffer = deque(maxlen=buffer_size)
+            bg_t = time.time()
+            print("Start recording...")
+            while True:
+                data = recorder.record_chunk_voice(
+                    CHUNK=record_chunk_size, 
+                    return_type=None, 
+                    exception_on_overflow=False)
+                if data is None:
+                    continue
+                
+                is_speech = recorder.is_speech(data)
+                if is_speech:
+                    _frames += 1
+                    frames.append(data)
+                    # print("add vad frame")
+                _total_frames += 1
+                full_frames.append(data)
+                
+                # send chunk data to client
+                stream_reset_status = self.stream_record_process(
+                    bytes_frames=recorder.write_wave_bytes(full_frames), 
+                    frames_size=len(full_frames),
+                    record_chunk_size=record_chunk_size,
+                    sample_rate=recorder.RATE,
+                    min_stream_record_time=recorder.min_stream_record_time,
+                    is_bgn=is_bgn,
+                    is_end=False)
+                if stream_reset_status:
+                    full_frames.clear()
+                    is_bgn = False
+                
+                if is_speech:
+                    if active_buffer.__len__() == buffer_size:
+                        active_buffer.popleft()
+                    active_buffer.append(True)
+                else:
+                    if active_buffer.__len__() == buffer_size:
+                        active_buffer.popleft()
+                    active_buffer.append(False)
+                    if active_buffer.count(False) != active_buffer.maxlen:
+                        continue
+                    if time.time() - bg_t > recorder.min_act_time:
+                        # end recording
+                        self.stream_record_process(
+                            bytes_frames=recorder.write_wave_bytes(full_frames), 
+                            frames_size=len(full_frames),
+                            record_chunk_size=record_chunk_size,
+                            sample_rate=recorder.RATE,
+                            min_stream_record_time=recorder.min_stream_record_time,
+                            is_bgn=is_bgn,
+                            is_end=True)
+                        # print(f"Tatal frames: {_total_frames*record_chunk_size}, valid frame: {_frames*record_chunk_size}, valid RATE: {_frames/_total_frames*100:.2f}%, {_frames*record_chunk_size/recorder.RATE} sec.")
+                        # print("End recording.")
+                        break
+            if self.board == 'orangepi':
+                recorder.hardware.set_led2_off()
+    
+    
+    def stream_record_process(self, 
+                            bytes_frames: bytes, 
+                            frames_size: int,
+                            record_chunk_size: int,
+                            sample_rate: int,
+                            min_stream_record_time: int,
+                            is_bgn: bool,
+                            is_end: bool):
+        '''
+        Args:
+            bytes_frames: bytes, audio data
+            frames_size: int, audio data size
+            record_chunk_size: int, audio data chunk size
+            is_bgn: bool, is begin of stream
+            is_end: bool, is end of stream
+        
+        Returns:
+            bool, if stream reset status
+        '''
+        if len(bytes_frames) == 0:
+            return False
+        if frames_size*record_chunk_size >= min_stream_record_time*sample_rate or is_end:
+            if is_bgn and is_end:
+                return False
+            stream_data = dict(
+                frames=bytes_frames, 
+                frames_size=frames_size,
+                chunk_size=record_chunk_size,
+                is_bgn=is_bgn,
+                is_end=is_end)
+            self.client_queue.put(('audio', stream_data))
+            if is_end:
+                # print("put None to client queue.")
+                self.client_queue.put(None)
+            return True
+        else:
+            return False
+    
+    def web_socket_client_process(self):
+        
+        client = BaseWebSocketClient(self.server_args['server_url'], self.server_args['session_id'])
+        print("Web socket client process started.")
+        # print("Web socket client process started.")
+        
+        while True:
+            if self.client_queue.empty():
+                continue
+            
+            # print(f"init skt time: {datetime.now()}")
+            # 唤醒
+            client.wakeup_client()
+            
+            # 发送数据
+            for queue_data in QueueIterator(self.client_queue):
+                if queue_data[0] == 'audio':
+                    audio_dict = queue_data[1]
+                
+                    client.send_per_data(
+                        audio=audio_dict['frames'],
+                        stream=True,
+                        voice_synthesize=True,
+                        is_end=audio_dict['is_end'],
+                        encoding='base64',
+                    )
+                    # print(f"send skt time: {datetime.now()}")
+            # print(f"fnsh skt time: {datetime.now()}")
+                    
+            # 接收数据
+            while True:
+                response, data_type = client.receive_per_data()
+                if data_type == dict:
+                    print(response)  # 打印接收到的消息
+                    '''
+                    try:
+                        response = json.loads(response['msg'])
+                        if 'content' in response.keys():
+                            self.excute_queue.put((response['instruct'], response['content']))
+                    except json.JSONDecodeError as e:
+                        print(f"json decode error: {e}")
+                        continue
+                    # print(f"recv json time: {datetime.now()}")
+                    '''
+                elif data_type == bytes:
+                    # print(f"recv bytes time: {datetime.now()}")
+                    self.audio_play_queue.put(('audio_bytes', response))
+                elif data_type == None:
+                    break  # 如果没有接收到消息，则退出循环
+            # print("接收完毕:", datetime.now())
+            
+    def audio_play_process(self):
+        '''
+        Args:
+            audio_play_queue: multiprocessing.Queue, audio play queue
+            share_time_dict: multiprocessing.Manager.dict, shared time dict
+        '''
+        audio_player = AudioPlayer(**self.player_args)
+        print("Audio play process started.")
+        while True:
+            item = self.audio_play_queue.get()
+            if item[0] == 'audio_bytes':
+                # 播放音频
+                print("Playing audio...")
+                tts_audio = item[1]
+                print(f"tts_audio len: {len(tts_audio)}")
+                print(f"play audio time: {datetime.now()}")
+                try:
+                    # 播放
+                    self.speaker_active_set.set()
+                    tts_audio = audio_player.check_audio_type(tts_audio, return_type=None)
+                    for i in range(0, len(tts_audio), audio_player.CHUNK):
+                        audio_player.stream.write(tts_audio[i:i+audio_player.CHUNK])
+                        print("Playing {} data...{}/{}".format(item[0], i, len(tts_audio)))
+                        if self.mircophone_active_set.is_set():
+                            print("mirophone is active.")
+                            self.mircophone_active_set.wait()
+                            break
+                        
+                    audio_player.stream.write(tts_audio[i+audio_player.CHUNK:])
+                    
+                    # svae bytes to file, 追加写
+                    with open("chat_audio.txt", 'ab') as f:
+                        f.write(tts_audio)
+                        print("Audio saved.")
+                    print(f"audio data played.")
+                except TypeError as e:
+                    print(f"audio play error: {e}")
+                    continue
+            else:
+                if item[0] == 'story':
+                    audio_data = audio_player.load_audio_file(f"/home/orangepi/story_22050/{item[1]}.wav")
+                elif item[0] == 'music':
+                    audio_data = audio_player.load_audio_file("/home/orangepi/music_22050/1.wav")
+                # 播放
+                self.speaker_active_set.set()
+                audio_data = audio_player.check_audio_type(audio_data, return_type=None)
+                time.sleep(0.5)
+                for i in range(0, len(audio_data), audio_player.CHUNK):
+                    audio_player.stream.write(audio_data[i:i+audio_player.CHUNK])
+                    print("Playing {} data...{}/{}".format(item[0], i, len(audio_data)))
+                    if self.mircophone_active_set.is_set():
+                        audio_player.close()
+                        print("Reinit audio player.")
+                        print("mirophone is active.")
+                        self.mircophone_active_set.wait()
+                        time.sleep(0.5)
+                        audio_player = AudioPlayer(**self.player_args)
+                        break
+                    
+                # audio_player.stream.write(audio_data[i+audio_player.CHUNK:])
+                # print(f"{item[0]} data played.")
+                
+                
+        
+
+    def excute_process(self):
+        '''
+        Args:
+            excute_queue: multiprocessing.Queue, excute display queue
+        '''
+        print("Excute process started.")
+        
+        while True:
+            if self.excute_queue.empty():
+                continue
+            
+            if self.speaker_active_set.is_set():
+                instruct, content = self.excute_queue.get()
+                
+                print(f"Got speaker info: {instruct, content}")
+                
+                print(f"Playing {instruct} {content}...")
+                print(f"play {instruct} time: {datetime.now()}")
+                self.audio_play_queue.put((instruct, content))
+
+                self.speaker_active_set.clear()
+                

takway/common_utils.py

@@ -0,0 +1,96 @@
+
+# ############################################################# #
+# format table function
+# ############################################################# #
+
+def format_table(header, rows):
+    # 计算列宽
+    col_width = max(len(str(word)) for row in rows for word in row) + 2  # 最大单词长度 + 2 作为列宽
+    # 打印表头
+    print("".join(word.ljust(col_width) for word in header))
+    # 打印分隔线
+    print("".join("-" * col_width for _ in header))
+    # 打印内容
+    for row in rows:
+        print("".join(str(word).ljust(col_width) for word in row))
+
+# ############################################################# #
+# encode and decode bytes and string
+# ############################################################# #
+
+import base64
+def encode_bytes2str(data):
+    # 将字节串编码为Base64
+    if data is None:
+        return None
+    return base64.b64encode(data).decode('utf-8')
+
+def decode_str2bytes(data):
+    # 将Base64编码的字节串解码为字节串
+    if data is None:
+        return None
+    return base64.b64decode(data.encode('utf-8'))
+
+import re
+def split_sentences(text: str):
+    # 定义中文标点符号的正则表达式
+    pattern = r'[\。\，\、\；\：\？\！\“\”\（\）\《\》]+'
+    # 使用正则表达式分割字符串
+    sentences = re.split(pattern, text)
+    # 过滤掉空字符串
+    sentences = [sentence for sentence in sentences if sentence]
+    return sentences
+'''
+# 示例文本
+text = "今天天气真好，我们去公园玩吧！你觉得怎么样？好的，那就这么定了。"
+# 调用函数进行断句
+sentences = split_sentences(text)
+
+print(sentences)
+'''
+
+def split_chinese_text(text: str, return_patch=False, punctuations=None):
+    # 定义中文标点符号集合
+    punctuations = set('。！？，；：、“”（）《》【】')
+    # 初始化断句结果列表和标点符号列表
+    sentences = []
+    punctuation_list = []
+    
+    text_patch = []
+    
+    start = 0  # 断句开始位置
+    for i, char in enumerate(text):
+        if char in punctuations:
+            # 如果当前字符是标点符号，则进行断句，并记录标点符号
+            sentences.append(text[start:i+1])
+            punctuation_list.append(char)
+            start = i + 1  # 更新断句开始位置
+    
+    # 处理最后一句（如果最后一句后没有标点符号）
+    if start < len(text):
+        sentences.append(text[start:])
+        
+        
+    if return_patch:
+        if len(punctuation_list) == 0:
+            return [text], False       # 有残留语句
+        elif len(sentences) == len(punctuation_list):
+            return [''.join(sentences)], True
+        else:
+            return [''.join(sentences[:-1]), sentences[-1]], True
+    return sentences, punctuation_list
+'''
+# 示例文本
+text = "你好，世界！今天天气怎么样？希望你有一个美好的一天。{}"
+sentences, punctuation_list = split_chinese_text(text)
+
+print("断句结果:", sentences)
+print("标点符号列表:", punctuation_list)
+'''
+
+def remove_brackets_and_contents(text):
+    # 使用sub函数替换匹配的文本为空字符串
+    result = re.sub(r'\(.*?\)', '', text)
+    result =  re.sub(r'\（.*?\）', '', result)
+    result =  re.sub(r'\【.*?\】', '', result)
+    return result

takway/emo_utils.py

@@ -0,0 +1,176 @@
+import time
+import av
+import os
+import copy
+import random
+import numpy as np
+try:
+    from maix import display, image
+    print("import maix success.")
+except:
+    import cv2
+    print("import cv2 success.")
+
+class EmoVideoPlayer:
+    def __init__(self, player='maixsense', emo_dir='ResizedEmoji'):
+        assert player in ['maixsense', 'opencv'], "player must be'maixsense' or 'opencv'"
+        self.player = player
+        
+        self.emo_list = ['兴奋', '愤怒', '静态', '不屑', '惊恐', '难过']
+        
+        self.emo_init(emo_dir)
+
+    def emo_init(self, emo_dir):
+        # 将此路径替换为Emoji文件夹的实际路径
+        self.emo_av_dict = self.get_emo_av(emo_dir)
+        self.emo_time_dict = {
+            '兴奋': 0.00,
+            '愤怒': 0.01,
+            '静态': 0.01,
+            '不屑': 0.01,
+            '惊恐': 0.01,
+            '难过': 0.01,
+        }
+        
+    def get_emo_av(self, emo_dir):
+        emo_av_dict = {emo: dict() for emo in self.emo_list}
+        for emo in self.emo_list:
+            emo_path = os.path.join(emo_dir, emo)
+            for file in os.listdir(emo_path):
+                if not os.path.isfile(os.path.join(emo_path, file)):
+                    continue
+                av_container = av.open(os.path.join(emo_path, file))
+                if emo == '静态':
+                    if "单次眨眼偶发" in file:
+                        emo_av_dict[emo]['seldom_wink'] = av_container
+                    if "快速双眨眼偶发" in file:
+                        emo_av_dict[emo]['quick_wink'] = av_container
+                else:
+                    if "进入姿势" in file:
+                        emo_av_dict[emo]['start'] = av_container
+                    elif "可循环动作" in file:
+                        emo_av_dict[emo]['loop'] = av_container
+                    elif "回正" in file:
+                        emo_av_dict[emo]['end'] = av_container
+        self.av_info = emo_av_dict[emo]['loop'].streams.video[0]
+        return emo_av_dict
+    
+    def get_emo_frames(self, emo_dir):
+        emo_av_dict = {emo: dict() for emo in self.emo_list}
+        for emo in self.emo_list:
+            emo_path = os.path.join(emo_dir, emo)
+            for file in os.listdir(emo_path):
+                if not os.path.isfile(os.path.join(emo_path, file)):
+                    continue
+                av_container = av.open(os.path.join(emo_path, file))
+                
+                frame_list = []
+                av_info = av_container.streams.video[0]
+                for frame in av_container.decode(video=0):
+                    if self.player =='maixsense':
+                        img = image.load(bytes(frame.to_rgb().planes[0]), (av_info.width, av_info.height))
+                    elif self.player == 'opencv':
+                        img = cv2.cvtColor(numpy.array(frame.to_image()), cv2.COLOR_RGB2BGR)
+                    frame_list.append(img)
+                # add to dict
+                if emo == '静态':
+                    if "单次眨眼偶发" in file:
+                        emo_av_dict[emo]['seldom_wink'] = frame_list
+                    if "快速双眨眼偶发" in file:
+                        emo_av_dict[emo]['quick_wink'] = frame_list
+                else:
+                    if "进入姿势" in file:
+                        emo_av_dict[emo]['start'] = frame_list
+                    elif "可循环动作" in file:
+                        emo_av_dict[emo]['loop'] = frame_list
+                    elif "回正" in file:
+                        emo_av_dict[emo]['end'] = frame_list
+        return emo_av_dict
+    
+    def display_emo_frame(self, emo_name, stage='default'):
+        emo_frame_list = self.emo_av_dict[emo_name][stage]
+        emo_time = self.emo_time_dict[emo_name]
+        for img in emo_frame_list:
+            if self.player =='maixsense':
+                display.show(img)
+            elif self.player == 'opencv':
+                cv2.imshow("video", img)
+                cv2.waitKey(1)  # 你可能需要根据视频的帧率调整这个延时
+            time.sleep(emo_time) 
+        
+    def display_emo(self, emo_name, stage='default'):
+        if self.player =='maixsense':
+            self.display_emo_maixsense(emo_name, stage)
+        elif self.player == 'opencv':
+            self.display_emo_opencv(emo_name, stage)
+        
+    def display_emo_maixsense(self, emo_name, stage):
+        emo_container = self.emo_av_dict[emo_name][stage]
+        emo_time = self.emo_time_dict[emo_name]
+        for frame in emo_container.decode(video=0):
+            img = image.load(bytes(frame.to_rgb().planes[0]), (self.av_info.width, self.av_info.height))
+            display.show(img)
+            time.sleep(emo_time) 
+        emo_container.seek(0)  # 重置视频的读取位置
+    
+    def display_emo_opencv(self, emo_name, stage='default'):
+        import cv2
+        import numpy
+        if stage == 'default':
+            if emo_name == '静态':
+                stage = 'quick_wink'
+            else:
+                stage = 'loop'
+        emo_container = self.emo_av_dict[emo_name][stage]
+        emo_time = self.emo_time_dict[emo_name]
+        
+        for frame in emo_container.decode(video=0):
+            img = cv2.cvtColor(numpy.array(frame.to_image()), cv2.COLOR_RGB2BGR)
+            cv2.imshow("video", img)
+            time.sleep(emo_time) 
+            cv2.waitKey(1)  # 你可能需要根据视频的帧率调整这个延时
+        cv2.destroyAllWindows()
+        emo_container.seek(0)  # 重置视频的读取位置
+        
+    def get_emo_status(self, answer):
+        # `兴奋`, `愤怒`, `静态`, `不屑`, `惊 恐`, `难过`
+        if any([emo in answer for emo in self.emo_list]):
+            # 找出是answer中出现了哪个emo    
+            emo_status = [emo for emo in self.emo_list if emo in answer][0]
+            print(f"emo_status: {emo_status}")
+        else:
+            emo_status = '静态'
+        return emo_status
+    
+    def random_wink(self):
+        seed = random.randrange(0, 1000)
+        if seed < 100:
+            self.display_emo(emo_name='静态', stage='seldom_wink')
+            # print("random wink")
+    
+    
+    
+if __name__ == '__main__':
+
+    emo = EmoVideoPlayer()
+    # emo.display_emo_opencv(emo_name='兴奋', stage='start')
+    # emo.display_emo_opencv(emo_name='兴奋', stage='loop')
+    # emo.display_emo_opencv(emo_name='兴奋', stage='loop')
+    # emo.display_emo_opencv(emo_name='兴奋', stage='loop')
+    # emo.display_emo_opencv(emo_name='兴奋', stage='end')
+    emo.display_emo_opencv(emo_name='静态', stage='seldom_wink')
+    emo.display_emo_opencv(emo_name='静态', stage='quick_wink')
+    # emo.display_emo_opencv(emo_name='愤怒', stage='start')
+    # emo.display_emo_opencv(emo_name='愤怒', stage='loop')
+    # emo.display_emo_opencv(emo_name='愤怒', stage='end')
+    # emo.display_emo_opencv(emo_name='静态', stage='seldom_wink')
+    # emo.display_emo_opencv(emo_name='静态', stage='quick_wink')
+    # emo.display_emo_opencv(emo_name='不屑', stage='start')
+    # emo.display_emo_opencv(emo_name='不屑', stage='loop')
+    # emo.display_emo_opencv(emo_name='不屑', stage='end')
+    # emo.display_emo_opencv(emo_name='惊恐', stage='start')
+    # emo.display_emo_opencv(emo_name='惊恐', stage='loop')
+    # emo.display_emo_opencv(emo_name='惊恐', stage='end')
+    # emo.display_emo_opencv(emo_name='难过', stage='start')
+    # emo.display_emo_opencv(emo_name='难过', stage='loop')
+    # emo.display_emo_opencv(emo_name='难过', stage='end')

takway/picovoice_utils.py

@@ -0,0 +1,83 @@
+import os
+import struct
+import wave
+from datetime import datetime
+
+import pvporcupine
+from pvrecorder import PvRecorder
+
+
+class PorcupineKeywordDetector:
+    def __init__(self, access_key, keywords=None, keyword_paths=None, library_path=None, model_path=None, sensitivities=None, audio_device_index=-1, output_path=None):
+        self.access_key = access_key
+        self.keywords = keywords
+        self.keyword_paths = keyword_paths
+        self.library_path = library_path
+        self.model_path = model_path
+        self.sensitivities = sensitivities if sensitivities is not None else [0.5] * len(self.keyword_paths)
+        self.audio_device_index = audio_device_index
+        self.output_path = output_path
+        self.porcupine = None
+        self.recorder = None
+        self.wav_file = None
+
+        if len(self.keyword_paths) != len(self.sensitivities):
+            raise ValueError('Number of keywords does not match the number of sensitivities.')
+
+        self._init_porcupine()
+
+    def _init_porcupine(self):
+        try:
+            self.porcupine = pvporcupine.create(
+                access_key=self.access_key,
+                library_path=self.library_path,
+                model_path=self.model_path,
+                keyword_paths=self.keyword_paths,
+                sensitivities=self.sensitivities)
+        except pvporcupine.PorcupineError as e:
+            print("Failed to initialize Porcupine:", e)
+            raise e
+
+    def start_detection(self):
+        self.recorder = PvRecorder(frame_length=self.porcupine.frame_length, device_index=self.audio_device_index)
+        self.recorder.start()
+
+        if self.output_path is not None:
+            self.wav_file = wave.open(self.output_path, "w")
+            self.wav_file.setnchannels(1)
+            self.wav_file.setsampwidth(2)
+            self.wav_file.setframerate(16000)
+
+        print('Listening ... (press Ctrl+C to exit)')
+        self._run_detection_loop()
+
+    def _run_detection_loop(self):
+        try:
+            while True:
+                pcm = self.recorder.read()
+                result = self.porcupine.process(pcm)
+
+                if self.wav_file is not None:
+                    self.wav_file.writeframes(struct.pack("h" * len(pcm), *pcm))
+
+                if result >= 0:
+                    print('[%s] Detected %s' % (str(datetime.now()), self.keywords[result]))
+        except KeyboardInterrupt:
+            print('Stopping ...')
+        finally:
+            self.stop_detection()
+
+    def stop_detection(self):
+        if self.recorder is not None:
+            self.recorder.delete()
+        if self.porcupine is not None:
+            self.porcupine.delete()
+        if self.wav_file is not None:
+            self.wav_file.close()
+
+    # You can add more methods here as needed, such as a method to list audio devices.
+
+# Usage example
+if __name__ == '__main__':
+    detector = PorcupineKeywordDetector(access_key='hqNqw85hkJRXVjEevwpkreB8n8so3w9JPQ27qnCR5qTH8a3+XnkZTA==')
+    detector.start_detection()