first commit for takway.ai

funasr/datasets/large_datasets/utils/clipping.py

@@ -0,0 +1,44 @@
+import numpy as np
+import torch
+
+from funasr.datasets.large_datasets.collate_fn import crop_to_max_size
+
+
+def clipping(data):
+    assert isinstance(data, list)
+    assert "key" in data[0]
+
+    keys = [x["key"] for x in data]
+
+    batch = {}
+    data_names = data[0].keys()
+    for data_name in data_names:
+        if data_name == "key":
+            continue
+        else:
+            if data[0][data_name].dtype.kind == "i":
+                tensor_type = torch.int64
+            else:
+                tensor_type = torch.float32
+
+            tensor_list = [torch.tensor(np.copy(d[data_name]), dtype=tensor_type) for d in data]
+            tensor_lengths = torch.tensor([len(d[data_name]) for d in data], dtype=torch.int32)
+
+            length_clip = min(tensor_lengths)
+            tensor_clip = tensor_list[0].new_zeros(
+                len(tensor_list), length_clip, tensor_list[0].shape[1]
+            )
+            for i, (tensor, length) in enumerate(zip(tensor_list, tensor_lengths)):
+                diff = length - length_clip
+                assert diff >= 0
+                if diff == 0:
+                    tensor_clip[i] = tensor
+                else:
+                    tensor_clip[i] = crop_to_max_size(tensor, length_clip)
+
+            batch[data_name] = tensor_clip
+            batch[data_name + "_lengths"] = torch.tensor(
+                [tensor.shape[0] for tensor in tensor_clip], dtype=torch.long
+            )
+
+    return keys, batch

funasr/datasets/large_datasets/utils/filter.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+
+
+def filter(
+    data, speech_length_min=100, speech_length_max=15000, token_length_min=0, token_length_max=200
+):
+    assert "speech" in data or "text" in data
+
+    if "speech" in data and "text" in data:
+        if "sampling_rate" in data:
+            speech_length = (data["speech"].shape[0] / data["sampling_rate"]) * 1000.0
+        else:
+            speech_length = data["speech"].shape[0]
+        num_tokens = len(data["text"])
+        return (
+            speech_length_min < speech_length < speech_length_max
+            and token_length_min < num_tokens < token_length_max
+        )
+    elif "speech" in data:
+        if "sampling_rate" in data:
+            speech_length = (data["speech"].shape[0] / data["sampling_rate"]) * 1000.0
+        else:
+            speech_length = data["speech"].shape[0]
+        return speech_length_min < speech_length < speech_length_max
+    else:
+        num_tokens = len(data["text"])
+        return token_length_min < num_tokens < token_length_max

funasr/datasets/large_datasets/utils/hotword_utils.py

@@ -0,0 +1,42 @@
+import random
+
+
+def sample_hotword(
+    length,
+    hotword_min_length,
+    hotword_max_length,
+    sample_rate,
+    double_rate,
+    pre_prob,
+    pre_index=None,
+    pre_hwlist=None,
+):
+    if length < hotword_min_length:
+        return [-1]
+    if random.random() < sample_rate:
+        if pre_prob > 0 and random.random() < pre_prob and pre_index is not None:
+            return pre_index
+        if length == hotword_min_length:
+            return [0, length - 1]
+        elif random.random() < double_rate and length > hotword_max_length + hotword_min_length + 2:
+            # sample two hotwords in a sentence
+            _max_hw_length = min(hotword_max_length, length // 2)
+            # first hotword
+            start1 = random.randint(0, length // 3)
+            end1 = random.randint(start1 + hotword_min_length - 1, start1 + _max_hw_length - 1)
+            # second hotword
+            start2 = random.randint(end1 + 1, length - hotword_min_length)
+            end2 = random.randint(
+                min(length - 1, start2 + hotword_min_length - 1),
+                min(length - 1, start2 + hotword_max_length - 1),
+            )
+            return [start1, end1, start2, end2]
+        else:  # single hotword
+            start = random.randint(0, length - hotword_min_length)
+            end = random.randint(
+                min(length - 1, start + hotword_min_length - 1),
+                min(length - 1, start + hotword_max_length - 1),
+            )
+            return [start, end]
+    else:
+        return [-1]

funasr/datasets/large_datasets/utils/low_frame_rate.py

@@ -0,0 +1,30 @@
+import numpy as np
+
+
+def build_LFR_features(data, m, n):
+    """
+    Actually, this implements stacking frames and skipping frames.
+    if m = 1 and n = 1, just return the origin features.
+    if m = 1 and n > 1, it works like skipping.
+    if m > 1 and n = 1, it works like stacking but only support right frames.
+    if m > 1 and n > 1, it works like LFR.
+
+    Args:
+        inputs_batch: inputs is T x D np.ndarray
+        m: number of frames to stack
+        n: number of frames to skip
+    """
+
+    LFR_inputs = []
+    T = data.shape[0]
+    T_lfr = int(np.ceil(T / n))
+    for i in range(T_lfr):
+        if m <= T - i * n:
+            LFR_inputs.append(np.hstack(data[i * n : i * n + m]))
+        else:
+            num_padding = m - (T - i * n)
+            frame = np.hstack(data[i * n :])
+            for _ in range(num_padding):
+                frame = np.hstack((frame, data[-1]))
+            LFR_inputs.append(frame)
+    return np.vstack(LFR_inputs)

funasr/datasets/large_datasets/utils/padding.py

@@ -0,0 +1,72 @@
+import numpy as np
+import torch
+from torch.nn.utils.rnn import pad_sequence
+
+
+def padding(data, float_pad_value=0.0, int_pad_value=-1):
+    assert isinstance(data, list)
+    assert "key" in data[0]
+    assert "speech" in data[0] or "text" in data[0]
+
+    keys = [x["key"] for x in data]
+
+    batch = {}
+    data_names = data[0].keys()
+    for data_name in data_names:
+        if data_name == "key" or data_name == "sampling_rate":
+            continue
+        else:
+            if data_name != "hotword_indxs":
+                if data[0][data_name].dtype.kind == "i":
+                    pad_value = int_pad_value
+                    tensor_type = torch.int64
+                else:
+                    pad_value = float_pad_value
+                    tensor_type = torch.float32
+
+            tensor_list = [torch.tensor(np.copy(d[data_name]), dtype=tensor_type) for d in data]
+            tensor_lengths = torch.tensor([len(d[data_name]) for d in data], dtype=torch.int32)
+            tensor_pad = pad_sequence(tensor_list, batch_first=True, padding_value=pad_value)
+            batch[data_name] = tensor_pad
+            batch[data_name + "_lengths"] = tensor_lengths
+
+    # SAC LABEL INCLUDE
+    if "hotword_indxs" in batch:
+        # if hotword indxs in batch
+        # use it to slice hotwords out
+        hotword_list = []
+        hotword_lengths = []
+        text = batch["text"]
+        text_lengths = batch["text_lengths"]
+        hotword_indxs = batch["hotword_indxs"]
+        dha_pad = torch.ones_like(text) * -1
+        _, t1 = text.shape
+        t1 += 1  # TODO: as parameter which is same as predictor_bias
+        nth_hw = 0
+        for b, (hotword_indx, one_text, length) in enumerate(
+            zip(hotword_indxs, text, text_lengths)
+        ):
+            dha_pad[b][:length] = 8405
+            if hotword_indx[0] != -1:
+                start, end = int(hotword_indx[0]), int(hotword_indx[1])
+                hotword = one_text[start : end + 1]
+                hotword_list.append(hotword)
+                hotword_lengths.append(end - start + 1)
+                dha_pad[b][start : end + 1] = one_text[start : end + 1]
+                nth_hw += 1
+                if len(hotword_indx) == 4 and hotword_indx[2] != -1:
+                    # the second hotword if exist
+                    start, end = int(hotword_indx[2]), int(hotword_indx[3])
+                    hotword_list.append(one_text[start : end + 1])
+                    hotword_lengths.append(end - start + 1)
+                    dha_pad[b][start : end + 1] = one_text[start : end + 1]
+                    nth_hw += 1
+        hotword_list.append(torch.tensor([1]))
+        hotword_lengths.append(1)
+        hotword_pad = pad_sequence(hotword_list, batch_first=True, padding_value=0)
+        batch["hotword_pad"] = hotword_pad
+        batch["hotword_lengths"] = torch.tensor(hotword_lengths, dtype=torch.int32)
+        batch["dha_pad"] = dha_pad
+        del batch["hotword_indxs"]
+        del batch["hotword_indxs_lengths"]
+    return keys, batch

funasr/datasets/large_datasets/utils/tokenize.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+import re
+import numpy as np
+from funasr.datasets.large_datasets.utils.hotword_utils import sample_hotword
+
+
+def forward_segment(text, seg_dict):
+    word_list = []
+    i = 0
+    while i < len(text):
+        longest_word = text[i]
+        for j in range(i + 1, len(text) + 1):
+            word = text[i:j]
+            if word in seg_dict:
+                if len(word) > len(longest_word):
+                    longest_word = word
+        word_list.append(longest_word)
+        i += len(longest_word)
+    return word_list
+
+
+def seg_tokenize(txt, seg_dict):
+    pattern = re.compile(r"^[\u4E00-\u9FA50-9]+$")
+    out_txt = ""
+    for word in txt:
+        word = word.lower()
+        if word in seg_dict:
+            out_txt += seg_dict[word] + " "
+        else:
+            if pattern.match(word):
+                for char in word:
+                    if char in seg_dict:
+                        out_txt += seg_dict[char] + " "
+                    else:
+                        out_txt += "<unk>" + " "
+            else:
+                out_txt += "<unk>" + " "
+    return out_txt.strip().split()
+
+
+def tokenize(data, vocab=None, seg_dict=None, punc_dict=None, bpe_tokenizer=None, hw_config=None):
+    assert "text" in data
+    assert isinstance(vocab, dict)
+    text = data["text"]
+    token = []
+    vad = -2
+    if bpe_tokenizer is not None:
+        text = bpe_tokenizer.text2tokens(" ".join(text))
+    if seg_dict is not None:
+        assert isinstance(seg_dict, dict)
+        text = seg_tokenize(text, seg_dict)
+
+    length = len(text)
+    if "hw_tag" in data:
+        pre_index = None
+        if hw_config["pre_hwlist"] is not None and hw_config["pre_prob"] > 0:
+            # enable preset hotword detect in sampling
+            for hw in hw_config["pre_hwlist"]:
+                hw = " ".join(seg_tokenize(hw, seg_dict))
+                _find = " ".join(text).find(hw)
+                if _find != -1:
+                    # _find = text[:_find].count(" ")  # bpe sometimes
+                    pre_index = [_find, _find + max(hw.count(" "), 1)]
+                    break
+        hotword_indxs = sample_hotword(length, **hw_config, pre_index=pre_index)
+        data["hotword_indxs"] = hotword_indxs
+        del data["hw_tag"]
+    for i in range(length):
+        x = text[i]
+        if i == length - 1 and "punc" in data and x.startswith("vad:"):
+            vad = x[4:]
+            if len(vad) == 0:
+                vad = -1
+            else:
+                vad = int(vad)
+        elif x in vocab:
+            token.append(vocab[x])
+        else:
+            token.append(vocab["<unk>"])
+
+    if "punc" in data and punc_dict is not None:
+        punc_token = []
+        for punc in data["punc"]:
+            if punc in punc_dict:
+                punc_token.append(punc_dict[punc])
+            else:
+                punc_token.append(punc_dict["_"])
+        data["punc"] = np.array(punc_token)
+
+    data["text"] = np.array(token)
+    if vad is not -2:
+        data["vad_indexes"] = np.array([vad], dtype=np.int64)
+    return data