开发者

python利用joblib进行并行数据处理的代码示例

在数据量比较大的情况下,数据预处理有时候会非常耗费时间。

可以利用 joblib 中的 Parallel 和 delayed 进行多CPU并行处理

示例如下:

import random
import os
from glob import glob
from tqdm impo编程客栈rt tqdm
from joblib import Parallel, delayed
import soundfile as sf
import pycantonese as pct
from opencc import OpenCC
cc = OpenCC('s2hk')
######### lJSPeech ##########
def process_ljspeech_one_utterance(wav_path, text, mode, save_root):
    try:
        tmp = wav_path.split('/')
        spk = 'LJSpeech-1.1'
        wname = tmp[-1]
        tname = wname.replace('.wav','.txt')
        tex编程客栈t_to_path = f'{save_root}/{mode}/{spk}/{tname}'
        os.makedirs(os.path.dirname(text_to_path), exist_ok=True)
        fp = open(text_to_path, 'w')
        fp.write(text)
        fp.close()
        wav_to_path = f'{save_root}/{mode}/{spk}/{wname}'
        _, fs = sf.read(wav_path)
        if fs != 16000:
            cmd = f'sox {wav_path} -r 16000 {wav_to_path}'
        else:
            cmd = f'cp {wav_path} {wav_to_path}'
        os.system(cmd)
        assert False
    except BaseException:
        return
wavs_root = 'source_data/LJSpeech/LJSpeech-1.www.devze.com1'
data = []
with open(f'{wavs_root}/metadata.csv', 'r') as f:
    lines = f.readlines()
    for line in lines:
        uttid = line.strip().split('|')[0]
        wav_path = f'{wavs_root}/wavs/{uttid}.wav'
        text = line.strip().split('|')[2]
        data.append([wav_path, text])
    f.close()
valid_data = random.sample(data, 100)
train_data = [dt for dt in data if dt not in valid_data]
Parallel(n_jobs=20)(delayed(process_ljspeech_one_utterance)(wav_path, text, mode='train', save_root='wavs/LJSpeech') for wav_path,text in tqdm(train_data))
Parallel(20)(delayed(process_ljspeech_one_utterance)(wav_path, text, mode='valid', save_root='wavs/LJSpeech') for wav_path,text in tqdm(valid_data))
# Parallel(n_jobs=20): 指定20个CPU(默认是分配给不同的CPU)
all_wavs = glob('wavs/LJSpeech/*/*编程客栈/*.wav')
print(f'obtain {len(all_wavs)} wavs...')

到此这篇关于python利用joblib进行并行数据处理的代码示例的文章就介绍到这了,更多相关python joblib并行数据处理内容请搜索编程客栈(www.devze.com)以前的文章或继续浏览下面的相关文章希望大家以后多多支持编程javascript客栈(www.devze.com)!

0

上一篇:

下一篇:

精彩评论

暂无评论...
验证码 换一张
取 消

最新开发

开发排行榜