forked from Kedreamix/Linly-Talker
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathwebui.py
1377 lines (1258 loc) · 72.3 KB
/
webui.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
import os
import random
import gradio as gr
import time
import torch
import gc
import warnings
warnings.filterwarnings('ignore')
from zhconv import convert
from LLM import LLM
from TTS import EdgeTTS
from src.cost_time import calculate_time
from pyngrok import ngrok
from configs import *
os.environ["GRADIO_TEMP_DIR"]= './temp'
os.environ["WEBUI"] = "true"
ngrok.set_auth_token("2j1PCv3xmHA5Vjw4eBWSZqRfqy6_5zjuHaK8urTiiwUgRjxQs")
def get_title(title = 'Linly 智能对话系统 (Linly-Talker)'):
description = f"""
<p style="text-align: center; font-weight: bold;">
<span style="font-size: 28px;">{title}</span>
<br>
<span style="font-size: 18px;" id="paper-info">
[<a href="https://zhuanlan.zhihu.com/p/671006998" target="_blank">知乎</a>]
[<a href="https://www.bilibili.com/video/BV1rN4y1a76x/" target="_blank">bilibili</a>]
[<a href="https://github.com/Kedreamix/Linly-Talker" target="_blank">GitHub</a>]
[<a herf="https://kedreamix.github.io/" target="_blank">个人主页</a>]
</span>
<br>
<span>Linly-Talker是一款创新的数字人对话系统，它融合了最新的人工智能技术，包括大型语言模型（LLM）🤖、自动语音识别（ASR）🎙️、文本到语音转换（TTS）🗣️和语音克隆技术🎤。</span>
</p>
"""
return description
# 设置默认system
default_system = '你是一个很有帮助的助手'
# 设置默认的prompt
prefix_prompt = '''请用少于25个字回答以下问题\n\n'''
edgetts = EdgeTTS()
# 设定默认参数值，可修改
blink_every = True
size_of_image = 256
preprocess_type = 'crop'
facerender = 'facevid2vid'
enhancer = False
is_still_mode = False
exp_weight = 1
use_ref_video = False
ref_video = None
ref_info = 'pose'
use_idle_mode = False
length_of_audio = 5
@calculate_time
def Asr(audio):
try:
question = asr.transcribe(audio)
question = convert(question, 'zh-cn')
except Exception as e:
print("ASR Error: ", e)
question = 'Gradio存在一些bug，麦克风模式有时候可能音频还未传入，请重新点击一下语音识别即可'
gr.Warning(question)
return question
def clear_memory():
"""
清理PyTorch的显存和系统内存缓存。
"""
# 1. 清理缓存的变量
gc.collect() # 触发Python垃圾回收
torch.cuda.empty_cache() # 清理PyTorch的显存缓存
torch.cuda.ipc_collect() # 清理PyTorch的跨进程通信缓存
# 2. 打印显存使用情况（可选）
print(f"Memory allocated: {torch.cuda.memory_allocated() / (1024 ** 2):.2f} MB")
print(f"Max memory allocated: {torch.cuda.max_memory_allocated() / (1024 ** 2):.2f} MB")
print(f"Cached memory: {torch.cuda.memory_reserved() / (1024 ** 2):.2f} MB")
print(f"Max cached memory: {torch.cuda.max_memory_reserved() / (1024 ** 2):.2f} MB")
@calculate_time
def TTS_response(text,
voice, rate, volume, pitch,
am, voc, lang, male,
inp_ref, prompt_text, prompt_language, text_language, how_to_cut,
question_audio, question, use_mic_voice,
tts_method = 'PaddleTTS', save_path = 'answer.wav'):
# print(text, voice, rate, volume, pitch, am, voc, lang, male, tts_method, save_path)
if tts_method == 'Edge-TTS':
if not edgetts.network:
gr.Warning("请检查网络或者使用其他模型，例如PaddleTTS")
return None, None
try:
edgetts.predict(text, voice, rate, volume, pitch , 'answer.wav', 'answer.vtt')
except:
os.system(f'edge-tts --text "{text}" --voice {voice} --write-media answer.wav --write-subtitles answer.vtt')
return 'answer.wav', 'answer.vtt'
elif tts_method == 'PaddleTTS':
tts.predict(text, am, voc, lang = lang, male=male, save_path = save_path)
return save_path, None
elif tts_method == 'GPT-SoVITS克隆声音':
if use_mic_voice:
try:
vits.predict(ref_wav_path = question_audio,
prompt_text = question,
prompt_language = "中文",
text = text, # 回答
text_language = "中文",
how_to_cut = "凑四句一切",
save_path = 'answer.wav')
return 'answer.wav', None
except Exception as e:
gr.Warning("无克隆环境或者无克隆模型权重，无法克隆声音", e)
return None, None
else:
try:
vits.predict(ref_wav_path = inp_ref,
prompt_text = prompt_text,
prompt_language = prompt_language,
text = text, # 回答
text_language = text_language,
how_to_cut = how_to_cut,
save_path = 'answer.wav')
return 'answer.wav', None
except Exception as e:
gr.Warning("无克隆环境或者无克隆模型权重，无法克隆声音", e)
return None, None
return None, None
@calculate_time
def LLM_response(question_audio, question,
voice = 'zh-CN-XiaoxiaoNeural', rate = 0, volume = 0, pitch = 0,
am='fastspeech2', voc='pwgan',lang='zh', male=False,
inp_ref = None, prompt_text = "", prompt_language = "", text_language = "", how_to_cut = "", use_mic_voice = False,
tts_method = 'Edge-TTS'):
if len(question) == 0:
gr.Warning("请输入问题")
return None, None, None
answer = llm.generate(question, default_system)
print(answer)
driven_audio, driven_vtt = TTS_response(answer, voice, rate, volume, pitch,
am, voc, lang, male,
inp_ref, prompt_text, prompt_language, text_language, how_to_cut, question_audio, question, use_mic_voice,
tts_method)
return driven_audio, driven_vtt, answer
@calculate_time
def Talker_response(question_audio = None, method = 'SadTalker', text = '',
voice = 'zh-CN-XiaoxiaoNeural', rate = 0, volume = 100, pitch = 0,
am = 'fastspeech2', voc = 'pwgan', lang = 'zh', male = False,
inp_ref = None, prompt_text = "", prompt_language = "", text_language = "", how_to_cut = "", use_mic_voice = False,
tts_method = 'Edge-TTS',batch_size = 2, character = '女性角色',
progress=gr.Progress(track_tqdm=True)):
default_voice = None
if character == '女性角色':
# 女性角色
source_image, pic_path = r'inputs/girl.png', r'inputs/girl.png'
crop_pic_path = "./inputs/first_frame_dir_girl/girl.png"
first_coeff_path = "./inputs/first_frame_dir_girl/girl.mat"
crop_info = ((403, 403), (19, 30, 502, 513), [40.05956541381802, 40.17324339233366, 443.7892505041507, 443.9029284826663])
default_voice = 'zh-CN-XiaoxiaoNeural'
elif character == '男性角色':
# 男性角色
source_image = r'./inputs/boy.png'
pic_path = "./inputs/boy.png"
crop_pic_path = "./inputs/first_frame_dir_boy/boy.png"
first_coeff_path = "./inputs/first_frame_dir_boy/boy.mat"
crop_info = ((876, 747), (0, 0, 886, 838), [10.382158280494476, 0, 886, 747.7078990925525])
default_voice = 'zh-CN-YunyangNeural'
else:
gr.Warning('未知角色')
return None
voice = default_voice if not voice else voice
if not voice:
gr.Warning('请选择声音')
driven_audio, driven_vtt, _ = LLM_response(question_audio, text,
voice, rate, volume, pitch,
am, voc, lang, male,
inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
tts_method)
if driven_audio is None:
gr.Warning("音频没有正常生成，请检查TTS是否正确")
return None
if method == 'SadTalker':
pose_style = random.randint(0, 45)
video = talker.test(pic_path,
crop_pic_path,
first_coeff_path,
crop_info,
source_image,
driven_audio,
preprocess_type,
is_still_mode,
enhancer,
batch_size,
size_of_image,
pose_style,
facerender,
exp_weight,
use_ref_video,
ref_video,
ref_info,
use_idle_mode,
length_of_audio,
blink_every,
fps=20)
elif method == 'Wav2Lip':
video = talker.predict(crop_pic_path, driven_audio, batch_size, enhancer)
elif method == 'NeRFTalk':
video = talker.predict(driven_audio)
else:
gr.Warning("不支持的方法：" + method)
return None
if driven_vtt:
return video, driven_vtt
else:
return video
@calculate_time
def Talker_response_img(question_audio, method, text, voice, rate, volume, pitch,
am, voc, lang, male,
inp_ref , prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
tts_method,
source_image,
preprocess_type,
is_still_mode,
enhancer,
batch_size,
size_of_image,
pose_style,
facerender,
exp_weight,
blink_every,
fps, progress=gr.Progress(track_tqdm=True)
):
if enhancer:
gr.Warning("记得请先安装GFPGAN库，pip install gfpgan, 已安装可忽略")
if not voice:
gr.Warning("请先选择声音")
driven_audio, driven_vtt, _ = LLM_response(question_audio, text, voice, rate, volume, pitch,
am, voc, lang, male,
inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
tts_method = tts_method)
if driven_audio is None:
gr.Warning("音频没有正常生成，请检查TTS是否正确")
return None
if method == 'SadTalker':
video = talker.test2(source_image,
driven_audio,
preprocess_type,
is_still_mode,
enhancer,
batch_size,
size_of_image,
pose_style,
facerender,
exp_weight,
use_ref_video,
ref_video,
ref_info,
use_idle_mode,
length_of_audio,
blink_every,
fps=fps)
elif method == 'Wav2Lip':
video = talker.predict(source_image, driven_audio, batch_size)
elif method == 'NeRFTalk':
video = talker.predict(driven_audio)
else:
return None
if driven_vtt:
return video, driven_vtt
else:
return video
@calculate_time
def Talker_Say(preprocess_type,
is_still_mode,
enhancer,
batch_size,
size_of_image,
pose_style,
facerender,
exp_weight,
blink_every,
fps,source_image = None, source_video = None, question_audio = None, method = 'SadTalker', text = '',
voice = 'zh-CN-XiaoxiaoNeural', rate = 0, volume = 100, pitch = 0,
am = 'fastspeech2', voc = 'pwgan', lang = 'zh', male = False,
inp_ref = None, prompt_text = "", prompt_language = "", text_language = "", how_to_cut = "", use_mic_voice = False,
tts_method = 'Edge-TTS', character = '女性角色',
progress=gr.Progress(track_tqdm=True)):
if source_video:
source_image = source_video
default_voice = None
voice = default_voice if not voice else voice
if not voice:
gr.Warning('请选择声音')
driven_audio, driven_vtt = TTS_response(text, voice, rate, volume, pitch,
am, voc, lang, male,
inp_ref, prompt_text, prompt_language, text_language, how_to_cut, question_audio, text, use_mic_voice,
tts_method)
if driven_audio is None:
gr.Warning("音频没有正常生成，请检查TTS是否正确")
return None
if method == 'SadTalker':
pose_style = random.randint(0, 45)
video = talker.test2(source_image,
driven_audio,
preprocess_type,
is_still_mode,
enhancer,
batch_size,
size_of_image,
pose_style,
facerender,
exp_weight,
use_ref_video,
ref_video,
ref_info,
use_idle_mode,
length_of_audio,
blink_every,
fps=fps)
elif method == 'Wav2Lip':
video = talker.predict(source_image, driven_audio, batch_size, enhancer)
elif method == 'NeRFTalk':
video = talker.predict(driven_audio)
else:
gr.Warning("不支持的方法：" + method)
return None
if driven_vtt:
return video, driven_vtt
else:
return video
def chat_response(system, message, history):
# response = llm.generate(message)
response, history = llm.chat(system, message, history)
print(history)
# 流式输出
for i in range(len(response)):
time.sleep(0.01)
yield "", history[:-1] + [(message, response[:i+1])]
return "", history
def modify_system_session(system: str) -> str:
if system is None or len(system) == 0:
system = default_system
llm.clear_history()
return system, system, []
def clear_session():
# clear history
llm.clear_history()
return '', []
def human_response(source_image, history, question_audio, talker_method, voice, rate, volume, pitch,
am, voc, lang, male,
inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
tts_method, character,
preprocess_type, is_still_mode, enhancer, batch_size, size_of_image,
pose_style, facerender, exp_weight, blink_every, fps = 20, progress=gr.Progress(track_tqdm=True)):
response = history[-1][1]
qusetion = history[-1][0]
# driven_audio, video_vtt = 'answer.wav', 'answer.vtt'
if character == '女性角色':
# 女性角色
source_image, pic_path = r'./inputs/girl.png', r"./inputs/girl.png"
crop_pic_path = "./inputs/first_frame_dir_girl/girl.png"
first_coeff_path = "./inputs/first_frame_dir_girl/girl.mat"
crop_info = ((403, 403), (19, 30, 502, 513), [40.05956541381802, 40.17324339233366, 443.7892505041507, 443.9029284826663])
default_voice = 'zh-CN-XiaoxiaoNeural'
elif character == '男性角色':
# 男性角色
source_image = r'./inputs/boy.png'
pic_path = "./inputs/boy.png"
crop_pic_path = "./inputs/first_frame_dir_boy/boy.png"
first_coeff_path = "./inputs/first_frame_dir_boy/boy.mat"
crop_info = ((876, 747), (0, 0, 886, 838), [10.382158280494476, 0, 886, 747.7078990925525])
default_voice = 'zh-CN-YunyangNeural'
elif character == '自定义角色':
if source_image is None:
gr.Error("自定义角色需要上传正确的图片")
return None
default_voice = 'zh-CN-XiaoxiaoNeural'
voice = default_voice if not voice else voice
# tts.predict(response, voice, rate, volume, pitch, driven_audio, video_vtt)
driven_audio, driven_vtt = TTS_response(response, voice, rate, volume, pitch,
am, voc, lang, male,
inp_ref, prompt_text, prompt_language, text_language, how_to_cut, question_audio, qusetion, use_mic_voice,
tts_method)
if driven_audio is None:
gr.Warning("音频没有正常生成，请检查TTS是否正确")
return None
if talker_method == 'SadTalker':
pose_style = random.randint(0, 45)
video = talker.test(pic_path,
crop_pic_path,
first_coeff_path,
crop_info,
source_image,
driven_audio,
preprocess_type,
is_still_mode,
enhancer,
batch_size,
size_of_image,
pose_style,
facerender,
exp_weight,
use_ref_video,
ref_video,
ref_info,
use_idle_mode,
length_of_audio,
blink_every,
fps=fps)
elif talker_method == 'Wav2Lip':
video = talker.predict(crop_pic_path, driven_audio, batch_size, enhancer)
elif talker_method == 'NeRFTalk':
video = talker.predict(driven_audio)
else:
gr.Warning("不支持的方法：" + talker_method)
return None
if driven_vtt:
return video, driven_vtt
else:
return video
@calculate_time
def MuseTalker_response(source_video, bbox_shift, question_audio = None, text = '',
voice = 'zh-CN-XiaoxiaoNeural', rate = 0, volume = 100, pitch = 0,
am = 'fastspeech2', voc = 'pwgan', lang = 'zh', male = False,
inp_ref = None, prompt_text = "", prompt_language = "", text_language = "", how_to_cut = "", use_mic_voice = False,
tts_method = 'Edge-TTS', batch_size = 4,
progress=gr.Progress(track_tqdm=True)):
default_voice = None
voice = default_voice if not voice else voice
if not voice:
gr.Warning('请选择声音')
driven_audio, driven_vtt, _ = LLM_response(question_audio, text,
voice, rate, volume, pitch,
am, voc, lang, male,
inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
tts_method)
print(driven_audio, driven_vtt)
video = musetalker.inference_noprepare(driven_audio,
source_video,
bbox_shift,
batch_size,
fps = 25)
if driven_vtt:
return (video, driven_vtt)
else:
return video
GPT_SoVITS_ckpt = "GPT_SoVITS/pretrained_models"
def load_vits_model(gpt_path, sovits_path, progress=gr.Progress(track_tqdm=True)):
global vits
print("模型加载中...", gpt_path, sovits_path)
all_gpt_path, all_sovits_path = os.path.join(GPT_SoVITS_ckpt, gpt_path), os.path.join(GPT_SoVITS_ckpt, sovits_path)
vits.load_model(all_gpt_path, all_sovits_path)
gr.Info("模型加载成功")
return gpt_path, sovits_path
def list_models(dir, endwith = ".pth"):
list_folder = os.listdir(dir)
list_folder = [i for i in list_folder if i.endswith(endwith)]
return list_folder
def character_change(character):
if character == '女性角色':
# 女性角色
source_image = r'./inputs/girl.png'
elif character == '男性角色':
# 男性角色
source_image = r'./inputs/boy.png'
elif character == '自定义角色':
# gr.Warnings("自定义角色暂未更新，请继续关注后续，可通过自由上传图片模式进行自定义角色")
source_image = None
return source_image
def webui_setting(talk = False):
if not talk:
with gr.Tabs():
with gr.TabItem('数字人形象设定'):
source_image = gr.Image(label="Source image", type="filepath")
else:
source_image = None
with gr.Tabs("TTS Method"):
with gr.Accordion("TTS Method语音方法调节 ", open=True):
with gr.Tab("Edge-TTS"):
voice = gr.Dropdown(edgetts.SUPPORTED_VOICE,
value='zh-CN-XiaoxiaoNeural',
label="Voice 声音选择")
rate = gr.Slider(minimum=-100,
maximum=100,
value=0,
step=1.0,
label='Rate 速率')
volume = gr.Slider(minimum=0,
maximum=100,
value=100,
step=1,
label='Volume 音量')
pitch = gr.Slider(minimum=-100,
maximum=100,
value=0,
step=1,
label='Pitch 音调')
with gr.Tab("PaddleTTS"):
am = gr.Dropdown(["FastSpeech2"], label="声学模型选择", value = 'FastSpeech2')
voc = gr.Dropdown(["PWGan", "HifiGan"], label="声码器选择", value = 'PWGan')
lang = gr.Dropdown(["zh", "en", "mix", "canton"], label="语言选择", value = 'zh')
male = gr.Checkbox(label="男声(Male)", value=False)
with gr.Tab('GPT-SoVITS'):
with gr.Row():
gpt_path = gr.FileExplorer(root = GPT_SoVITS_ckpt, glob = "*.ckpt", value = "s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt", file_count='single', label="GPT模型路径")
sovits_path = gr.FileExplorer(root = GPT_SoVITS_ckpt, glob = "*.pth", value = "s2G488k.pth", file_count='single', label="SoVITS模型路径")
# gpt_path = gr.Dropdown(choices=list_models(GPT_SoVITS_ckpt, 'ckpt'))
# sovits_path = gr.Dropdown(choices=list_models(GPT_SoVITS_ckpt, 'pth'))
# gpt_path = gr.Textbox(label="GPT模型路径",
# value="GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt")
# sovits_path = gr.Textbox(label="SoVITS模型路径",
# value="GPT_SoVITS/pretrained_models/s2G488k.pth")
button = gr.Button("加载模型")
button.click(fn = load_vits_model,
inputs=[gpt_path, sovits_path],
outputs=[gpt_path, sovits_path])
with gr.Row():
inp_ref = gr.Audio(label="请上传3~10秒内参考音频，超过会报错！", sources=["microphone", "upload"], type="filepath")
use_mic_voice = gr.Checkbox(label="使用语音问答的麦克风")
prompt_text = gr.Textbox(label="参考音频的文本", value="")
prompt_language = gr.Dropdown(
label="参考音频的语种", choices=["中文", "英文", "日文"], value="中文"
)
asr_button = gr.Button("语音识别 - 克隆参考音频")
asr_button.click(fn=Asr,inputs=[inp_ref],outputs=[prompt_text])
with gr.Row():
text_language = gr.Dropdown(
label="需要合成的语种", choices=["中文", "英文", "日文", "中英混合", "日英混合", "多语种混合"], value="中文"
)
how_to_cut = gr.Dropdown(
label="怎么切",
choices=["不切", "凑四句一切", "凑50字一切", "按中文句号。切", "按英文句号.切", "按标点符号切" ],
value="凑四句一切",
interactive=True,
)
with gr.Column(variant='panel'):
batch_size = gr.Slider(minimum=1,
maximum=10,
value=2,
step=1,
label='Talker Batch size')
character = gr.Radio(['女性角色',
'男性角色',
'自定义角色'],
label="角色选择", value='自定义角色')
character.change(fn = character_change, inputs=[character], outputs = [source_image])
tts_method = gr.Radio(['Edge-TTS', 'PaddleTTS', 'GPT-SoVITS克隆声音', 'Comming Soon!!!'], label="Text To Speech Method",
value = 'Edge-TTS')
tts_method.change(fn = tts_model_change, inputs=[tts_method], outputs = [tts_method])
asr_method = gr.Radio(choices = ['Whisper-tiny', 'Whisper-base', 'FunASR', 'Comming Soon!!!'], value='Whisper-base', label = '语音识别模型选择')
asr_method.change(fn = asr_model_change, inputs=[asr_method], outputs = [asr_method])
talker_method = gr.Radio(choices = ['SadTalker', 'Wav2Lip', 'NeRFTalk', 'Comming Soon!!!'],
value = 'SadTalker', label = '数字人模型选择')
talker_method.change(fn = talker_model_change, inputs=[talker_method], outputs = [talker_method])
llm_method = gr.Dropdown(choices = ['Qwen', 'Qwen2', 'Linly', 'Gemini', 'ChatGLM', 'ChatGPT', 'GPT4Free', '直接回复 Direct Reply', 'Comming Soon!!!'], value = '直接回复 Direct Reply', label = 'LLM 模型选择')
llm_method.change(fn = llm_model_change, inputs=[llm_method], outputs = [llm_method])
return (source_image, voice, rate, volume, pitch,
am, voc, lang, male,
inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
tts_method, batch_size, character, talker_method, asr_method, llm_method)
def exmaple_setting(asr, text, character, talk , tts, voice, llm):
# 默认text的Example
examples = [
['Whisper-base', '应对压力最有效的方法是什么？', '女性角色', 'SadTalker', 'Edge-TTS', 'zh-CN-XiaoxiaoNeural', '直接回复 Direct Reply'],
['Whisper-tiny', '应对压力最有效的方法是什么？', '女性角色', 'SadTalker', 'PaddleTTS', 'None', '直接回复 Direct Reply'],
['Whisper-base', '应对压力最有效的方法是什么？', '女性角色', 'SadTalker', 'Edge-TTS', 'zh-CN-XiaoxiaoNeural', 'Qwen'],
['FunASR', '如何进行时间管理？','男性角色', 'SadTalker', 'Edge-TTS', 'zh-CN-YunyangNeural', 'Qwen'],
['Whisper-tiny', '为什么有些人选择使用纸质地图或寻求方向，而不是依赖GPS设备或智能手机应用程序？','女性角色', 'Wav2Lip', 'PaddleTTS', 'None', 'Qwen'],
]
with gr.Row(variant='panel'):
with gr.Column(variant='panel'):
gr.Markdown("## Test Examples")
gr.Examples(
examples = examples,
inputs = [asr, text, character, talk , tts, voice, llm],
)
def app():
with gr.Blocks(analytics_enabled=False, title = 'Linly-Talker') as inference:
gr.HTML(get_title("Linly 智能对话系统 (Linly-Talker) 文本/语音对话"))
with gr.Row(equal_height=False):
with gr.Column(variant='panel'):
(source_image, voice, rate, volume, pitch,
am, voc, lang, male,
inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
tts_method, batch_size, character, talker_method, asr_method, llm_method)= webui_setting()
with gr.Column(variant='panel'):
with gr.Tabs():
with gr.TabItem('对话'):
with gr.Group():
question_audio = gr.Audio(sources=['microphone','upload'], type="filepath", label = '语音对话')
input_text = gr.Textbox(label="输入文字/问题", lines=3)
asr_text = gr.Button('语音识别（语音对话后点击）')
asr_text.click(fn=Asr,inputs=[question_audio],outputs=[input_text])
# with gr.TabItem('SadTalker数字人参数设置'):
# with gr.Accordion("Advanced Settings",
# open=False):
# gr.Markdown("SadTalker: need help? please visit our [[best practice page](https://github.com/OpenTalker/SadTalker/blob/main/docs/best_practice.md)] for more detials")
# with gr.Column(variant='panel'):
# # width = gr.Slider(minimum=64, elem_id="img2img_width", maximum=2048, step=8, label="Manually Crop Width", value=512) # img2img_width
# # height = gr.Slider(minimum=64, elem_id="img2img_height", maximum=2048, step=8, label="Manually Crop Height", value=512) # img2img_width
# with gr.Row():
# pose_style = gr.Slider(minimum=0, maximum=45, step=1, label="Pose style", value=0) #
# exp_weight = gr.Slider(minimum=0, maximum=3, step=0.1, label="expression scale", value=1) #
# blink_every = gr.Checkbox(label="use eye blink", value=True)
# with gr.Row():
# size_of_image = gr.Radio([256, 512], value=256, label='face model resolution', info="use 256/512 model? 256 is faster") #
# preprocess_type = gr.Radio(['crop', 'resize','full'], value='full', label='preprocess', info="How to handle input image?")
# with gr.Row():
# is_still_mode = gr.Checkbox(label="Still Mode (fewer head motion, works with preprocess `full`)")
# facerender = gr.Radio(['facevid2vid'], value='facevid2vid', label='facerender', info="which face render?")
# with gr.Row():
# # batch_size = gr.Slider(label="batch size in generation", step=1, maximum=10, value=1)
# fps = gr.Slider(label='fps in generation', step=1, maximum=30, value =20)
# enhancer = gr.Checkbox(label="GFPGAN as Face enhancer(slow)")
with gr.Tabs():
with gr.TabItem('数字人问答'):
gen_video = gr.Video(label="生成视频", format="mp4", autoplay=False)
video_button = gr.Button("🎬 生成数字人视频", variant='primary')
video_button.click(fn=Talker_response,inputs=[question_audio, talker_method, input_text, voice, rate, volume, pitch,
am, voc, lang, male,
inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
tts_method, batch_size, character],outputs=[gen_video])
exmaple_setting(asr_method, input_text, character, talker_method, tts_method, voice, llm_method)
return inference
def app_multi():
with gr.Blocks(analytics_enabled=False, title = 'Linly-Talker') as inference:
gr.HTML(get_title("Linly 智能对话系统 (Linly-Talker) 多轮GPT对话"))
with gr.Row():
with gr.Column():
(source_image, voice, rate, volume, pitch,
am, voc, lang, male,
inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
tts_method, batch_size, character, talker_method, asr_method, llm_method)= webui_setting()
video = gr.Video(label = '数字人问答', scale = 0.5)
video_button = gr.Button("🎬 生成数字人视频（对话后）", variant = 'primary')
with gr.Column():
with gr.Tabs(elem_id="sadtalker_checkbox"):
with gr.TabItem('SadTalker数字人参数设置'):
with gr.Accordion("Advanced Settings",
open=False):
gr.Markdown("SadTalker: need help? please visit our [[best practice page](https://github.com/OpenTalker/SadTalker/blob/main/docs/best_practice.md)] for more detials")
with gr.Column(variant='panel'):
# width = gr.Slider(minimum=64, elem_id="img2img_width", maximum=2048, step=8, label="Manually Crop Width", value=512) # img2img_width
# height = gr.Slider(minimum=64, elem_id="img2img_height", maximum=2048, step=8, label="Manually Crop Height", value=512) # img2img_width
with gr.Row():
pose_style = gr.Slider(minimum=0, maximum=45, step=1, label="Pose style", value=0) #
exp_weight = gr.Slider(minimum=0, maximum=3, step=0.1, label="expression scale", value=1) #
blink_every = gr.Checkbox(label="use eye blink", value=True)
with gr.Row():
size_of_image = gr.Radio([256, 512], value=256, label='face model resolution', info="use 256/512 model? 256 is faster") #
preprocess_type = gr.Radio(['crop', 'resize','full', 'extcrop', 'extfull'], value='crop', label='preprocess', info="How to handle input image?")
with gr.Row():
is_still_mode = gr.Checkbox(label="Still Mode (fewer head motion, works with preprocess `full`)")
facerender = gr.Radio(['facevid2vid'], value='facevid2vid', label='facerender', info="which face render?")
with gr.Row():
fps = gr.Slider(label='fps in generation', step=1, maximum=30, value =20)
enhancer = gr.Checkbox(label="GFPGAN as Face enhancer(slow)")
with gr.Row():
with gr.Column(scale=3):
system_input = gr.Textbox(value=default_system, lines=1, label='System (设定角色)')
with gr.Column(scale=1):
modify_system = gr.Button("🛠️ 设置system并清除历史对话", scale=2)
system_state = gr.Textbox(value=default_system, visible=False)
chatbot = gr.Chatbot(height=400, show_copy_button=True)
with gr.Group():
question_audio = gr.Audio(sources=['microphone','upload'], type="filepath", label='语音对话', autoplay=False)
asr_text = gr.Button('🎤 语音识别（语音对话后点击）')
# 创建一个文本框组件，用于输入 prompt。
msg = gr.Textbox(label="Prompt/问题")
asr_text.click(fn=Asr,inputs=[question_audio],outputs=[msg])
with gr.Row():
clear_history = gr.Button("🧹 清除历史对话")
sumbit = gr.Button("🚀 发送", variant = 'primary')
# 设置按钮的点击事件。当点击时，调用上面定义的 函数，并传入用户的消息和聊天历史记录，然后更新文本框和聊天机器人组件。
sumbit.click(chat_response, inputs=[system_input, msg, chatbot],
outputs=[msg, chatbot])
# 点击后清空后端存储的聊天记录
clear_history.click(fn = clear_session, outputs = [msg, chatbot])
# 设置system并清除历史对话
modify_system.click(fn=modify_system_session,
inputs=[system_input],
outputs=[system_state, system_input, chatbot])
video_button.click(fn = human_response, inputs = [source_image, chatbot, question_audio, talker_method, voice, rate, volume, pitch,
am, voc, lang, male,
inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
tts_method, character,preprocess_type,
is_still_mode, enhancer, batch_size, size_of_image,
pose_style, facerender, exp_weight, blink_every, fps], outputs = [video])
exmaple_setting(asr_method, msg, character, talker_method, tts_method, voice, llm_method)
return inference
def app_img():
with gr.Blocks(analytics_enabled=False, title = 'Linly-Talker') as inference:
gr.HTML(get_title("Linly 智能对话系统 (Linly-Talker) 个性化角色互动"))
with gr.Row(equal_height=False):
with gr.Column(variant='panel'):
(source_image, voice, rate, volume, pitch,
am, voc, lang, male,
inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
tts_method, batch_size, character, talker_method, asr_method, llm_method)= webui_setting()
# driven_audio = 'answer.wav'
with gr.Column(variant='panel'):
with gr.Tabs():
with gr.TabItem('对话'):
with gr.Group():
question_audio = gr.Audio(sources=['microphone','upload'], type="filepath", label = '语音对话')
input_text = gr.Textbox(label="输入文字/问题", lines=3)
asr_text = gr.Button('语音识别（语音对话后点击）')
asr_text.click(fn=Asr,inputs=[question_audio],outputs=[input_text])
with gr.Tabs(elem_id="text_examples"):
gr.Markdown("## Text Examples")
examples = [
['应对压力最有效的方法是什么？'],
['如何进行时间管理？'],
['为什么有些人选择使用纸质地图或寻求方向，而不是依赖GPS设备或智能手机应用程序？'],
]
gr.Examples(
examples = examples,
inputs = [input_text],
)
with gr.Tabs(elem_id="sadtalker_checkbox"):
with gr.TabItem('SadTalker数字人参数设置'):
with gr.Accordion("Advanced Settings",
open=False):
gr.Markdown("SadTalker: need help? please visit our [[best practice page](https://github.com/OpenTalker/SadTalker/blob/main/docs/best_practice.md)] for more detials")
with gr.Column(variant='panel'):
# width = gr.Slider(minimum=64, elem_id="img2img_width", maximum=2048, step=8, label="Manually Crop Width", value=512) # img2img_width
# height = gr.Slider(minimum=64, elem_id="img2img_height", maximum=2048, step=8, label="Manually Crop Height", value=512) # img2img_width
with gr.Row():
pose_style = gr.Slider(minimum=0, maximum=45, step=1, label="Pose style", value=0) #
exp_weight = gr.Slider(minimum=0, maximum=3, step=0.1, label="expression scale", value=1) #
blink_every = gr.Checkbox(label="use eye blink", value=True)
with gr.Row():
size_of_image = gr.Radio([256, 512], value=256, label='face model resolution', info="use 256/512 model? 256 is faster") #
preprocess_type = gr.Radio(['crop', 'resize','full', 'extcrop', 'extfull'], value='crop', label='preprocess', info="How to handle input image?")
with gr.Row():
is_still_mode = gr.Checkbox(label="Still Mode (fewer head motion, works with preprocess `full`)")
facerender = gr.Radio(['facevid2vid'], value='facevid2vid', label='facerender', info="which face render?")
with gr.Row():
fps = gr.Slider(label='fps in generation', step=1, maximum=30, value =20)
enhancer = gr.Checkbox(label="GFPGAN as Face enhancer(slow)")
with gr.Tabs(elem_id="sadtalker_genearted"):
gen_video = gr.Video(label="数字人视频", format="mp4")
submit = gr.Button('🎬 生成数字人视频', elem_id="sadtalker_generate", variant='primary')
submit.click(
fn=Talker_response_img,
inputs=[question_audio,
talker_method,
input_text,
voice, rate, volume, pitch,
am, voc, lang, male,
inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
tts_method,
source_image,
preprocess_type,
is_still_mode,
enhancer,
batch_size,
size_of_image,
pose_style,
facerender,
exp_weight,
blink_every,
fps],
outputs=[gen_video]
)
with gr.Row():
examples = [
[
'examples/source_image/full_body_2.png', 'SadTalker',
'crop',
False,
False
],
[
'examples/source_image/full_body_1.png', 'SadTalker',
'full',
True,
False
],
[
'examples/source_image/full4.jpeg', 'SadTalker',
'crop',
False,
True
],
]
gr.Examples(examples=examples,
inputs=[
source_image, talker_method,
preprocess_type,
is_still_mode,
enhancer],
outputs=[gen_video],
# cache_examples=True,
)
return inference
def app_vits():
with gr.Blocks(analytics_enabled=False, title = 'Linly-Talker') as inference:
gr.HTML(get_title("Linly 智能对话系统 (Linly-Talker) 语音克隆"))
with gr.Row(equal_height=False):
with gr.Column(variant='panel'):
(source_image, voice, rate, volume, pitch,
am, voc, lang, male,
inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
tts_method, batch_size, character, talker_method, asr_method, llm_method)= webui_setting()
with gr.Column(variant='panel'):
with gr.Tabs():
with gr.TabItem('对话'):
with gr.Group():
question_audio = gr.Audio(sources=['microphone','upload'], type="filepath", label = '语音对话')
input_text = gr.Textbox(label="输入文字/问题", lines=3)
asr_text = gr.Button('语音识别（语音对话后点击）')
asr_text.click(fn=Asr,inputs=[question_audio],outputs=[input_text])
with gr.Tabs():
with gr.TabItem('数字人问答'):
gen_video = gr.Video(label="数字人视频", format="mp4", autoplay=False)
video_button = gr.Button("🎬 生成数字人视频", variant='primary')
video_button.click(fn=Talker_response,inputs=[question_audio, talker_method, input_text, voice, rate, volume, pitch, am, voc, lang, male,
inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
tts_method, batch_size, character],outputs=[gen_video])
exmaple_setting(asr_method, input_text, character, talker_method, tts_method, voice, llm_method)
return inference
def app_talk():
with gr.Blocks(analytics_enabled=False, title = 'Linly-Talker') as inference:
gr.HTML(get_title("Linly 智能对话系统 (Linly-Talker) 数字人播报"))
with gr.Row(equal_height=False):
with gr.Column(variant='panel'):
with gr.Tabs():
with gr.Tab("图片人物"):
source_image = gr.Image(label='Source image', type = 'filepath')
with gr.Tab("视频人物"):
source_video = gr.Video(label="Source video")
(_, voice, rate, volume, pitch,
am, voc, lang, male,
inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
tts_method, batch_size, character, talker_method, asr_method, llm_method)= webui_setting()
with gr.Column(variant='panel'):
with gr.Tabs():
with gr.TabItem('对话'):
with gr.Group():
question_audio = gr.Audio(sources=['microphone','upload'], type="filepath", label = '语音对话')
input_text = gr.Textbox(label="输入文字/问题", lines=3)
asr_text = gr.Button('语音识别（语音对话后点击）')
asr_text.click(fn=Asr,inputs=[question_audio],outputs=[input_text])
with gr.Tabs():
with gr.TabItem('SadTalker数字人参数设置'):
with gr.Accordion("Advanced Settings",
open=False):
gr.Markdown("SadTalker: need help? please visit our [[best practice page](https://github.com/OpenTalker/SadTalker/blob/main/docs/best_practice.md)] for more detials")
with gr.Column(variant='panel'):
# width = gr.Slider(minimum=64, elem_id="img2img_width", maximum=2048, step=8, label="Manually Crop Width", value=512) # img2img_width
# height = gr.Slider(minimum=64, elem_id="img2img_height", maximum=2048, step=8, label="Manually Crop Height", value=512) # img2img_width
with gr.Row():
pose_style = gr.Slider(minimum=0, maximum=45, step=1, label="Pose style", value=0) #
exp_weight = gr.Slider(minimum=0, maximum=3, step=0.1, label="expression scale", value=1) #
blink_every = gr.Checkbox(label="use eye blink", value=True)
with gr.Row():
size_of_image = gr.Radio([256, 512], value=256, label='face model resolution', info="use 256/512 model? 256 is faster") #
preprocess_type = gr.Radio(['crop', 'resize','full'], value='full', label='preprocess', info="How to handle input image?")
with gr.Row():
is_still_mode = gr.Checkbox(label="Still Mode (fewer head motion, works with preprocess `full`)")
facerender = gr.Radio(['facevid2vid'], value='facevid2vid', label='facerender', info="which face render?")
with gr.Row():
# batch_size = gr.Slider(label="batch size in generation", step=1, maximum=10, value=1)
fps = gr.Slider(label='fps in generation', step=1, maximum=30, value =20)
enhancer = gr.Checkbox(label="GFPGAN as Face enhancer(slow)")
with gr.Tabs():
gen_video = gr.Video(label="数字人视频", format="mp4")
video_button = gr.Button('🎬 生成数字人视频', elem_id="sadtalker_generate", variant='primary')
video_button.click(fn=Talker_Say,inputs=[preprocess_type, is_still_mode, enhancer, batch_size, size_of_image,
pose_style, facerender, exp_weight, blink_every, fps,
source_image, source_video, question_audio, talker_method, input_text, voice, rate, volume, pitch, am, voc, lang, male,
inp_ref, prompt_text, prompt_language, text_language, how_to_cut, use_mic_voice,
tts_method, character],outputs=[gen_video])
with gr.Row():
with gr.Column(variant='panel'):
gr.Markdown("## Test Examples")
gr.Examples(
examples = [
[
'examples/source_image/full_body_2.png',
'应对压力最有效的方法是什么？',
],
[
'examples/source_image/full_body_1.png',
'如何进行时间管理？',
],
[
'examples/source_image/full3.png',
'为什么有些人选择使用纸质地图或寻求方向，而不是依赖GPS设备或智能手机应用程序？',
],
],
fn = Talker_Say,
inputs = [source_image, input_text],
)
return inference
def load_musetalk_model():
gr.Warning("若显存不足，可能会导致模型加载失败，可以尝试使用其他摸型或者换其他设备尝试。")
gr.Info("MuseTalk模型导入中...")
musetalker.init_model()
gr.Info("MuseTalk模型导入成功")
return "MuseTalk模型导入成功"
def musetalk_prepare_material(source_video, bbox_shift):
if musetalker.load is False:
gr.Warning("请先加载MuseTalk模型后重新上传文件")
return source_video, None
return musetalker.prepare_material(source_video, bbox_shift)
def app_muse():
with gr.Blocks(analytics_enabled=False, title = 'Linly-Talker') as inference:
gr.HTML(get_title("Linly 智能对话系统 (Linly-Talker) MuseTalker数字人实时对话"))
with gr.Row(equal_height=False):
with gr.Column(variant='panel'):
with gr.TabItem('MuseV Video'):
gr.Markdown("MuseV: need help? please visit MuseVDemo to generate Video https://huggingface.co/spaces/AnchorFake/MuseVDemo")
with gr.Row():
source_video = gr.Video(label="Reference Video",sources=['upload'])
gr.Markdown("BBox_shift 推荐值下限，在生成初始结果后生成相应的 bbox 范围。如果结果不理想，可以根据该参考值进行调整。\n一般来说，在我们的实验观察中，我们发现正值（向下半部分移动）通常会增加嘴巴的张开度，而负值（向上半部分移动）通常会减少嘴巴的张开度。然而，需要注意的是，这并不是绝对的规则，用户可能需要根据他们的具体需求和期望效果来调整该参数。")
with gr.Row():
bbox_shift = gr.Number(label="BBox_shift value, px", value=0)
bbox_shift_scale = gr.Textbox(label="bbox_shift_scale",
value="",interactive=False)
load_musetalk = gr.Button("加载MuseTalk模型(传入视频前先加载)", variant='primary')
load_musetalk.click(fn=load_musetalk_model, outputs=bbox_shift_scale)
# (_, voice, rate, volume, pitch,