Skip to content

Commit 3e512d5

Browse files
x4nth055x4nth055
committed
add speech recognition tutorial that covers 3 methods: speechrecognition library, openai whisper API & transformers
1 parent 97976fe commit 3e512d5

10 files changed

+313
-0
lines changed

‎machine-learning/asr/7601-291468-0006.wav

1.07 MB
Binary file not shown.

‎machine-learning/asr/OSR_us_000_0060_8k.wav

910 KB
Binary file not shown.

‎machine-learning/asr/arabic-audio.wav

1.4 MB
Binary file not shown.

‎machine-learning/asr/cual-es-la-fecha-cumple.mp3

23.8 KB
Binary file not shown.

‎machine-learning/asr/requirements.txt

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
transformers==4.28.1
2+
soundfile
3+
sentencepiece
4+
torchaudio
5+
pydub
6+
openai
7+
SpeechRecognition
8+
pyaudio

‎machine-learning/asr/speechrecognition.py

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
# importing libraries
2+
import speech_recognition as sr
3+
import os
4+
from pydub import AudioSegment
5+
from pydub.silence import split_on_silence
6+
7+
# create a speech recognition object
8+
r = sr.Recognizer()
9+
10+
# a function to recognize speech in the audio file
11+
# so that we don't repeat ourselves in in other functions
12+
def transcribe_audio(path):
13+
# use the audio file as the audio source
14+
with sr.AudioFile(path) as source:
15+
audio_listened = r.record(source)
16+
# try converting it to text
17+
text = r.recognize_google(audio_listened)
18+
return text
19+
20+
# a function that splits the audio file into chunks on silence
21+
# and applies speech recognition
22+
def get_large_audio_transcription_on_silence(path):
23+
"""
24+
Splitting the large audio file into chunks
25+
and apply speech recognition on each of these chunks
26+
"""
27+
# open the audio file using pydub
28+
sound = AudioSegment.from_file(path)
29+
# split audio sound where silence is 700 miliseconds or more and get chunks
30+
chunks = split_on_silence(sound,
31+
# experiment with this value for your target audio file
32+
min_silence_len = 500,
33+
# adjust this per requirement
34+
silence_thresh = sound.dBFS-14,
35+
# keep the silence for 1 second, adjustable as well
36+
keep_silence=500,
37+
)
38+
folder_name = "audio-chunks"
39+
# create a directory to store the audio chunks
40+
if not os.path.isdir(folder_name):
41+
os.mkdir(folder_name)
42+
whole_text = ""
43+
# process each chunk
44+
for i, audio_chunk in enumerate(chunks, start=1):
45+
# export audio chunk and save it in
46+
# the `folder_name` directory.
47+
chunk_filename = os.path.join(folder_name, f"chunk{i}.wav")
48+
audio_chunk.export(chunk_filename, format="wav")
49+
# recognize the chunk
50+
with sr.AudioFile(chunk_filename) as source:
51+
audio_listened = r.record(source)
52+
# try converting it to text
53+
try:
54+
text = r.recognize_google(audio_listened)
55+
except sr.UnknownValueError as e:
56+
print("Error:", str(e))
57+
else:
58+
text = f"{text.capitalize()}. "
59+
print(chunk_filename, ":", text)
60+
whole_text += text
61+
# return the text for all chunks detected
62+
return whole_text
63+
64+
65+
# a function that splits the audio file into fixed interval chunks
66+
# and applies speech recognition
67+
def get_large_audio_transcription_fixed_interval(path, minutes=5):
68+
"""
69+
Splitting the large audio file into fixed interval chunks
70+
and apply speech recognition on each of these chunks
71+
"""
72+
# open the audio file using pydub
73+
sound = AudioSegment.from_file(path)
74+
# split the audio file into chunks
75+
chunk_length_ms = int(1000 * 60 * minutes) # convert to milliseconds
76+
chunks = [sound[i:i + chunk_length_ms] for i in range(0, len(sound), chunk_length_ms)]
77+
folder_name = "audio-fixed-chunks"
78+
# create a directory to store the audio chunks
79+
if not os.path.isdir(folder_name):
80+
os.mkdir(folder_name)
81+
whole_text = ""
82+
# process each chunk
83+
for i, audio_chunk in enumerate(chunks, start=1):
84+
# export audio chunk and save it in
85+
# the `folder_name` directory.
86+
chunk_filename = os.path.join(folder_name, f"chunk{i}.wav")
87+
audio_chunk.export(chunk_filename, format="wav")
88+
# recognize the chunk
89+
with sr.AudioFile(chunk_filename) as source:
90+
audio_listened = r.record(source)
91+
# try converting it to text
92+
try:
93+
text = r.recognize_google(audio_listened)
94+
except sr.UnknownValueError as e:
95+
print("Error:", str(e))
96+
else:
97+
text = f"{text.capitalize()}. "
98+
print(chunk_filename, ":", text)
99+
whole_text += text
100+
# return the text for all chunks detected
101+
return whole_text
102+
103+
104+
if __name__ == "__main__":
105+
print(get_large_audio_transcription_on_silence("7601-291468-0006.wav"))

‎machine-learning/asr/transformers_whisper.py

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
from transformers import WhisperProcessor, WhisperForConditionalGeneration
2+
import torch
3+
import torchaudio
4+
5+
device = "cuda:0" if torch.cuda.is_available() else "cpu"
6+
# whisper_model_name = "openai/whisper-tiny.en" # English-only, ~ 151 MB
7+
# whisper_model_name = "openai/whisper-base.en" # English-only, ~ 290 MB
8+
# whisper_model_name = "openai/whisper-small.en" # English-only, ~ 967 MB
9+
# whisper_model_name = "openai/whisper-medium.en" # English-only, ~ 3.06 GB
10+
whisper_model_name = "openai/whisper-tiny" # multilingual, ~ 151 MB
11+
# whisper_model_name = "openai/whisper-base" # multilingual, ~ 290 MB
12+
# whisper_model_name = "openai/whisper-small" # multilingual, ~ 967 MB
13+
# whisper_model_name = "openai/whisper-medium" # multilingual, ~ 3.06 GB
14+
# whisper_model_name = "openai/whisper-large-v2" # multilingual, ~ 6.17 GB
15+
16+
# load the model and the processor
17+
whisper_processor = WhisperProcessor.from_pretrained(whisper_model_name)
18+
whisper_model = WhisperForConditionalGeneration.from_pretrained(whisper_model_name).to(device)
19+
20+
def load_audio(audio_path):
21+
"""Load the audio file & convert to 16,000 sampling rate"""
22+
# load our wav file
23+
speech, sr = torchaudio.load(audio_path)
24+
resampler = torchaudio.transforms.Resample(sr, 16000)
25+
speech = resampler(speech)
26+
return speech.squeeze()
27+
28+
29+
def get_transcription_whisper(audio_path, model, processor, language="english", skip_special_tokens=True):
30+
# resample from whatever the audio sampling rate to 16000
31+
speech = load_audio(audio_path)
32+
# get the input features from the audio file
33+
input_features = processor(speech, return_tensors="pt", sampling_rate=16000).input_features.to(device)
34+
# get the forced decoder ids
35+
forced_decoder_ids = processor.get_decoder_prompt_ids(language=language, task="transcribe")
36+
# print(forced_decoder_ids)
37+
# generate the transcription
38+
predicted_ids = model.generate(input_features, forced_decoder_ids=forced_decoder_ids)
39+
# decode the predicted ids
40+
transcription = processor.batch_decode(predicted_ids, skip_special_tokens=skip_special_tokens)[0]
41+
return transcription
42+
43+
44+
if __name__ == "__main__":
45+
english_transcription = get_transcription_whisper("7601-291468-0006.wav",
46+
whisper_model,
47+
whisper_processor,
48+
language="english",
49+
skip_special_tokens=True)
50+
print("English transcription:", english_transcription)
51+
arabic_transcription = get_transcription_whisper("arabic-audio.wav",
52+
whisper_model,
53+
whisper_processor,
54+
language="arabic",
55+
skip_special_tokens=True)
56+
print("Arabic transcription:", arabic_transcription)
57+
spanish_transcription = get_transcription_whisper("cual-es-la-fecha-cumple.mp3",
58+
whisper_model,
59+
whisper_processor,
60+
language="spanish",
61+
skip_special_tokens=True)
62+
print("Spanish transcription:", spanish_transcription)
63+

‎machine-learning/asr/transformers_whisper_long.py

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
2+
from transformers import pipeline
3+
import torch
4+
import torchaudio
5+
6+
device = "cuda:0" if torch.cuda.is_available() else "cpu"
7+
# whisper_model_name = "openai/whisper-tiny.en" # English-only, ~ 151 MB
8+
# whisper_model_name = "openai/whisper-base.en" # English-only, ~ 290 MB
9+
# whisper_model_name = "openai/whisper-small.en" # English-only, ~ 967 MB
10+
# whisper_model_name = "openai/whisper-medium.en" # English-only, ~ 3.06 GB
11+
whisper_model_name = "openai/whisper-tiny" # multilingual, ~ 151 MB
12+
# whisper_model_name = "openai/whisper-base" # multilingual, ~ 290 MB
13+
# whisper_model_name = "openai/whisper-small" # multilingual, ~ 967 MB
14+
# whisper_model_name = "openai/whisper-medium" # multilingual, ~ 3.06 GB
15+
# whisper_model_name = "openai/whisper-large-v2" # multilingual, ~ 6.17 GB
16+
17+
def load_audio(audio_path):
18+
"""Load the audio file & convert to 16,000 sampling rate"""
19+
# load our wav file
20+
speech, sr = torchaudio.load(audio_path)
21+
resampler = torchaudio.transforms.Resample(sr, 16000)
22+
speech = resampler(speech)
23+
return speech.squeeze()
24+
25+
26+
def get_long_transcription_whisper(audio_path, pipe, return_timestamps=True,
27+
chunk_length_s=10, stride_length_s=1):
28+
"""Get the transcription of a long audio file using the Whisper model"""
29+
return pipe(load_audio(audio_path).numpy(), return_timestamps=return_timestamps,
30+
chunk_length_s=chunk_length_s, stride_length_s=stride_length_s)
31+
32+
33+
if __name__ == "__main__":
34+
# initialize the pipeline
35+
pipe = pipeline("automatic-speech-recognition",
36+
model=whisper_model_name, device=device)
37+
# get the transcription of a sample long audio file
38+
output = get_long_transcription_whisper(
39+
"7601-291468-0006.wav", pipe, chunk_length_s=10, stride_length_s=2)
40+
print(f"Transcription: {output}")
41+
print("="*50)
42+
for chunk in output["chunks"]:
43+
# print the timestamp and the text
44+
print(chunk["timestamp"], ":", chunk["text"])

‎machine-learning/asr/whisper_api.py

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
import openai
2+
3+
# API key
4+
openai.api_key = "<API_KEY>"
5+
6+
def get_openai_api_transcription(audio_filename):
7+
# open the audio file
8+
with open(audio_filename, "rb") as audio_file:
9+
# transcribe the audio file
10+
transcription = openai.Audio.transcribe("whisper-1", audio_file) # whisper-1 is the model name
11+
return transcription
12+
13+
14+
if __name__ == "__main__":
15+
transcription = get_openai_api_transcription("7601-291468-0006.wav")
16+
print(transcription.get("text"))

‎machine-learning/asr/whisper_api_long.py

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
from pydub.silence import split_on_silence
2+
from pydub import AudioSegment
3+
from whisper_api import get_openai_api_transcription
4+
import os
5+
6+
# a function that splits the audio file into chunks
7+
# and applies speech recognition
8+
def get_large_audio_transcription_on_silence(path):
9+
"""
10+
Splitting the large audio file into chunks
11+
and apply speech recognition on each of these chunks
12+
"""
13+
# open the audio file using pydub
14+
sound = AudioSegment.from_file(path)
15+
# split audio sound where silence is 700 miliseconds or more and get chunks
16+
chunks = split_on_silence(sound,
17+
# experiment with this value for your target audio file
18+
min_silence_len = 500,
19+
# adjust this per requirement
20+
silence_thresh = sound.dBFS-14,
21+
# keep the silence for 1 second, adjustable as well
22+
keep_silence=500,
23+
)
24+
folder_name = "audio-chunks"
25+
# create a directory to store the audio chunks
26+
if not os.path.isdir(folder_name):
27+
os.mkdir(folder_name)
28+
whole_text = ""
29+
# process each chunk
30+
for i, audio_chunk in enumerate(chunks, start=1):
31+
# export audio chunk and save it in
32+
# the `folder_name` directory.
33+
chunk_filename = os.path.join(folder_name, f"chunk{i}.wav")
34+
audio_chunk.export(chunk_filename, format="wav")
35+
# recognize the chunk
36+
transcription = get_openai_api_transcription(chunk_filename)
37+
print(f"{chunk_filename}: {transcription.get('text')}")
38+
whole_text += " " + transcription.get("text")
39+
# return the text for all chunks detected
40+
return whole_text
41+
42+
43+
# a function that splits the audio file into fixed interval chunks
44+
# and applies speech recognition
45+
def get_large_audio_transcription_fixed_interval(path, minutes=5):
46+
"""
47+
Splitting the large audio file into 5-minute chunks
48+
and apply speech recognition on each of these chunks
49+
"""
50+
# open the audio file using pydub
51+
sound = AudioSegment.from_file(path)
52+
# split the audio file into chunks
53+
chunk_length_ms = int(1000 * 60 * minutes) # convert to milliseconds
54+
chunks = [sound[i:i + chunk_length_ms] for i in range(0, len(sound), chunk_length_ms)]
55+
folder_name = "audio-fixed-chunks"
56+
# create a directory to store the audio chunks
57+
if not os.path.isdir(folder_name):
58+
os.mkdir(folder_name)
59+
whole_text = ""
60+
# process each chunk
61+
for i, audio_chunk in enumerate(chunks, start=1):
62+
# export audio chunk and save it in
63+
# the `folder_name` directory.
64+
chunk_filename = os.path.join(folder_name, f"chunk{i}.wav")
65+
audio_chunk.export(chunk_filename, format="wav")
66+
# recognize the chunk
67+
transcription = get_openai_api_transcription(chunk_filename)
68+
print(f"{chunk_filename}: {transcription.get('text')}")
69+
whole_text += " " + transcription.get("text")
70+
# return the text for all chunks detected
71+
return whole_text
72+
73+
74+
75+
if __name__ == "__main__":
76+
# print("\nFull text:", get_large_audio_transcription_fixed_interval("032.mp3", minutes=1))
77+
print("\nFull text:", get_large_audio_transcription_on_silence("7601-291468-0006.wav"))

0 commit comments

Comments
 (0)