Spaces:
Runtime error
Runtime error
File size: 6,484 Bytes
08bc68d 4376ec7 08bc68d 4376ec7 08bc68d 4376ec7 694ab9a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 | import os
import crepe.crepe as crepe
import matplotlib.pyplot as plt
import librosa
import librosa.display
import numpy as np
import torch
import torchaudio
import pretty_midi
import gradio as gr
from scipy.io import wavfile
# Helper functions
def dynamic_range_compression_torch(x, C=1, clip_val=1e-5):
return torch.log(torch.clamp(x, min=clip_val) * C)
def dynamic_range_decompression_torch(x, C=1):
return torch.exp(x) / C
def spectral_normalize_torch(magnitudes):
output = dynamic_range_compression_torch(magnitudes)
return output
def spectral_de_normalize_torch(magnitudes):
output = dynamic_range_decompression_torch(magnitudes)
return output
def get_melspec(y: np.ndarray, sr: int = 16000, n_mels: int = 64,
window_length: int = 1024, hop_length: int = 160, fmax: int = 8000, fmin: int = 0) -> torch.tensor:
y = np.pad(y, (int((window_length - hop_length) / 2), int((window_length - hop_length) / 2)), mode="reflect")
S = librosa.feature.melspectrogram(y=y, sr=sr, n_mels=n_mels,
n_fft=window_length,
hop_length=hop_length,
win_length=window_length, window='hann',
fmin=fmin, fmax=fmax,
center=False, pad_mode="reflect")
S_norm = spectral_normalize_torch(torch.tensor(S))
return S_norm
def get_f0_melspec(frequency: np.ndarray, confidence: np.ndarray = None,
n_mels: int = 64, fmax: int = 8000, fmin: int = 0) -> torch.tensor:
n_frames = len(frequency)
S = np.zeros((n_mels, n_frames))
mel_bin_frequencies = librosa.core.mel_frequencies(n_mels=n_mels + 1, fmin=fmin, fmax=fmax, htk=False)
mel_band_indices = [
np.clip(np.searchsorted(mel_bin_frequencies, mel_f, side='right') - 1, 0, n_mels - 1) if mel_f is not None else None
for mel_f in frequency
]
for t, mel_band in enumerate(mel_band_indices):
if mel_band is None:
continue
if confidence is not None:
S[mel_band, t] = confidence[t]
else:
S[mel_band, t] = 1.0
S_norm = spectral_normalize_torch(torch.tensor(S))
return S_norm
def midi_to_frequency(midi_file, sr=16000, duration=10.24, hop_length=160):
midi_data = pretty_midi.PrettyMIDI(midi_file)
max_len = int(duration * sr // hop_length)
frequency = np.full(max_len, None)
for instrument in midi_data.instruments:
if instrument.is_drum:
continue
for note in instrument.notes:
start_frame = int(note.start * sr // hop_length)
end_frame = int(note.end * sr // hop_length)
frequency[start_frame:end_frame] = note.pitch
frequency = np.array([pretty_midi.note_number_to_hz(note) if note is not None else None for note in frequency])
return frequency
def get_item(filepath: str, config: dict, input_type: str = 'midi'):
max_len = int(config["audio_length"] * config["sr"])
if input_type == 'audio':
if filepath.endswith('.flac'):
audio, sr = librosa.load(filepath)
elif filepath.endswith('.wav'):
sr, audio = wavfile.read(filepath)
else:
raise ValueError('Unsupported file format')
if audio.ndim > 1:
audio = np.mean(audio, axis=1)
if sr != config["sr"]:
audio = torchaudio.functional.resample(torch.Tensor(audio), sr, config['sr']).numpy()
sr = config["sr"]
hop_length = config["hop_length"]
if len(audio) > max_len:
audio = audio[:max_len]
elif len(audio) < max_len:
pad_length = max_len - len(audio)
audio = np.pad(audio, (0, pad_length), mode="constant")
S_waveform = torch.tensor(audio).unsqueeze(0)
S = get_melspec(audio, sr=config['sr'], n_mels=config['n_mels'], window_length=config['window_length'],
hop_length=config['hop_length'], fmax=config['fmax'], fmin=config['fmin'])
time, frequency, confidence, activation = crepe.predict(audio, sr, viterbi=True)
if not config['use_confidence']:
frequency = np.array([f if c >= config['confidence_threshold'] else None for f, c in zip(frequency, confidence)])
if len(frequency) < S.shape[1]:
if config['use_confidence']:
assert len(confidence) == len(frequency)
confidence = np.pad(confidence, (0, S.shape[1] - len(confidence)), mode="constant", constant_values=None)
frequency = np.pad(frequency, (0, S.shape[1] - len(frequency)), mode="constant", constant_values=None)
elif len(frequency) > S.shape[1]:
frequency = frequency[:S.shape[1]]
if config['use_confidence']:
confidence = confidence[:S.shape[1]]
elif input_type == 'midi':
confidence = 1.0
frequency = midi_to_frequency(filepath, sr=config["sr"], duration=config["audio_length"], hop_length=config["hop_length"])
S_f0 = get_f0_melspec(frequency, confidence=confidence if config['use_confidence'] else None,
n_mels=config['n_mels'], fmax=config['fmax'], fmin=config['fmin'])
item_dict = {
'f0_condition': S_f0
}
return item_dict
def spec_show(S, sr):
fig, ax = plt.subplots(figsize=(6, 2))
img = librosa.display.specshow(S, sr=sr, ax=ax)
buf = BytesIO()
fig.savefig(buf, format='png')
buf.seek(0)
plt.close(fig)
return buf
config_AudioLDM = {
"sr": 16000,
"audio_length": 10.24,
"n_mels": 64,
"window_length": 1024,
"hop_length": 160,
"fmax": 8000,
"fmin": 0,
"use_confidence": False,
"confidence_threshold": 0.8,
}
cfg = config_AudioLDM
def process_file(file):
if file.name.endswith('.wav') or file.name.endswith('.flac'):
item_dict = get_item(file.name, config=cfg, input_type='audio')
elif file.name.endswith('.mid') or file.name.endswith('.midi'):
item_dict = get_item(file.name, config=cfg, input_type='midi')
else:
return "Unsupported file format"
f0_condition = item_dict['f0_condition']
return spec_show(f0_condition.numpy(), cfg["sr"])
demo = gr.Interface(fn=process_file, inputs=gr.inputs.File(), outputs="image", title="Pitch ControlNet")
demo.launch() |