Hugging Face's logo
Hugging Face
Models
Datasets
Spaces
Buckets
new
Docs
Pricing


Spaces:
wasmdashai
/
wasm-dataset


like
8

App
Files
Community
Settings
wasm-dataset
/
app.py

wasmdashai's picture
wasmdashai
Update app.py
9ee1a46
verified
almost 2 years ago
Raw

Download with hf CLI

Copy download link
History
Blame
Edit
Delete
22.3 kB
import gradio as gr
import pandas as pd
import numpy as np
from df.enhance import enhance, init_df, load_audio, save_audio
import time
import os
import gradio as gr
import re
from gradio.themes.base import Base
from datasets import load_dataset
from datasets import Dataset,DatasetDict
import librosa
import torch

model_enhance, df_state, _ = init_df()
def Read_DataSet(link):
dataset = load_dataset(link,token=os.environ.get("auth_acess_data"))
df = dataset["train"].to_pandas()
return df




def remove_nn(wav, sample_rate=16000):

audio=librosa.resample(wav,orig_sr=sample_rate,target_sr=df_state.sr(),)

audio=torch.tensor([audio])
# audio, _ = load_audio('full_generation.wav', sr=df_state.sr())
print(audio)

enhanced = enhance(model_enhance, df_state, audio)
print(enhanced)
# save_audio("enhanced.wav", enhanced, df_state.sr())
audiodata=librosa.resample(enhanced[0].numpy(),orig_sr=df_state.sr(),target_sr=sample_rate)

return 16000, audiodata/np.max(audiodata)




class DataViewerApp:
def __init__(self,df):
#df=Read_DataSet(link)
self.df=df
# self.df1=df
self.data =self.df[['text','speaker_id','secs','flag']]
self.dataa =self.df[['text','speaker_id','secs','flag']]
self.sdata =self.df['audio'].to_list() # Separate audio data storage
self.current_page = 0
self.current_selected = -1
self.speaker_id= -1
class Seafoam(Base):
pass
self.seafoam = Seafoam()

#self.data =df[['text','speaker_id']]
#self.sdata = df['audio'].to_list() # Separate audio data storage
#self.current_page = 0
#self.current_selected = -1
def set1(self,df):
self.data =df[['text','speaker_id','secs','flag']]
self.sdata =df['audio'].to_list()
return self.get_page_data(self.current_page)
def settt(self,df):
self.df=pd.DataFrame()
self.data =pd.DataFrame()
self.sdata =[]
self.df=df
self.data =df[['text','speaker_id','secs','flag']]
self.dataa =df[['text','speaker_id','secs','flag']]
self.sdata =df['audio'].to_list()
self.current_page = 0
self.current_selected =1
self.speaker_id= -1
return self.data
def clear(self,text):
text=re.sub(r'[a-zA-Z]', '', text)
return text
def clearenglish(self):
for i in range(len(self.df)):
x=self.clear(self.df['text'][i])
x1=self.df['text'][i]
if x!=x1:
self.df.drop(i, inplace=True)

self.df.reset_index(drop=True, inplace=True)
return self.settt(self.df)
def splitt(self,link,num):
df=download_youtube_video(link,num)
v=self.settt(df)
return self.get_page_data(self.current_page),len(v)
def getdataset(self,link):
self.link_dataset=link
df=Read_DataSet(link)
v=self.settt(df)
return self.get_page_data(self.current_page),len(v),self.link_dataset
def remove_hamza_from_alif_and_symbols(self,text):
text = re.sub(r"[أإآ]", "ا", text)
text = re.sub(r"ٱ", "ا", text)
text = re.sub(r"[_\-\+\,\(\)]", " ", text)
text = re.sub(r"\d", " ", text)
return text
def save_row(self, text,data_oudio):
if text!="" :
row = self.data.iloc[self.current_selected]
row['text'] = text
row['flag']=1
self.data.iloc[self.current_selected] = row
sr,audio=data_oudio
if sr!=16000:
audio=audio.astype(np.float32)
audio/=np.max(np.abs(audio))
audio=librosa.resample(audio,orig_sr=sr,target_sr=16000)





self.sdata[self.current_selected] = audio
self.df['text'][self.current_selected] =text
self.df['audio'][self.current_selected] = audio
self.df['flag'][self.current_selected] =1
return self.get_page_data(self.current_page),None,""
def GetDataset_2(self,filename,ds=1.5):
audios_data = []
audios_samplerate = []
num_specker=[]
texts=[]
secs=[]

audiodata,samplerate = librosa.load(filename, sr=16000) # Removed extra indent here
audios_data.append(audiodata*ds)
audios_samplerate.append(samplerate)
texts.append(filename.replace('.wav',''))
secs.append(round(len(audiodata)/samplerate,2))
df = pd.DataFrame()
df['secs'] = secs
df['audio'] = audios_data
df['samplerate'] = audios_samplerate
df['text'] =os.path.splitext(os.path.basename(filename))[0]
df['speaker_id'] =self.speaker_id
df['_speaker_id'] =self.speaker_id
df['flag']=1
df = df[['text','audio','samplerate','secs','speaker_id','_speaker_id','flag']]
self.df = pd.concat([self.df, df], axis=0, ignore_index=True)
self.data =self.df[['text','speaker_id','secs','flag']]
self.sdata =self.df['audio'].to_list()

return self.get_page_data(self.current_page)
def trim_audio(self, text,data_oudio):
if text!="" :
audios_data = []
audios_samplerate = []
sr,audio=data_oudio
audio=audio.astype(np.float32)
audio/=np.max(np.abs(audio))
audio=librosa.resample(audio,orig_sr=sr,target_sr=16000)
audios_data.append(audio)
secs=round(len(audios_data)/16000,2)
audios_samplerate.append(16000)
df = pd.DataFrame()
df['secs'] = secs
df['audio'] =[ audio]
df['samplerate'] = 16000
df['text'] =text
df['speaker_id'] =self.speaker_id
df['_speaker_id'] =self.speaker_id
df['flag']=1
df = df[['text','audio','samplerate','secs','speaker_id','_speaker_id','flag']]
self.df = pd.concat([self.df, df], axis=0, ignore_index=True)
self.data =self.df[['text','speaker_id','secs','flag']]
self.sdata =self.df['audio'].to_list()
return self.get_page_data(self.current_page),None,""
def order_data(self):
self.df[['text','speaker_id','secs','flag']]=self.data
self.df=self.df.sort_values(by=['flag'], ascending=False)
vv=self.settt(self.df)
return vv
def connect_drive(self):
from google.colab import drive
drive.mount('/content/drive')
def get_page_data(self, page_number):
start_index = page_number * 10
end_index = start_index + 10
return self.data.iloc[start_index:end_index]
def update_page(self, new_page):
self.current_page = new_page
return (
self.get_page_data(self.current_page),
self.current_page > 0,
self.current_page < len(self.data) // 10 - 1,
self.current_page
)
def clear_txt(self):
self.data['text'] =self.data['text'].apply(self.remove_hamza_from_alif_and_symbols)
return self.get_page_data(self.current_page)
def get_text_from_audio(self,audio):
if len(audio)!=0:
sf.write("temp.wav", audio, 16000,format='WAV')

client = Client("MohamedRashad/Arabic-Whisper-CodeSwitching-Edition")
result = client.predict(
inputs=handle_file('temp.wav'),
api_name="/predict_1"
)
return result
else:
return ""

def on_column_dropdown_change_operater(self,selected_column,selected_column1):
if selected_column1==">":
return self.data[self.data['secs'] > selected_column ]
elif selected_column1=="<":
return self.data[self.data['secs'] < selected_column]
elif selected_column1=="=":
return self.data[self.data['secs'] == selected_column]
else:
return self.data
# Perform actions based on the selected column

def on_column_dropdown_change(self,selected_column):
data=self.df
if selected_column=="all":

return self.set1(data),len(data)
elif selected_column=="0":
data=data[data['flag'] ==0]
return self.set1(data),len(data)
else :
data=data[data['flag'] ==1]
return self.set1(data),len(data)

def on_select(self,evt:gr.SelectData):
index_now = evt.index[0]
self.current_selected = (self.current_page * 10) + index_now
row = self.data.iloc[self.current_selected]
row_audio = self.sdata[self.current_selected]
self.speaker_id=row['speaker_id']
return (16000, row_audio), row['text']
def finsh_data(self):
self.df['audio'] = self.sdata
self.df[['text','speaker_id','secs','flag']]=self.data

return self.df
def All_enhance(self):
for i in range(0,len(self.sdata)):
_,y=remove_nn(self.sdata[i])
self.sdata[i]=y
return self.data

return self.get_page_data(self.current_page)
def get_output_audio(self):
return self.sdata[self.current_selected] if self.current_selected >= 0 else None
def Convert_DataFreme_To_DataSet(self,namedata):
df=self.df

df['audio'] = df['audio'].apply(lambda x: np.array(x, dtype=np.float32))
if "__index_level_0__" in df.columns:
df =df.drop(columns=["__index_level_0__"])
train_df =df



ds = {
"train": Dataset.from_panda