In [1]:
# Initialize Notebook
from IPython.core.display import HTML,Image
#%run ../library/v1.0.5/init.ipy
HTML('''<script> code_show=true; function code_toggle() { if (code_show){ $('div.input').hide(); } else { $('div.input').show(); } code_show = !code_show } $( document ).ready(code_toggle); </script> <form action="javascript:code_toggle()"><input type="submit" value="Toggle Code"></form>''')
Out[1]:
In [2]:
import warnings
warnings.filterwarnings('ignore')
import gc, argparse, sys, os, errno
%pylab inline
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt;
import seaborn as sns
#sns.set()
#sns.set_style('whitegrid')
import h5py
from PIL import Image
import os
from tqdm import tqdm_notebook as tqdm
import scipy
import sklearn
from scipy.stats import pearsonr
from scipy.io import loadmat
import IPython.display as ipd
import IPython
import librosa.display
import librosa
from pystoi import stoi
In [3]:
from mcd import dtw
import mcd.metrics as mt
def mel_cep_dtw_dist(target, converted):
"""
Compute the distance between two unaligned speech waveforms
:param target: reference speech numpy array
:param converted: synthesized speech numpy array
:return: mel cep distance in dB
"""
total_cost = 0
total_frames = 0
for (tar, conv) in zip(target, converted):
tar, conv = tar.astype('float64'), conv.astype('float64')
cost, _ = dtw.dtw(tar, conv, mt.logSpecDbDist)
frames = len(tar)
total_cost += cost
total_frames += frames
return total_cost / total_frames
def plot_stft(audio,ax=None,n_fft=256,hop_length=128,show=False,n_mels=128,y_axis='mel'):
X = librosa.stft(audio,n_fft=n_fft,hop_length=hop_length)
if y_axis=='mel':
#x_stft_db = librosa.feature.melspectrogram(x, sr=16000,n_fft=n_fft,win_length=win_length,hop_length=hop_length)
S = librosa.feature.melspectrogram(audio, sr=16000,n_mels=n_mels,fmax=8000,n_fft=n_fft,hop_length=hop_length)
#print (S.shape)
if show:
librosa.display.specshow(librosa.power_to_db(S,
ref=np.max),
y_axis='mel',cmap='gray_r',ax=ax, fmax=8000)
else:
spec_db = librosa.power_to_db(S,ref=np.max)
#level = 80
#spec_db[spec_db<=-level] = -100
#spec_db[spec_db==-level] = -100
return spec_db
else:
if show:
specshow(librosa.amplitude_to_db(abs(X)),cmap=cm.Blues,#cm.gray_r,
sr=16000,ax=ax)
else:
return librosa.amplitude_to_db(abs(X))
def MSE_pcc(A,B,ax=None):
mse =np.mean(((A - B)**2/B.var()))
pcc = pearsonr(A.ravel(),B.ravel())[0]
return mse,pcc
def analyze(predict,GT_STFT_test_spkr,audio_pred,audio_gt,mode='test',ind=-1,plot=False,mcd=None):
samples = predict.shape[0]
pcc = np.zeros([samples])
mse = np.zeros([samples])
for i in range(samples):
mse[i], pcc[i] = MSE_pcc(predict[i],GT_STFT_test_spkr[i])
#mse[i], pcc[i] = MSE_pcc(predict[i] ,GT_STFT_test_spkr[i] )
stois = []
timedur = 0#0.06
for i in range(samples):
stois.append(stoi(np.concatenate((np.ones([int(interval*timedur)]),\
audio_pred[i*interval:(i+1)*interval],np.ones([int(interval*timedur)]))), \
np.concatenate((np.ones([int(interval*timedur)]),\
audio_gt[i*interval:(i+1)*interval],np.ones([int(interval*timedur)]))), 16000, extended=False))
stois = np.array(stois)
if plot:
if mcd is not None:
fig,ax=plt.subplots(1,4,figsize=(18,4))
ax[3].hist(mcd,bins=50,color='m')
ax[3].set_title(mode+' MCD: %g(%g)' %(np.round(mcd.mean(),3),np.round(mcd.std(),3)))
else:
fig,ax=plt.subplots(1,3,figsize=(20,4))
#fig,ax=plt.subplots(1,3,figsize=(18,4))
ax[0].hist(mse,bins=25,color='b')
ax[0].set_title('ind '+str(ind)+' '+mode+' MSE: %g(%g)' %(np.round(mse.mean(),3),np.round(mse.std(),3)))
ax[1].hist(pcc,bins=50,color='g')
ax[1].set_title(mode+' PCC: %g(%g)' %(np.round(pcc.mean(),3),np.round(pcc.std(),3)))
ax[2].hist(stois,bins=50,color='r')
ax[2].set_title(mode+' STOI: %g(%g)' %(np.round(stois.mean(),3),np.round(stois.std(),3)))
return mse,pcc,stois
def play(audio,sr=16000):
'''
audio: tensor, eg: ex['audio']
'''
if len(audio.shape) >=2:
audio = audio.ravel()
display(ipd.Audio(audio,rate=sr))
def amplitude(x,noise_db=-50,max_db=22.5,trim_noise=True):
if trim_noise:
x_db = (x+1)/2*(max_db-noise_db)+noise_db
if type(x) is np.ndarray:
return 10**(x_db/10)*(x_db>noise_db).astype(np.float32)
else:
return 10**(x_db/10)*(x_db>noise_db).float()
else:
return 10**(((x+1)/2*(max_db-noise_db)+noise_db)/10)
def log_spec_dB_dist(x, y):
log_spec_dB_const = 10.0 / math.log(10.0) * math.sqrt(2.0)
diff = x - y
return log_spec_dB_const * math.sqrt(np.inner(diff, diff))
interval = 16384
In [4]:
cd /scratch/xc1490/projects/ecog/ALAE_1023
In [5]:
def get_result_dict(sampleind):
result_dict = np.load('data/formant_result/{}.npy'.format(sampleind),allow_pickle=1).item()
#print (result_dict.keys())
wave_key_list = ['wave_org_denoise','wave_rec','wave_rec_denoise','wave_rec_ecog','wave_rec_ecog_denoise']
for key in result_dict.keys():
if key!='components' and key!='components_ecog' and key!='lable':
#print (key)
#print (key,result_dict[key].shape)
if key =='org_denoise':
result_dict[key] = amplitude(result_dict[key])
if key =='rec_denoise' or key =='rec_ecog' or key =='rec_ecog_denoise' or key =='org' or key =='rec':
result_dict[key] = (result_dict[key]-0.5)*2
if key in wave_key_list:
#print (key)
factor = np.sqrt(sum(result_dict['wave_org']**2)/sum(result_dict[key]**2))
result_dict[key] = result_dict[key]*factor
for key in ['org','rec','rec_ecog','rec_ecog_denoise','rec_ecog','rec_denoise']:
result_dict[key] = np.swapaxes(result_dict[key].reshape(256,50,-1),1,0)
return result_dict
In [6]:
def get_metric_from_result_dict(result_dict):
#e2a
spec_gt = result_dict['org']
spec_pred = result_dict['rec_ecog']
#spec_gt_mel = np.array([librosa.feature.melspectrogram(S=spec_gt [i] , sr=16000,n_fft=511,hop_length=129,n_mels=256,fmax=8000) \
# for i in range(50)])
#spec_pred_mel = np.array([librosa.feature.melspectrogram(S=spec_pred[i] , sr=16000,n_fft=511,hop_length=129,n_mels=256,fmax=8000) \
# for i in range(50)])
#mcds = []
#frames = 128
#for i in range(50):
# spec_gt_tmp = spec_gt_mel[i]
# spec_pred_tmp = spec_pred_mel[i]
# total_cost =0
# mcds.append(np.mean(librosa.sequence.dtw(spec_gt_tmp[1: ], spec_pred_tmp[1: ], metric=log_spec_dB_dist)[0])/frames)
wave_gt = result_dict['wave_org'][:,0,:].ravel()
wave_pred =result_dict['wave_rec_ecog'][:819582//interval*interval].ravel()
spec_pred_mel = np.zeros([50,32,128])
spec_gt_mel = np.zeros([50,32,128])
for i in range(50):
spec_pred_mel[i] = plot_stft(wave_pred[i*interval:(i+1)*interval],n_fft=511,hop_length=129,ax=None ,y_axis='mel',n_mels=32)
spec_gt_mel[i] = plot_stft(wave_gt[i*interval:(i+1)*interval],n_fft=511,hop_length=129,ax=None,y_axis='mel',n_mels=32 )
spec_concat_e2a = np.concatenate(( np.flip(spec_gt_mel,axis=1), np.flip(spec_pred_mel,axis=1)),axis=1)
mcds = []
for i in range(50) :
mcds.append(mel_cep_dtw_dist(spec_pred_mel[i].T[:,1:]/10,spec_gt_mel[i].T[:,1:]/10))
mcds = np.array(mcds)
mcd_e2a = mcds
mse_test_e2a,pcc_test_e2a,stois_test_e2a = analyze(spec_pred_mel,spec_gt_mel,wave_pred,wave_gt,plot=False,mcd=mcds)
#a2a
spec_gt = result_dict['org']
spec_pred = result_dict['rec']
#spec_gt_mel = np.array([librosa.feature.melspectrogram(S=spec_gt [i] , sr=16000,n_fft=511,hop_length=129,n_mels=256,fmax=8000) \
# for i in range(50)])
#spec_pred_mel = np.array([librosa.feature.melspectrogram(S=spec_pred[i] , sr=16000,n_fft=511,hop_length=129,n_mels=256,fmax=8000) \
# for i in range(50)])
#mcds = []
#frames = 128
#for i in range(50):
# spec_gt_tmp = spec_gt_mel[i]
# spec_pred_tmp = spec_pred_mel[i]
# total_cost =0
# mcds.append(np.mean(librosa.sequence.dtw(spec_gt_tmp[1: ], spec_pred_tmp[1: ], metric=log_spec_dB_dist)[0])/frames)
wave_gt = result_dict['wave_org'][:,0,:].ravel()
wave_pred =result_dict['wave_rec'][:819582//interval*interval].ravel()
spec_pred_mel = np.zeros([50,32,128])
spec_gt_mel = np.zeros([50,32,128])
for i in range(50):
spec_pred_mel[i] = plot_stft(wave_pred[i*interval:(i+1)*interval],n_fft=511,hop_length=129,ax=None ,y_axis='mel',n_mels=32)
spec_gt_mel[i] = plot_stft(wave_gt[i*interval:(i+1)*interval],n_fft=511,hop_length=129,ax=None,y_axis='mel',n_mels=32 )
spec_concat_a2a = np.concatenate(( np.flip(spec_gt_mel,axis=1), np.flip(spec_pred_mel,axis=1)),axis=1)
mcds = []
for i in range(50) :
mcds.append(mel_cep_dtw_dist(spec_pred_mel[i].T[:,1:]/10,spec_gt_mel[i].T[:,1:]/10))
mcds = np.array(mcds)
mcd_a2a = mcds
mse_test_a2a,pcc_test_a2a,stois_test_a2a = analyze(spec_pred_mel,spec_gt_mel,wave_pred,wave_gt,plot=False,mcd=mcds)
#mfcc e2a
wave_gt = result_dict['wave_org'][:,0,:].ravel()
wave_pred =result_dict['wave_rec_ecog'][:819582//interval*interval].ravel()
spec_pred = np.zeros([50,32,33])
spec_gt = np.zeros([50,32,33])
mfcc_e2a = np.zeros([50])
for i in range(50):
spec_pred[i] = librosa.feature.mfcc(y=wave_pred[i*interval:(i+1)*interval], sr=16000,n_mfcc=32)
spec_gt[i] = librosa.feature.mfcc(y=wave_gt[i*interval:(i+1)*interval], sr=16000,n_mfcc=32)
mfcc_e2a[i] = pearsonr(spec_pred[i].ravel(),spec_gt[i].ravel())[0]
#mfcc a2a
wave_gt = result_dict['wave_org'][:,0,:].ravel()
wave_pred =result_dict['wave_rec'][:819582//interval*interval].ravel()
spec_pred = np.zeros([50,32,33])
spec_gt = np.zeros([50,32,33])
mfcc_a2a = np.zeros([50])
for i in range(50):
spec_pred[i] = librosa.feature.mfcc(y=wave_pred[i*interval:(i+1)*interval], sr=16000,n_mfcc=32)
spec_gt[i] = librosa.feature.mfcc(y=wave_gt[i*interval:(i+1)*interval], sr=16000,n_mfcc=32)
mfcc_a2a[i] = pearsonr(spec_pred[i].ravel(),spec_gt[i].ravel())[0]
components_keys = ['f0','loudness', 'amplitudes', 'amplitudes_h', 'freq_formants_hamon_hz', 'bandwidth_formants_hamon_hz', 'amplitude_formants_hamon','freq_formants_noise_hz', 'bandwidth_formants_noise_hz', 'amplitude_formants_noise']
components_pcc = {}
for key in components_keys:
components_pcc[key] = np.zeros([50])
for i in range(50):
components_pcc[key][i] =pearsonr((result_dict['components'][key][i] *(result_dict['components']['amplitudes'][i,0:1,:] >=0.2)).ravel(),\
(result_dict['components_ecog'][key][i] *(result_dict['components']['amplitudes'][i,0:1,:] >=0.2)).ravel())[0]
#print (key,result_dict['components'][key].shape,np.mean(components_pcc[key]))
return mse_test_e2a, pcc_test_e2a, stois_test_e2a, mcd_e2a, mfcc_e2a, \
mse_test_a2a, pcc_test_a2a, stois_test_a2a, mcd_a2a, mfcc_a2a, components_pcc
In [7]:
mse_test_e2a_dict, pcc_test_e2a_dict, stois_test_e2a_dict, mcd_e2a_dict, mfcc_e2a_dict, \
mse_test_a2a_dict, pcc_test_a2a_dict, stois_test_a2a_dict, mcd_a2a_dict, mfcc_a2a_dict, components_pcc_dict = {},{},{},{},{},{},{},{},{},{},{}
sample_inds = [717,742,749]
for sample_ind in sample_inds:
result_dict = get_result_dict(sample_ind)
mse_test_e2a_dict[sample_ind], pcc_test_e2a_dict[sample_ind], stois_test_e2a_dict[sample_ind], \
mcd_e2a_dict[sample_ind], mfcc_e2a_dict[sample_ind], mse_test_a2a_dict[sample_ind], pcc_test_a2a_dict[sample_ind],\
stois_test_a2a_dict[sample_ind], mcd_a2a_dict[sample_ind], mfcc_a2a_dict[sample_ind], components_pcc_dict[sample_ind] \
= get_metric_from_result_dict(result_dict)
In [8]:
flatui = ['#FF7373','#BBFF5C','#F0B9FF',"#FEBE87"]
sns.set_palette(sns.color_palette(flatui))
In [9]:
task = ['Reference','Decoded']
sample_inds = [717, 742, 749]
arr_new = np.array([])
for sample_ind in sample_inds:
arr_new = np.concatenate((arr_new, \
np.concatenate((pcc_test_a2a_dict[sample_ind],pcc_test_e2a_dict[sample_ind])) ))
sample_name = np.repeat(np.repeat(['1', '2', '3'] ,len(task)).ravel(),50).reshape(-1,1)
task_name = np.repeat(np.array([task for j in range(len(sample_inds))]).ravel(),50).reshape(-1,1)
df = pd.DataFrame(np.concatenate((arr_new.reshape(-1,1),sample_name,task_name),axis=1))
df.columns= ['value','sample','task']
df.value = arr_new.reshape(-1,1)
fig,ax=plt.subplots(1,figsize=(6,6))
b = sns.boxplot(ax=ax,data=df,y='value',hue='task',x='sample',width=0.5,linewidth=4,saturation=1) #the middle line is median
plt.legend(bbox_to_anchor=(1.01, 1), loc=2, borderaxespad=0.,fontsize=15)
b.set_xlabel("Participant",fontsize=20)
b.set_ylabel('Spectrogram Correlation',fontsize=20)
b.tick_params(labelsize=20)
ax.set_ylim(0.4,1)
ax.spines['right'].set_visible(False)
ax.spines['top'].set_visible(False)
rc('axes', linewidth=2)
In [10]:
task = ['Reference','Decoded']
sample_inds = [717, 742, 749]
arr_new = np.array([])
for sample_ind in sample_inds:
arr_new = np.concatenate((arr_new, \
np.concatenate((mfcc_a2a_dict[sample_ind],mfcc_e2a_dict[sample_ind])) ))
sample_name = np.repeat(np.repeat(['1', '2', '3'] ,len(task)).ravel(),50).reshape(-1,1)
task_name = np.repeat(np.array([task for j in range(len(sample_inds))]).ravel(),50).reshape(-1,1)
df = pd.DataFrame(np.concatenate((arr_new.reshape(-1,1),sample_name,task_name),axis=1))
df.columns= ['value','sample','task']
df.value = arr_new.reshape(-1,1)
fig,ax=plt.subplots(1,figsize=(6,6))
b = sns.boxplot(ax=ax,data=df,y='value',hue='task',x='sample',width=0.5,linewidth=4,saturation=1) #the middle line is median
plt.legend(bbox_to_anchor=(1.01, 1), loc=2, borderaxespad=0.,fontsize=15)
add_text = ''
b.set_xlabel("Participant",fontsize=20)
b.set_ylabel('MFCC Correlation',fontsize=20)
b.tick_params(labelsize=20)
#ax.set_ylim(0.8,1)
ax.spines['right'].set_visible(False)
ax.spines['top'].set_visible(False)
rc('axes', linewidth=2)
In [11]:
task = ['Reference','Decoded']
sample_inds = [717, 742, 749]
arr_new = np.array([])
for sample_ind in sample_inds:
arr_new = np.concatenate((arr_new, \
np.concatenate((stois_test_a2a_dict[sample_ind],stois_test_e2a_dict[sample_ind])) ))
sample_name = np.repeat(np.repeat(['1', '2', '3'] ,len(task)).ravel(),50).reshape(-1,1)
task_name = np.repeat(np.array([task for j in range(len(sample_inds))]).ravel(),50).reshape(-1,1)
df = pd.DataFrame(np.concatenate((arr_new.reshape(-1,1),sample_name,task_name),axis=1))
df.columns= ['value','sample','task']
df.value = arr_new.reshape(-1,1)
#with plt.style.context(['science', 'ieee']):
fig,ax=plt.subplots(1,figsize=(6,6))
b = sns.boxplot(ax=ax,data=df,y='value',hue='task',x='sample',width=0.5,linewidth=4,saturation=1) #the middle line is median
plt.legend(bbox_to_anchor=(1.01, 1), loc=2, borderaxespad=0.,fontsize=15)
#b.axes.set_title('STOI',fontsize=30)
add_text = ''
#if metric=='mcd':
# add_text = '( dB)'
b.set_xlabel("Participant",fontsize=20)
b.set_ylabel('STOI',fontsize=20)
b.tick_params(labelsize=20)
ax.set_ylim(0,1)
ax.spines['right'].set_visible(False)
ax.spines['top'].set_visible(False)
rc('axes', linewidth=2)
In [12]:
task = ['Reference','Decoded']
sample_inds = [717, 742, 749]
arr_new = np.array([])
for sample_ind in sample_inds:
arr_new = np.concatenate((arr_new, \
np.concatenate((mcd_a2a_dict[sample_ind],mcd_e2a_dict[sample_ind])) ))
sample_name = np.repeat(np.repeat(['1', '2', '3'] ,len(task)).ravel(),50).reshape(-1,1)
task_name = np.repeat(np.array([task for j in range(len(sample_inds))]).ravel(),50).reshape(-1,1)
df = pd.DataFrame(np.concatenate((arr_new.reshape(-1,1),sample_name,task_name),axis=1))
df.columns= ['value','sample','task']
df.value = arr_new.reshape(-1,1)
#with plt.style.context(['science', 'ieee']):
fig,ax=plt.subplots(1,figsize=(6,6))
b = sns.boxplot(ax=ax,data=df,y='value',hue='task',x='sample',width=0.5,linewidth=4,saturation=1) #the middle line is median
plt.legend(bbox_to_anchor=(1.01, 1), loc=2, borderaxespad=0.,fontsize=15)
#b.axes.set_title('MCD',fontsize=30)
add_text = ''
#if metric=='mcd':
# add_text = '( dB)'
b.set_xlabel("Participant",fontsize=20)
b.set_ylabel('MCD (dB)',fontsize=20)
b.tick_params(labelsize=20)
ax.set_ylim(0,np.max(arr_new)+0.5)
ax.spines['right'].set_visible(False)
ax.spines['top'].set_visible(False)
rc('axes', linewidth=2)
In [13]:
fig,axes=plt.subplots(2,2,figsize=(12,10))
task = ['Reference','Decoded']
sample_inds = [717, 742, 749]
arr_new = np.array([])
for sample_ind in sample_inds:
arr_new = np.concatenate((arr_new, \
np.concatenate((pcc_test_a2a_dict[sample_ind],pcc_test_e2a_dict[sample_ind])) ))
sample_name = np.repeat(np.repeat(['1', '2', '3'] ,len(task)).ravel(),50).reshape(-1,1)
task_name = np.repeat(np.array([task for j in range(len(sample_inds))]).ravel(),50).reshape(-1,1)
df = pd.DataFrame(np.concatenate((arr_new.reshape(-1,1),sample_name,task_name),axis=1))
df.columns= ['value','sample','task']
df.value = arr_new.reshape(-1,1)
ax = axes[0,0]
b = sns.boxplot(ax=ax,data=df,y='value',hue='task',x='sample',width=0.5,linewidth=4,saturation=1) #the middle line is median
#plt.legend(bbox_to_anchor=(1.01, 1), loc=2, borderaxespad=0.,fontsize=15)
b.set_xlabel("Participant",fontsize=20)
b.set_ylabel('Spectrogram Correlation',fontsize=20)
b.tick_params(labelsize=20)
ax.set_ylim(0.4,1)
ax.spines['right'].set_visible(False)
ax.spines['top'].set_visible(False)
arr_new = np.array([])
for sample_ind in sample_inds:
arr_new = np.concatenate((arr_new, \
np.concatenate((mfcc_a2a_dict[sample_ind],mfcc_e2a_dict[sample_ind])) ))
sample_name = np.repeat(np.repeat(['1', '2', '3'] ,len(task)).ravel(),50).reshape(-1,1)
task_name = np.repeat(np.array([task for j in range(len(sample_inds))]).ravel(),50).reshape(-1,1)
df = pd.DataFrame(np.concatenate((arr_new.reshape(-1,1),sample_name,task_name),axis=1))
df.columns= ['value','sample','task']
df.value = arr_new.reshape(-1,1)
ax = axes[0,1]
b = sns.boxplot(ax=ax,data=df,y='value',hue='task',x='sample',width=0.5,linewidth=4,saturation=1) #the middle line is median
#plt.legend(bbox_to_anchor=(1.01, 1), loc=2, borderaxespad=0.,fontsize=15)
b.set_xlabel("Participant",fontsize=20)
b.set_ylabel('MFCC Correlation',fontsize=20)
b.tick_params(labelsize=20)
#ax.set_ylim(0.8,1)
ax.spines['right'].set_visible(False)
ax.spines['top'].set_visible(False)
arr_new = np.array([])
for sample_ind in sample_inds:
arr_new = np.concatenate((arr_new, \
np.concatenate((stois_test_a2a_dict[sample_ind],stois_test_e2a_dict[sample_ind])) ))
sample_name = np.repeat(np.repeat(['1', '2', '3'] ,len(task)).ravel(),50).reshape(-1,1)
task_name = np.repeat(np.array([task for j in range(len(sample_inds))]).ravel(),50).reshape(-1,1)
df = pd.DataFrame(np.concatenate((arr_new.reshape(-1,1),sample_name,task_name),axis=1))
df.columns= ['value','sample','task']
df.value = arr_new.reshape(-1,1)
ax = axes[1,0]
b = sns.boxplot(ax=ax,data=df,y='value',hue='task',x='sample',width=0.5,linewidth=4,saturation=1) #the middle line is median
#plt.legend(bbox_to_anchor=(1.01, 1), loc=2, borderaxespad=0.,fontsize=15)
b.set_xlabel("Participant",fontsize=20)
b.set_ylabel('STOI',fontsize=20)
b.tick_params(labelsize=20)
ax.set_ylim(0,1)
ax.spines['right'].set_visible(False)
ax.spines['top'].set_visible(False)
arr_new = np.array([])
for sample_ind in sample_inds:
arr_new = np.concatenate((arr_new, \
np.concatenate((mcd_a2a_dict[sample_ind],mcd_e2a_dict[sample_ind])) ))
sample_name = np.repeat(np.repeat(['1', '2', '3'] ,len(task)).ravel(),50).reshape(-1,1)
task_name = np.repeat(np.array([task for j in range(len(sample_inds))]).ravel(),50).reshape(-1,1)
df = pd.DataFrame(np.concatenate((arr_new.reshape(-1,1),sample_name,task_name),axis=1))
df.columns= ['value','sample','task']
df.value = arr_new.reshape(-1,1)
ax = axes[1,1]
b = sns.boxplot(ax=ax,data=df,y='value',hue='task',x='sample',width=0.5,linewidth=4,saturation=1) #the middle line is median
#plt.legend(bbox_to_anchor=(1.01, 1), loc=2, borderaxespad=0.,fontsize=15)
b.set_xlabel("Participant",fontsize=20)
b.set_ylabel('MCD (dB)',fontsize=20)
b.tick_params(labelsize=20)
ax.set_ylim(0,np.max(arr_new)+0.5)
ax.spines['right'].set_visible(False)
ax.spines['top'].set_visible(False)
rc('axes', linewidth=2)
handles, labels = ax.get_legend_handles_labels()
lgd = fig.legend(handles, labels,loc="upper right",borderaxespad=0.,fontsize=15, ncol=2)
axes[0,0].get_legend().remove()
axes[0,1].get_legend().remove()
axes[1,0].get_legend().remove()
axes[1,1].get_legend().remove()
plt.tight_layout()
fig.savefig('data/formant_result/ner_metric.pdf', bbox_extra_artists=(lgd,), bbox_inches='tight')
In [14]:
mse_new = np.zeros([4,3])#actually a2a 4 metric* 3 sample
sample_inds = [717,742,749]
for j in range(3):
mse_new[0,j] = np.mean(pcc_test_a2a_dict[sample_inds[j]])
mse_new[1,j] = np.mean(stois_test_a2a_dict[sample_inds[j]])
mse_new[2,j] = np.mean(mfcc_a2a_dict[sample_inds[j]])
mse_new[3,j] = np.mean(mcd_a2a_dict[sample_inds[j]])
pcc_new = np.zeros([4,3])#actually e2a 4 metric* 3 sample
sample_inds = [717,742,749]
for j in range(3):
pcc_new[0,j] = np.mean(pcc_test_e2a_dict[sample_inds[j]])
pcc_new[1,j] = np.mean(stois_test_e2a_dict[sample_inds[j]])
pcc_new[2,j] = np.mean(mfcc_e2a_dict[sample_inds[j]])
pcc_new[3,j] = np.mean(mcd_e2a_dict[sample_inds[j]])
In [15]:
df_new = pd.DataFrame(np.zeros([4,4]))
df_new = df_new.astype('str')
for i in range(4):
for j in range(3):
df_new.iloc[i,j] = str(mse_new[i,j])+'/' +str(pcc_new[i,j])
for i in range(4):
df_new.iloc[i,3] = str(np.round(np.mean(mse_new[i,:]),3))+'/' +str(np.round(np.mean(pcc_new[i,:]),3))
In [16]:
df_new.columns = ['NY717','NY742','NY749','Average']
df_new.index = ['SPEC PCC','STOI','MFCC PCC','MCD']
In [17]:
df = df_new
df_index = pd.DataFrame(np.array([(str(i)+'|'+str(j)) for i in range(df.shape[0]) \
for j in range(df.shape[1])]).reshape(df.shape[0],-1))
df_index.index = df.index
df_index.columns = df.columns
def gradient_func(val):
row, col = val.split('|')
row, col = int(row), int(col)
split1, split2 = df.iloc[row, col].split('/') #metric
format_use = float(split2)*100
color = 'black'#{'pass': 'green', 'fail': 'red', 'warn': 'orange'}.get(status, 'gray')
if col!=3:
return '<a href="{sample_id}/NER_result_final_{sample_id}.html" style="color: {color}"><span style="background: linear-gradient(90deg, rgba(61,164,166,1) {format_use}%, transparent 0%)">{split1:.3f}|{split2:.3f}</span></a>'.format(
sample_id=int(df.columns[col][2:]), color=color, split1=float(split1),split2=float(split2),format_use=format_use)
else:
return '<span style="background: linear-gradient(90deg, rgba(61,164,166,1) {format_use}%, transparent 0%)">{split1:.3f}|{split2:.3f}</span>'.format(
sample_id=df.columns[col], area_id =df.index[row], color=color, split1=float(split1),split2=float(split2),format_use=format_use)
#df_index.style.format(style_func)
def display_dataframe(df, filename=None, encoding='utf-8', format='csv', type='button',gradientfunc=False, **kwargs):
#display(df)
#if isinstance(df, pd.DataFrame):
# display(df.style.set_caption(filename))
#else:
if gradientfunc == False:
display(df.style.set_caption(filename))
else:
display(df.style.format(gradient_func).set_caption(filename))
if filename is None:
filename = "dataframe"
if format == 'csv':
data = df.to_csv(**kwargs)
mime_type = 'text/csv'
filename = filename + '.csv'
elif format == 'tsv':
data = df.to_csv(**kwargs)
mime_type = 'text/plain'
filename = filename + '.txt'
else:
raise ValueError('unknown file format: {}'.format(format))
data = 'data:{mime_type};base64,'.format(mime_type=mime_type) + str(b64encode(bytes(data, encoding=encoding)), encoding=encoding)
if type == 'hyperlink':
display(HTML('<a href=" " download={filename} target="_blank">{filename}</a >'.format(
mime_type=mime_type, filename=filename, data=data)))
elif type == 'button':
button_id = 'button_{}'.format(np.random.randint(1000000000))
display(HTML(r'<input type="button" id="{0}" value="Download">'.format(button_id)))
display(HTML('''<script>
document.getElementById("{button_id}").addEventListener("click", function(event){{
var filename = "{filename}";
var data = "{data}";
const element = document.createElement('a');
element.setAttribute('href', data);
element.setAttribute('download', filename);
element.style.display = 'none';
document.body.appendChild(element);
element.click();
document.body.removeChild(element);
}});
</script>'''.format(button_id=button_id, filename=filename, data=data)))
from matplotlib.backends.backend_pdf import PdfPages, PdfFile
from IPython.display import HTML, display, FileLink
from base64 import b64encode, b64decode
from io import StringIO, BytesIO
from contextlib import contextmanager
display_dataframe(df_index,gradientfunc=True,filename='overall performance')
Reference(audio2audio) | Decoded(ecog2audio)
In [ ]: