diff --git a/classifier/feature_extraction/batch_feature_extractor.py b/classifier/feature_extraction/batch_feature_extractor.py index 6acbe84..8d089de 100644 --- a/classifier/feature_extraction/batch_feature_extractor.py +++ b/classifier/feature_extraction/batch_feature_extractor.py @@ -1,7 +1,8 @@ from os import listdir from os.path import isfile, join import multiprocessing as mp -from .feature_extractor import extractFeatures +import pandas as pd +from feature_extractor import extractFeatures class bcolors: BLUE = '\033[94m' @@ -13,18 +14,21 @@ class bcolors: def batchExtract(audioFilesPath, featureFilesPath, sampleRate): audioFiles = [file for file in listdir(audioFilesPath) if isfile(join(audioFilesPath, file))] - # Without multithreading - # for file in audioFiles: - # extractFeatures(audioFilesPath + file, - # featureFilesPath + file[0:file.rfind('.')] + '.json', int(sampleRate)) + dataframesList = [None]*len(audioFiles) - pool = mp.Pool(processes = 8) - for file in audioFiles: - pool.apply(extractFeatures,args=(audioFilesPath + file, - featureFilesPath + file[0:file.rfind('.')] + '.json',int(sampleRate))) + pool = mp.Pool() + for process, file in enumerate(audioFiles): + dataframesList[process] = pool.apply_async(extractFeatures,args=(audioFilesPath + file, + featureFilesPath + file[0:file.rfind('.')] + '.json',int(sampleRate))).get() + pool.close() + pool.join() + + joinedDataset = pd.concat(dataframesList) print('Batch feature extraction finished successfully.') + return joinedDataset + # Prints a nice message to let the user know the module was imported print(bcolors.BLUE + 'batch_feature_extractor loaded' + bcolors.ENDC) diff --git a/classifier/feature_extraction/feature_extractor.py b/classifier/feature_extraction/feature_extractor.py index 34cc6b9..2fd1a82 100644 --- a/classifier/feature_extraction/feature_extractor.py +++ b/classifier/feature_extraction/feature_extractor.py @@ -1,4 +1,6 @@ import essentia +import pandas as pd +import numpy as np from essentia.standard import (MonoLoader, Windowing, Spectrum, MFCC, ZeroCrossingRate, SpectralCentroidTime, RollOff, Flux, Envelope, FlatnessSFX, LogAttackTime, StrongDecay, FlatnessDB, HFC, @@ -95,6 +97,8 @@ def extractFeatures(audio, outputPath, sampleRate): YamlOutput(filename = outputPath, format = 'json', writeVersion = False)(pool) + return pd.DataFrame(np.array([pool[i] for i in pool.descriptorNames()]).T, columns = pool.descriptorNames()) + def _4HzModulation(melEnergies, frameEnergy, sampleRate): from scipy.signal import butter, sosfilt, sosfreqz nyquist = 0.5 * sampleRate diff --git a/classifier/pipeline.py b/classifier/pipeline.py index f291d77..82150d7 100644 --- a/classifier/pipeline.py +++ b/classifier/pipeline.py @@ -1,24 +1,26 @@ import numpy as np +import pandas as pd from feature_extraction.feature_extractor import extractFeatures from feature_extraction.batch_feature_extractor import batchExtract -from preprocessing.data_preprocessing import arrayFromJSON, createSingleFeaturesArray, standardization, PCA +from preprocessing.data_preprocessing import arrayFromJSON, standardization, PCA from training.model_training import simpleTrain, kFCrossValid -batchExtract('../dataset/music_wav/', 'feature_extraction/music_features/', 22050) -batchExtract('../dataset/speech_wav/', 'feature_extraction/speech_features/', 22050) +musicFeatures = batchExtract('../dataset/music_wav/', 'feature_extraction/music_features/', 22050) +musicFeatures = musicFeatures.assign(target=0) +speechFeatures = batchExtract('../dataset/speech_wav/', 'feature_extraction/speech_features/', 22050) +speechFeatures = speechFeatures.assign(target=1) -dataset, target, featureKeys = createSingleFeaturesArray( - 'feature_extraction/music_features/', - 'feature_extraction/speech_features/') +dataset = pd.concat([musicFeatures, speechFeatures]) +target = dataset.pop('target').values dataset = standardization(dataset) # dataset = PCA(dataset) + print('Simple train accuracy achieved = ' + str(simpleTrain(dataset, target))) kFCrossValid(dataset, target, model = 'svm') clf = kFCrossValid(dataset, target, model = 'rndForest') -extractFeatures('compined.wav', 'featuresStream/tmp.json', 22050) -values = arrayFromJSON('featuresStream/tmp.json')[1] -values = standardization(values) -audioClass = clf.predict(values) +features = extractFeatures('compined.wav', 'tmp.json', 22050) +features = standardization(features) +audioClass = clf.predict(features) print(audioClass) \ No newline at end of file diff --git a/classifier/preprocessing/__init__.py b/classifier/preprocessing/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/classifier/preprocessing/data_preprocessing.py b/classifier/preprocessing/data_preprocessing.py index 8c21610..cbd6344 100644 --- a/classifier/preprocessing/data_preprocessing.py +++ b/classifier/preprocessing/data_preprocessing.py @@ -1,6 +1,7 @@ from os import listdir from os.path import isfile, join import numpy as np +import pandas as pd import json class bcolors: @@ -77,12 +78,10 @@ def PCA(dataset): from sklearn.decomposition import PCA print(bcolors.YELLOW + 'Running PCA' + bcolors.ENDC) - pca = PCA(n_components=15, svd_solver='full', whiten = True) + pca = PCA(n_components=10, svd_solver='full', whiten = True) transformedDataset = pca.fit(dataset).transform(dataset) - # TODO: change the return value after the values of the parameters are decided - # and the feature selection is complete - return transformedDataset + return pca, transformedDataset # Prints a nice message to let the user know the module was imported print(bcolors.BLUE + 'feature_preprocessing loaded' + bcolors.ENDC) @@ -91,9 +90,9 @@ print(bcolors.BLUE + 'feature_preprocessing loaded' + bcolors.ENDC) if __name__ == "__main__": import sys dataset, target, featureKeys = createSingleFeaturesArray(sys.argv[1], sys.argv[2]) - newDataset = PCA(standardization(dataset)) + scaledDataset = standardization(dataset) - print(bcolors.GREEN + 'Saving results to files' + bcolors.ENDC) - np.save('dataset.npy', newDataset) - np.save('target.npy', target) - np.save('featureKeys.npy', featureKeys) \ No newline at end of file + print(bcolors.GREEN + 'Saving scaled results to file' + bcolors.ENDC) + datasetFrame = pd.DataFrame(scaledDataset, columns = featureKeys) + datasetFrame = datasetFrame.assign(target=target) + datasetFrame.to_pickle("./dataset.pkl") \ No newline at end of file diff --git a/classifier/preprocessing/dataset.npy b/classifier/preprocessing/dataset.npy deleted file mode 100644 index e10dde5..0000000 Binary files a/classifier/preprocessing/dataset.npy and /dev/null differ diff --git a/classifier/preprocessing/dataset.pkl b/classifier/preprocessing/dataset.pkl new file mode 100644 index 0000000..20abcff Binary files /dev/null and b/classifier/preprocessing/dataset.pkl differ diff --git a/classifier/preprocessing/featureKeys.npy b/classifier/preprocessing/featureKeys.npy deleted file mode 100644 index 08144af..0000000 Binary files a/classifier/preprocessing/featureKeys.npy and /dev/null differ diff --git a/classifier/preprocessing/target.npy b/classifier/preprocessing/target.npy deleted file mode 100644 index 3f44b89..0000000 Binary files a/classifier/preprocessing/target.npy and /dev/null differ diff --git a/classifier/test/accuracyWithoutFeature.py b/classifier/test/accuracyWithoutFeature.py index 2c68cdf..8d116f9 100644 --- a/classifier/test/accuracyWithoutFeature.py +++ b/classifier/test/accuracyWithoutFeature.py @@ -1,55 +1,28 @@ import numpy as np +import pandas as pd from sys import path path.append('..') from feature_extraction.batch_feature_extractor import batchExtract -from preprocessing.data_preprocessing import createSingleFeaturesArray, standardization -from classification_model_training.model_training import simpleTrain +from preprocessing.data_preprocessing import standardization +from training.model_training import simpleTrain -batchExtract('../../dataset/music_wav/', '../feature_extraction/music_features/', 22050) -batchExtract('../../dataset/speech_wav/', '../feature_extraction/speech_features/', 22050) +musicFeatures = batchExtract('../../dataset/music_wav/', '../feature_extraction/music_features/', 22050) +musicFeatures = musicFeatures.assign(target=0) +speechFeatures = batchExtract('../../dataset/speech_wav/', '../feature_extraction/speech_features/', 22050) +speechFeatures = speechFeatures.assign(target=1) -dataset, target, featureKeys = createSingleFeaturesArray( - '../feature_extraction/music_features/', - '../feature_extraction/speech_features/') +dataset = pd.concat([musicFeatures, speechFeatures]) +target = dataset.pop('target').values -dataset = standardization(dataset) +dataset = pd.DataFrame(standardization(dataset), columns = dataset.columns.values) wholeAccuracy = simpleTrain(dataset, target, 'svm') print('Accuracy using whole dataset = ' + str(wholeAccuracy)) -damages = np.zeros(featureKeys.size) +damages = np.zeros(dataset.columns.values.size) -for index, key in enumerate(featureKeys): - acc = simpleTrain(np.delete(dataset, index, axis=1), target, 'svm') +for index, key in enumerate(dataset.columns.values): + acc = simpleTrain(dataset.drop(key, axis=1), target, 'svm') damages[index] = 100*(wholeAccuracy-acc) print('Accuracy without ' + key + '\t= ' + str(acc) + - ',\tdamage\t= ' + "%.2f" % damages[index] + '%') - -# Accuracy using whole dataset = 0.951902893127681 -# Accuracy without 4HzMod = 0.9456968148215752, damage = 0.62% -# Accuracy without Flat = 0.9523592224148946, damage = -0.05% -# Accuracy without HFC = 0.9526330199872228, damage = -0.07% -# Accuracy without LAtt = 0.9524504882723374, damage = -0.05% -# Accuracy without SC = 0.9520854248425664, damage = -0.02% -# Accuracy without SComp = 0.948160992972529, damage = 0.37% -# Accuracy without SDec = 0.9520854248425664, damage = -0.02% -# Accuracy without SEFlat = 0.9513552979830245, damage = 0.05% -# Accuracy without SF = 0.9492561832618417, damage = 0.26% -# Accuracy without SFlat = 0.9496212466916126, damage = 0.23% -# Accuracy without SLAtt = 0.9498950442639409, damage = 0.20% -# Accuracy without SR = 0.9523592224148946, damage = -0.05% -# Accuracy without SSDec = 0.9519941589851236, damage = -0.01% -# Accuracy without ZCR = 0.9500775759788264, damage = 0.18% -# Accuracy without mfcc0 = 0.9502601076937118, damage = 0.16% -# Accuracy without mfcc1 = 0.9510815004106964, damage = 0.08% -# Accuracy without mfcc10 = 0.9503513735511545, damage = 0.16% -# Accuracy without mfcc11 = 0.9492561832618417, damage = 0.26% -# Accuracy without mfcc12 = 0.9482522588299717, damage = 0.37% -# Accuracy without mfcc2 = 0.9446928903897052, damage = 0.72% -# Accuracy without mfcc3 = 0.9465182075385599, damage = 0.54% -# Accuracy without mfcc4 = 0.9470658026832162, damage = 0.48% -# Accuracy without mfcc5 = 0.9463356758236744, damage = 0.56% -# Accuracy without mfcc6 = 0.9452404855343616, damage = 0.67% -# Accuracy without mfcc7 = 0.9462444099662316, damage = 0.57% -# Accuracy without mfcc8 = 0.9490736515469563, damage = 0.28% -# Accuracy without mfcc9 = 0.9472483343981016, damage = 0.47% \ No newline at end of file + ',\tdamage\t= ' + "%.2f" % damages[index] + '%') \ No newline at end of file diff --git a/classifier/training/__init__.py b/classifier/training/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/classifier/visualization/output/figure_1.png b/classifier/visualization/output/figure_1.png new file mode 100644 index 0000000..6b5acfa Binary files /dev/null and b/classifier/visualization/output/figure_1.png differ diff --git a/classifier/visualization/output/figure_10.png b/classifier/visualization/output/figure_10.png new file mode 100644 index 0000000..c8f04ca Binary files /dev/null and b/classifier/visualization/output/figure_10.png differ diff --git a/classifier/visualization/output/figure_11.png b/classifier/visualization/output/figure_11.png new file mode 100644 index 0000000..dd69509 Binary files /dev/null and b/classifier/visualization/output/figure_11.png differ diff --git a/classifier/visualization/output/figure_12.png b/classifier/visualization/output/figure_12.png new file mode 100644 index 0000000..7a67d6b Binary files /dev/null and b/classifier/visualization/output/figure_12.png differ diff --git a/classifier/visualization/output/figure_13.png b/classifier/visualization/output/figure_13.png new file mode 100644 index 0000000..d52300f Binary files /dev/null and b/classifier/visualization/output/figure_13.png differ diff --git a/classifier/visualization/output/figure_14.png b/classifier/visualization/output/figure_14.png new file mode 100644 index 0000000..5ee4c1c Binary files /dev/null and b/classifier/visualization/output/figure_14.png differ diff --git a/classifier/visualization/output/figure_15.png b/classifier/visualization/output/figure_15.png new file mode 100644 index 0000000..7fbf9ed Binary files /dev/null and b/classifier/visualization/output/figure_15.png differ diff --git a/classifier/visualization/output/figure_2.png b/classifier/visualization/output/figure_2.png new file mode 100644 index 0000000..856ec2c Binary files /dev/null and b/classifier/visualization/output/figure_2.png differ diff --git a/classifier/visualization/output/figure_3.png b/classifier/visualization/output/figure_3.png new file mode 100644 index 0000000..99d7958 Binary files /dev/null and b/classifier/visualization/output/figure_3.png differ diff --git a/classifier/visualization/output/figure_4.png b/classifier/visualization/output/figure_4.png new file mode 100644 index 0000000..35db592 Binary files /dev/null and b/classifier/visualization/output/figure_4.png differ diff --git a/classifier/visualization/output/figure_5.png b/classifier/visualization/output/figure_5.png new file mode 100644 index 0000000..59b8de6 Binary files /dev/null and b/classifier/visualization/output/figure_5.png differ diff --git a/classifier/visualization/output/figure_6.png b/classifier/visualization/output/figure_6.png new file mode 100644 index 0000000..d7871bb Binary files /dev/null and b/classifier/visualization/output/figure_6.png differ diff --git a/classifier/visualization/output/figure_7.png b/classifier/visualization/output/figure_7.png new file mode 100644 index 0000000..9ddfd6f Binary files /dev/null and b/classifier/visualization/output/figure_7.png differ diff --git a/classifier/visualization/output/figure_8.png b/classifier/visualization/output/figure_8.png new file mode 100644 index 0000000..4366d31 Binary files /dev/null and b/classifier/visualization/output/figure_8.png differ diff --git a/classifier/visualization/output/figure_9.png b/classifier/visualization/output/figure_9.png new file mode 100644 index 0000000..8948f17 Binary files /dev/null and b/classifier/visualization/output/figure_9.png differ diff --git a/classifier/visualization/visualization.py b/classifier/visualization/visualization.py index b51bc8d..fafafe6 100644 --- a/classifier/visualization/visualization.py +++ b/classifier/visualization/visualization.py @@ -4,14 +4,31 @@ import matplotlib.pyplot as plt import seaborn as sns sns.set() -datasetArray = np.load('../preprocessing/dataset.npy') -target = np.load('../preprocessing/target.npy') -featureKeysVector = np.load('../preprocessing/featureKeys.npy') +dataset = pd.read_pickle('../preprocessing/dataset.pkl') +target = dataset.pop('target') -dataset = pd.DataFrame(datasetArray) -dataset.columns = featureKeysVector +# Feature evaluation +from sklearn.ensemble import ExtraTreesClassifier +from sklearn.feature_selection import SelectFromModel -sns.relplot(x="4HzMod", y="Flat", data=dataset[["4HzMod", "Flat"]], hue = target, style = target) -sns.jointplot(x="SLAtt", y="ZCR", data=dataset[["SLAtt", "ZCR"]]); -sns.pairplot(data=dataset[["SDec", "Flat", "mfcc2", "HFC"]]); -plt.show() +clf = ExtraTreesClassifier(n_estimators=1000) +clf = clf.fit(dataset, target) + +model = SelectFromModel(clf, prefit=True, max_features = 6) +print('Retaining features:') +print(dataset.columns.values[model.get_support()]) +reducedDataset = pd.DataFrame(model.transform(dataset), + columns = dataset.columns.values[model.get_support()]) + +# Every combination of the 6 best features with length equal to 4 features +import itertools +featureCombinations = itertools.combinations(range(6), 4) + +for plotIndex, subset in enumerate(featureCombinations): + featurePlot = sns.pairplot(data=(reducedDataset.iloc[:, list(subset)]).assign(target=target), + hue='target', palette='Set1', vars=reducedDataset.columns.values[list(subset)]); + featurePlot.fig.savefig("output/figure_" + str(plotIndex+1) + ".png") + +# sns.relplot(x="4HzMod", y="Flat", data=dataset[["4HzMod", "Flat"]], hue = target, style = target) +# sns.jointplot(x="SLAtt", y="ZCR", data=dataset[["SLAtt", "ZCR"]]); +# plt.show()