In [1]:

    

from morty.extras.foldgenerator import FoldGenerator
from morty.classifiers.knnclassifier import KNNClassifier
from morty.evaluator import Evaluator
import numpy as np
import json
import os


    

In [2]:

    

# data
data_folder = 'data'
annotation_file = os.path.join(data_folder, 'annotations.json')
annotations = json.load(open(annotation_file, 'r'))


    

In [3]:

    

# divide the data into stratified 5-fold
random_state = 1  # for reproducability
folds = FoldGenerator.stratified_k_fold(data_folder, annotations, n_folds=5, 
                                        random_state=random_state)


    

In [4]:

    

# instantiate the classifier object
step_size = 7.5
kernel_width = 15
model_type = 'multi'
distribution_type = "pcd"
distance_method = "bhat"
rank = 2
k_neighbor = 15
min_peak_ratio = 0.15
classifier = KNNClassifier(step_size=step_size, kernel_width=kernel_width, 
                           feature_type=distribution_type)
ev = Evaluator()


    

In [5]:

    

# train, test and evaluate the method for each fold
tonic_res = []
mode_res = []
joint_res = []
for fold in folds:
    classifier.train(model_type=model_type, **fold['training'])
    
    # testing
    testing_data = fold['testing']
    for t in testing_data:          
        # tonic identification
        tonic = classifier.estimate_tonic(
            t['pitch'], t['mode'], min_peak_ratio=min_peak_ratio, 
            distance_method=distance_method, k_neighbor=k_neighbor, rank=rank)
        
        # mode recognition
        mode = classifier.estimate_mode(
            t['pitch'], t['tonic'], distance_method=distance_method, 
            k_neighbor=k_neighbor, rank=rank)

        # joint estimation
        joint = classifier.estimate_joint(
            t['pitch'], min_peak_ratio=min_peak_ratio, distance_method=distance_method, 
            k_neighbor=k_neighbor, rank=rank)
        
        # evaluate
        tonic_res.append(ev.evaluate_tonic(tonic[0][0], t['tonic'], t['source']))
        mode_res.append(ev.evaluate_mode(mode[0][0], t['mode'], t['source']))
        joint_res.append(ev.evaluate_joint([joint[0][0][0], t["tonic"]], 
                                           [joint[0][0][1], t["mode"]], t['source']))
        
        # display results
        print(u"mbid: {0:s}".format(t["source"]))
        print(u"Tonic Iden: Est: {0:.1f}Hz\t Anno: {1:.1f}Hz\tEval: {2:s}"
              .format(tonic[0][0], t["tonic"], str(tonic_res[-1]['tonic_eval'])))
        print(u"Mode Recog: Est: {0:8.8s}\t Anno: {1:8.8s}\tEval: {2:s}"
              .format(mode[0][0], t["mode"], str(mode_res[-1]['mode_eval'])))
        print(u"Joint Estim:\n"
              u"\tTonic Iden: Est: {0:.1f}Hz\t Anno: {1:.1f}Hz\tEval: {2!r}\n"
              u"\tMode Recog: Est: {3:8.8s}\t Anno: {4:8.8s}\tEval: {5!r}\n"
              .
              format(joint[0][0][0], t['tonic'], joint_res[-1]['tonic_eval'],
                     joint[0][0][1], t['mode'], joint_res[-1]['mode_eval']))
        print("----------------------------------------------------------------------")


    

    




mbid: 0f0e4bc3-67f9-4727-818b-983320e897cb
Tonic Iden: Est: 451.6Hz	 Anno: 225.6Hz	Eval: True
Mode Recog: Est: Saba    	 Anno: Saba    	Eval: True
Joint Estim:
	Tonic Iden: Est: 451.6Hz	 Anno: 225.6Hz	Eval: True
	Mode Recog: Est: Saba    	 Anno: Saba    	Eval: True

----------------------------------------------------------------------
mbid: 0db48ce4-f018-4d7d-b75e-66a64db72067
Tonic Iden: Est: 303.1Hz	 Anno: 151.1Hz	Eval: True
Mode Recog: Est: Hicaz   	 Anno: Hicaz   	Eval: True
Joint Estim:
	Tonic Iden: Est: 303.1Hz	 Anno: 151.1Hz	Eval: True
	Mode Recog: Est: Hicaz   	 Anno: Hicaz   	Eval: True

----------------------------------------------------------------------
mbid: 0eac190d-13c4-442f-bb13-cf734d3cbe88
Tonic Iden: Est: 441.9Hz	 Anno: 221.0Hz	Eval: True
Mode Recog: Est: Hicaz   	 Anno: Huseyni 	Eval: False
Joint Estim:
	Tonic Iden: Est: 664.0Hz	 Anno: 221.0Hz	Eval: False
	Mode Recog: Est: Hicaz   	 Anno: Huseyni 	Eval: False

----------------------------------------------------------------------
mbid: 2ff5b6e0-7616-4b49-8fb6-c423d676d9a6
Tonic Iden: Est: 249.5Hz	 Anno: 248.8Hz	Eval: True
Mode Recog: Est: Saba    	 Anno: Saba    	Eval: True
Joint Estim:
	Tonic Iden: Est: 249.5Hz	 Anno: 248.8Hz	Eval: True
	Mode Recog: Est: Saba    	 Anno: Saba    	Eval: True

----------------------------------------------------------------------
mbid: 4cba17f6-686e-49ec-8a41-6f6b587d4489
Tonic Iden: Est: 292.8Hz	 Anno: 294.0Hz	Eval: True
Mode Recog: Est: Hicaz   	 Anno: Hicaz   	Eval: True
Joint Estim:
	Tonic Iden: Est: 292.8Hz	 Anno: 294.0Hz	Eval: True
	Mode Recog: Est: Hicaz   	 Anno: Hicaz   	Eval: True

----------------------------------------------------------------------
mbid: 3a29097a-a453-4d25-b601-d8113004655a
Tonic Iden: Est: 455.5Hz	 Anno: 304.2Hz	Eval: False
Mode Recog: Est: Huseyni 	 Anno: Huseyni 	Eval: True
Joint Estim:
	Tonic Iden: Est: 304.5Hz	 Anno: 304.2Hz	Eval: True
	Mode Recog: Est: Huseyni 	 Anno: Huseyni 	Eval: True

----------------------------------------------------------------------
mbid: 1e14ece7-f5cd-4a58-aa15-b82253b1b858
Tonic Iden: Est: 393.1Hz	 Anno: 394.0Hz	Eval: True
Mode Recog: Est: Saba    	 Anno: Saba    	Eval: True
Joint Estim:
	Tonic Iden: Est: 393.1Hz	 Anno: 394.0Hz	Eval: True
	Mode Recog: Est: Saba    	 Anno: Saba    	Eval: True

----------------------------------------------------------------------
mbid: 2c88acdf-685d-42c7-913d-1a9f2005587e
Tonic Iden: Est: 292.8Hz	 Anno: 292.5Hz	Eval: True
Mode Recog: Est: Hicaz   	 Anno: Hicaz   	Eval: True
Joint Estim:
	Tonic Iden: Est: 292.8Hz	 Anno: 292.5Hz	Eval: True
	Mode Recog: Est: Hicaz   	 Anno: Hicaz   	Eval: True

----------------------------------------------------------------------
mbid: 03bcfc5c-4a34-490f-bdfa-1c8c2766bb50
Tonic Iden: Est: 342.2Hz	 Anno: 170.5Hz	Eval: True
Mode Recog: Est: Huseyni 	 Anno: Huseyni 	Eval: True
Joint Estim:
	Tonic Iden: Est: 342.2Hz	 Anno: 170.5Hz	Eval: True
	Mode Recog: Est: Huseyni 	 Anno: Huseyni 	Eval: True

----------------------------------------------------------------------
mbid: 1f6af8f0-9c70-4afe-99ab-40720fb2f26f
Tonic Iden: Est: 294.1Hz	 Anno: 147.0Hz	Eval: True
Mode Recog: Est: Saba    	 Anno: Saba    	Eval: True
Joint Estim:
	Tonic Iden: Est: 294.1Hz	 Anno: 147.0Hz	Eval: True
	Mode Recog: Est: Saba    	 Anno: Saba    	Eval: True

----------------------------------------------------------------------
mbid: 3c25f0d8-a6df-4bde-87ef-e4af708b861d
Tonic Iden: Est: 603.7Hz	 Anno: 150.0Hz	Eval: True
Mode Recog: Est: Hicaz   	 Anno: Hicaz   	Eval: True
Joint Estim:
	Tonic Iden: Est: 603.7Hz	 Anno: 150.0Hz	Eval: True
	Mode Recog: Est: Hicaz   	 Anno: Hicaz   	Eval: True

----------------------------------------------------------------------
mbid: 3b014584-cbe2-4042-99ec-93249f43c040
Tonic Iden: Est: 443.8Hz	 Anno: 443.1Hz	Eval: True
Mode Recog: Est: Huseyni 	 Anno: Huseyni 	Eval: True
Joint Estim:
	Tonic Iden: Est: 443.8Hz	 Anno: 443.1Hz	Eval: True
	Mode Recog: Est: Huseyni 	 Anno: Huseyni 	Eval: True

----------------------------------------------------------------------
mbid: 0b45417b-acb4-4f8a-b180-5ad45be889af
Tonic Iden: Est: 327.7Hz	 Anno: 328.3Hz	Eval: True
Mode Recog: Est: Saba    	 Anno: Saba    	Eval: True
Joint Estim:
	Tonic Iden: Est: 327.7Hz	 Anno: 328.3Hz	Eval: True
	Mode Recog: Est: Saba    	 Anno: Saba    	Eval: True

----------------------------------------------------------------------
mbid: 4ec0d8ed-c913-4318-9237-9743ab58ecfb
Tonic Iden: Est: 373.2Hz	 Anno: 187.4Hz	Eval: True
Mode Recog: Est: Hicaz   	 Anno: Hicaz   	Eval: True
Joint Estim:
	Tonic Iden: Est: 373.2Hz	 Anno: 187.4Hz	Eval: True
	Mode Recog: Est: Hicaz   	 Anno: Hicaz   	Eval: True

----------------------------------------------------------------------
mbid: 3b4b9ccf-475b-4d0d-a16d-c42c9dbb0f85
Tonic Iden: Est: 391.4Hz	 Anno: 195.6Hz	Eval: True
Mode Recog: Est: Huseyni 	 Anno: Huseyni 	Eval: True
Joint Estim:
	Tonic Iden: Est: 391.4Hz	 Anno: 195.6Hz	Eval: True
	Mode Recog: Est: Huseyni 	 Anno: Huseyni 	Eval: True

----------------------------------------------------------------------

In [6]:

    

# get overall results 
tonic_accuracy = np.mean([b['tonic_eval'] for b in tonic_res])
mode_accuracy = np.mean([b['mode_eval'] for b in mode_res])
joint_accuracy = np.mean([b['joint_eval'] for b in joint_res])

# display
print("Tonic Accuracy: %.2f" %tonic_accuracy)
print("Mode Accuracy : %.2f" %mode_accuracy)
print("Joint Accuracy: %.2f" %joint_accuracy)


    

    




Tonic Accuracy: 0.93
Mode Accuracy : 0.93
Joint Accuracy: 0.93