Merge 3fe2b59d90621309fe5367516fa6c6b67226d055 into c17aa14601de3981842f6050306746e95dfa4d4f

MusicModelCreation/createMusicalFeaturesets.py

@@ -0,0 +1,110 @@
+'''
+Thomas Matlak, Avi Vajpeyi, Avery Rapson
+CS 310 Final Project
+
+Given textfiles with the musical notes in int format, this creates a pickle of
+the attributes and classes for all the musical data stored in the text files
+(each text file is for one class).
+
+The data is stored as frequencies of each note on a keyboard, and the class label
+is stored in 'one hot' format. 10 pre cent of data present set aside as testing data.
+
+
+Usage:
+python createMusicalFeaturesets.py
+
+OUTPUT: notesData.pickle
+    A pickle with the attributes and classes for music data
+    pickle data continas: train_attribute ,train_class, test_attribute, test_class
+
+NOTE: Need to update the follwoing depending on usage of script
+ROOT_DIR = root/directrory/where/text/reside
+DataFile = ["emotion1.txt","emotion2.txt"...])
+'''
+
+from mido import MidiFile, MidiTrack, Message
+import mido
+import random
+import pickle
+from collections import Counter
+import numpy as np
+import os
+
+
+'''
+Assume we have the following as our 'LEXICON'
+  unique word list : [chair, table, spoon, television]
+
+Assume this is our current sample data:
+  String: I pulled my chair up to the table
+
+Create a training vector that holds the count of each lexicon word:
+  training vector : [1, 1, 0, 0]
+  (since chair table are in string, but spoon TV arnt)
+
+
+Do this for all strings
+
+'''
+
+ROOT_DIR = "TrainingData/"
+DataFile = ["NegExamples/sadSongs.txt","PosExamples/happySongs.txt"]
+
+pianoSize = 128 # notes 0 - 127
+# this also defines our lexicon
+
+# larger dataset, more memory gets used up MemoryError
+def sample_handling(sample, classification):
+    featureset = []
+    '''
+    featureset =
+    [
+        [[0 1 0 0 1 0 0 ...], [1, 0]]
+        [[0 1 0 0 1 1 1 ...], [0, 1]]
+        ....
+    ]
+    so the first list is the array of matches with the lexicon
+    the second is which classification the features falls into (yes or no)
+    '''
+    with open(sample,'r') as f:
+        contents = f.readlines()
+        for l in contents:
+            notes = np.fromstring(l, dtype=int, sep=' ')
+            noteCount = np.zeros(pianoSize)
+            for note in notes:
+                noteCount[note] += 1
+            noteCount = list(noteCount)
+            featureset.append([noteCount, classification])
+    return featureset
+
+
+
+def create_feature_sets_and_labels(DataFile,test_size = 0.1):
+    features = []
+    features += sample_handling(ROOT_DIR+DataFile[0],[0,1])# neg
+    features += sample_handling(ROOT_DIR+DataFile[1],[1,0]) # pos
+    random.shuffle(features)
+    '''
+        does tf.argmax([output]) == tf.argmax([expectations]) will look like:
+                tf.argmax([55454, 342324]) == tf.argmax([1,0])
+    '''
+
+    features = np.array(features)
+    testing_size = int(test_size*len(features))
+    train_x = list(features[:,0][:-testing_size]) #[[5,8],[7,9]]  --> [:,0] does [5,7] (all of the 0 elememts) ie the labels in this case
+    train_y = list(features[:,1][:-testing_size])
+
+    test_x = list(features[:,0][-testing_size:])
+    test_y = list(features[:,1][-testing_size:])
+
+
+    return train_x,train_y,test_x,test_y
+
+
+if __name__ == '__main__':
+    train_x,train_y,test_x,test_y = create_feature_sets_and_labels(DataFile)
+    with open('notesData.pickle','wb') as f:
+        pickle.dump([train_x,train_y,test_x,test_y],f) # dump data as a list, into a file
+        # this saves the lexicon for pos and neg words
+        # every inputted value is converted to a lexicon saving this info
+        # a lot of memory!

MusicModelCreation/midiNoteSegmenter.py

@@ -0,0 +1,81 @@
+'''
+Thomas Matlak
+CS 310 Final Project
+
+Takes directory containing midi files as input, produces a text file containing only the midi note values for the first 10 seconds of each musical piece.
+
+Usage:
+python midiNoteSegments.py /path/to/midi/folder/ [/path/to/output/file.txt]
+'''
+
+import sys, glob
+from mido import MidiFile, MidiTrack, Message
+from keras.layers import LSTM, Dense, Activation, Dropout
+from keras.preprocessing import sequence
+from keras.models import Sequential
+from keras.optimizers import RMSprop
+from sklearn.preprocessing import MinMaxScaler
+import numpy as np
+import mido
+import csv
+
+indir = sys.argv[1]
+outfile_name = indir + "/out.txt"
+
+if 2 < len(sys.argv):
+    outfile_name = sys.argv[2]
+
+midi_files = glob.glob(indir + "/*.mid")
+
+transposition_intervals = {
+    'Cb': -11,
+    'Gb': -6,
+    'Db': -1,
+    'Ab': -8,
+    'Eb': -3,
+    'Bb': -10,
+    'F': -5,
+    'C': 0,
+    'G': -7,
+    'D': -2,
+    'A': -9,
+    'E': -4,
+    'B': -11,
+    'F#': -6,
+    'C#':-1
+}
+
+with open(outfile_name, 'wb') as outfile:
+    writer = csv.writer(outfile, delimiter=' ')
+
+    for midi_file in midi_files:
+        mid = MidiFile(midi_file)
+
+        notes = []
+
+        time = float(0)
+        prev = float(0)
+
+        key = "C"
+
+        for msg in mid:
+            if time >= 10:
+                break
+            ### this time is in seconds, not ticks
+            time += msg.time
+
+            if msg.type == "key_signature":
+                key = msg.key
+
+            if not msg.is_meta:
+                ### only interested in piano channel
+                if msg.channel == 0:
+                    if msg.type == 'note_on':
+                        # note in vector form to train on
+                        note = msg.bytes()
+                        # only interested in the note #and velocity. note message is in the form of [type, note, velocity]
+                        note = note[1] #:3]
+                        # note.append(time - prev)
+                        prev = time
+                        notes.append(note + transposition_intervals[key]) # this preserves the intervlas, but transposes a;; samples to C
+        writer.writerow(notes)

MusicModelCreation/trainMusicNN.py

@@ -0,0 +1,158 @@
+import tensorflow as tf
+import numpy as np
+import pickle
+import os
+# from tensorflow.examples.tutorials.mnist import input_data
+# mnist = input_data.read_data_sets("/tmp/data/", one_hot = True)
+# from createMusicalFeaturesets import create_feature_sets_and_labels
+train_x,train_y,test_x,test_y = pickle.load(open("notesData2.pickle", "rb"))
+
+
+saveFile = "savedModels/musicModelpy27"
+
+
+n_nodes_hl1 = 1000
+n_nodes_hl2 = 1000
+n_nodes_hl3 = 1000
+
+n_classes = 2
+batch_size = 10
+hm_epochs = 9
+
+input_data_size = len(train_x[0])# each train_x instance is one song, and so one lexicon of notes
+print("DEBUG: input data size = "+str(input_data_size))
+
+x = tf.placeholder('float')
+y = tf.placeholder('float')
+
+hidden_1_layer = {'f_fum':n_nodes_hl1,
+                  'weight':tf.Variable(tf.random_normal([128, n_nodes_hl1])),
+                  'bias':tf.Variable(tf.random_normal([n_nodes_hl1]))}
+
+hidden_2_layer = {'f_fum':n_nodes_hl2,
+                  'weight':tf.Variable(tf.random_normal([n_nodes_hl1, n_nodes_hl2])),
+                  'bias':tf.Variable(tf.random_normal([n_nodes_hl2]))}
+
+hidden_3_layer = {'f_fum':n_nodes_hl3,
+                  'weight':tf.Variable(tf.random_normal([n_nodes_hl2, n_nodes_hl3])),
+                  'bias':tf.Variable(tf.random_normal([n_nodes_hl3]))}
+
+output_layer = {'f_fum':None,
+                'weight':tf.Variable(tf.random_normal([n_nodes_hl3, n_classes])),
+                'bias':tf.Variable(tf.random_normal([n_classes])),}
+
+
+
+# Nothing changes
+def neural_network_model(data):
+    ####INPUT LAYER (HIDDEN LAYER 1)
+    l1 = tf.add(tf.matmul(data,hidden_1_layer['weight']), hidden_1_layer['bias'])
+    l1 = tf.nn.relu(l1)
+    ####HIDDEN LAYER 2
+    l2 = tf.add(tf.matmul(l1,hidden_2_layer['weight']), hidden_2_layer['bias'])
+    l2 = tf.nn.relu(l2)
+    ####HIDDEN LAYER 3
+    l3 = tf.add(tf.matmul(l2,hidden_3_layer['weight']), hidden_3_layer['bias'])
+    l3 = tf.nn.relu(l3)
+    ####OUTPUT LAYER
+    output = tf.matmul(l3,output_layer['weight']) + output_layer['bias']
+
+    return output
+
+
+def train_neural_network(x):
+    prediction = neural_network_model(x)
+    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=y) )
+    optimizer = tf.train.AdamOptimizer().minimize(cost)
+    with tf.Session() as sess:
+        sess.run(tf.global_variables_initializer())
+        # try:
+        #     epoch = int(open(tf_log,'r').read().split('\n')[-2])+1
+        #     print('STARTING EPOCH:',epoch)
+        # except:
+        #     epoch = 1
+        batches_run = 0
+        epoch = 1
+        while epoch <= hm_epochs:
+            # if epoch != 1:
+            #     #saver.restore(sess,'/'+saveFile)
+            #     print("Should Restore Saved File")
+            epoch_loss = 1
+            i=0
+            while i < len(train_x):
+                start = i
+                end = i+batch_size
+                batch_x = np.array(train_x[start:end])
+                batch_y = np.array(train_y[start:end])
+                _, c = sess.run([optimizer, cost], feed_dict={x: batch_x, y: batch_y})
+                epoch_loss += c
+                i+=batch_size
+                batches_run +=1
+                print('Batch run:',batches_run,'/',batch_size,'| Epoch:',
+                      epoch,'| Batch Loss:',c,)
+            saver.save(sess, saveFile)
+            print("Should Save session in "+ saveFile )
+            print('Epoch', epoch+1, 'completed out of',hm_epochs,'loss:', epoch_loss)
+            # with open(tf_log,'a') as f:
+            #     f.write(str(epoch)+'\n')
+            epoch +=1
+        correct = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1))
+        accuracy = tf.reduce_mean(tf.cast(correct, 'float'))
+        print('Trained',len(train_x),'samples.')
+        print('Tested',len(test_x),'samples.')
+        accPercent = accuracy.eval({x:test_x, y:test_y})*100
+        print('Accuracy: '+ str(accPercent)+ '%')
+
+
+saver = tf.train.Saver()
+# tf_log = 'tf.log' ## SAVES EPOCH NUMBER
+train_neural_network(x)
+
+
+def test_neural_network():
+    prediction = neural_network_model(x)
+    with tf.Session() as sess:
+        sess.run(tf.global_variables_initializer())
+        # for epoch in range(hm_epochs):
+        #     try:
+        #         y =2
+        #         # saver.restore(sess,'/'+saveFile)
+        #         print("Restoring "+ saveFile )
+        #     except Exception as e:
+        #         print(str(e))
+        #     epoch_loss = 0
+
+        correct = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1))
+        accuracy = tf.reduce_mean(tf.cast(correct, 'float'))
+
+        ## WHEN WE SAVE TESTING DATA SEPARATLY
+        # feature_sets = []
+        # labels = []
+        # counter = 0
+        # with open('processed-test-set.csv', buffering=20000) as f:
+        #     for line in f:
+        #         try:
+        #             features = list(eval(line.split('::')[0]))
+        #             label = list(eval(line.split('::')[1]))
+        #             feature_sets.append(features)
+        #             labels.append(label)
+        #             counter += 1
+        #         except:
+        #             pass
+
+        testx = np.array(test_x)
+        testy = np.array(test_y)
+        counter = len(test_x)
+        print(testx,testy)
+        print(test_x,test_y)
+        print('******RESULTS******')
+        print('Tested',counter,'samples.')
+        print('Accuracy:', accuracy.eval({x:testx, y:testy}) )
+
+
+#test_neural_network()
+
+
+print ("\n\n\nFINISHED\n\n\n")
+# x =os.remove("tf.log")
+# print("removed :" + str(x))

MusicModelCreation/usingMusicNN.py

@@ -0,0 +1,134 @@
+'''
+Thomas Matlak Avi Vajpeyi, Avery Rapson
+CS 310 Final Project
+
+Loads the NN saved in the dir 'savedFile'. The function predictmood(input_midi_file)
+takes a midi files in MIDO format and returns if it is happy or sad
+
+Usage:
+python usingMusicNN.py
+'''
+
+import tensorflow as tf
+import json
+from mido import MidiFile
+import numpy as np
+import tempfile
+
+
+midiFile =  "01.mid"
+saveFile = "savedModels/musicModelpy27"
+
+
+pianoSize = 128
+
+print("Bad ass Neural Net being loaded...")
+
+
+hm_data = 2000000
+
+
+n_nodes_hl1 = 1000
+n_nodes_hl2 = 1000
+n_nodes_hl3 = 1000
+
+n_classes = 2
+batch_size = 10
+hm_epochs = 9
+
+
+x = tf.placeholder('float')
+y = tf.placeholder('float')
+
+
+current_epoch = tf.Variable(1)
+
+hidden_1_layer = {'f_fum':n_nodes_hl1,
+                  'weight':tf.Variable(tf.random_normal([pianoSize, n_nodes_hl1])),
+                  'bias':tf.Variable(tf.random_normal([n_nodes_hl1]))}
+
+hidden_2_layer = {'f_fum':n_nodes_hl2,
+                  'weight':tf.Variable(tf.random_normal([n_nodes_hl1, n_nodes_hl2])),
+                  'bias':tf.Variable(tf.random_normal([n_nodes_hl2]))}
+
+hidden_3_layer = {'f_fum':n_nodes_hl3,
+                  'weight':tf.Variable(tf.random_normal([n_nodes_hl2, n_nodes_hl3])),
+                  'bias':tf.Variable(tf.random_normal([n_nodes_hl3]))}
+
+output_layer = {'f_fum':None,
+                'weight':tf.Variable(tf.random_normal([n_nodes_hl3, n_classes])),
+                'bias':tf.Variable(tf.random_normal([n_classes])),}
+
+
+
+def neural_network_model(data):
+    ####INPUT LAYER (HIDDEN LAYER 1)
+    l1 = tf.add(tf.matmul(data,hidden_1_layer['weight']), hidden_1_layer['bias'])
+    l1 = tf.nn.relu(l1)
+    ####HIDDEN LAYER 2
+    l2 = tf.add(tf.matmul(l1,hidden_2_layer['weight']), hidden_2_layer['bias'])
+    l2 = tf.nn.relu(l2)
+    ####HIDDEN LAYER 3
+    l3 = tf.add(tf.matmul(l2,hidden_3_layer['weight']), hidden_3_layer['bias'])
+    l3 = tf.nn.relu(l3)
+    ####OUTPUT LAYER
+    output = tf.matmul(l3,output_layer['weight']) + output_layer['bias']
+    return output
+
+
+
+
+
+
+#
+def predictmood(input_midi_file):
+    prediction = neural_network_model(x)
+    # with open('musicModel.pickle','rb') as f:
+    #     lexicon = pickle.load(f)
+    with tf.Session() as sess:
+        sess.run(tf.global_variables_initializer())
+        saver = tf.train.import_meta_graph(saveFile+'.meta')
+        saver.restore(sess, saveFile)
+        #### CONVERT THE MIDI TO NOTES AND FEATURES (without [0,1])
+        #### need it in the [0 112 1 1 0 0 0 ....] format
+        mid = input_midi_file
+        notes = []
+        time = float(0)
+        prev = float(0)
+        for msg in mid:
+            if time >= 10:
+                break
+            ### this time is in seconds, not ticks
+            time += msg.time
+            if not msg.is_meta:
+                ### only interested in piano channel
+                if msg.channel == 0:
+                    if msg.type == 'note_on':
+                        # note in vector form to train on
+                        note = msg.bytes()
+                        # only interested in the note #and velocity. note message is in the form of [type, note, velocity]
+                        note = note[1] #:3]
+                        # note.append(time - prev)
+                        prev = time
+                        notes.append(note)
+        noteCount = np.zeros(pianoSize)
+        for note in notes:
+            noteCount[note] += 1
+        noteCount = list(noteCount)
+        #features = np.array(list(features))
+        # pos: [1,0] , argmax: 0
+        # neg: [0,1] , argmax: 1
+        result = (sess.run(tf.argmax(prediction.eval(feed_dict={x:[noteCount]}),1)))
+        if result[0] == 0:
+            return ("Sad")
+        elif result[0] == 1:
+            return ("Happy")
+        # with open('mood.txt', 'w') as outfile:
+        #     mood_dict = dict()
+        #     if result[0] == 0:
+        #         mood_dict = {'Mood': "Happy"}
+        #     elif result[0] == 1:
+        #         mood_dict = {'Mood': "Sad"}
+        #     json.dump(mood_dict, outfile)
+#    output.seek(0) #resets the pointer to the data of the file to the start
+#    return output

README_CS310.md

@@ -0,0 +1,37 @@
+***Detecting Emotions in Music***
+**Thomas, Avi, and Avery**
+
+To extend the AI Duet program we added an extra step to the preexisting flow of music data from the client to the server, and back to the client.
+
+The files modified were:
+-server/server.py
+-server/savedModels/*
+-server/usingMusicNN.py
+-static/src/ai/AI.js
+-static/src/interface/Glow.js
+-static/src/style/common.scss
+-static/src/style/glow.css
+
+
+
+**To create a music sentiment detection model:**
+
+  ***-Store happy/sad songs in seperate folders.***
+  
+  ***-Run XXX path/to/folder/with/midi/files***
+    (This will generate a text file with the notes normailized to the C and c minor scales, and represents the notes as numbers.)
+    
+  ***-Run createMusicalFeatureSets.py***
+    (This takes the text files and generates musical feature sets, which is the input data of frequencies of notes, and the labels for each set of note frequencies. This is then saved as a pickle, with the testing and training data sepearated)
+    
+  ***-Run trainMusicNN.py***
+    (This takes the pickle created and passes the data through a NN. It saves the NN's weights as a model)
+    
+ 
+ 
+**To use the sentiment detection model to classify a song according it its sentiment:**
+
+  ***-Run usingMusicNN.py***
+    (This takes the sentiment detection model and a midi file. It converts the MIDI file to a feature set of fequencies, which it passes through the trained NN. It prints the classification of the set.)
+    
+    

server/predict.py

@@ -22,7 +22,7 @@ from magenta.models.melody_rnn import melody_rnn_model
 from magenta.models.melody_rnn import melody_rnn_sequence_generator
 from magenta.protobuf import generator_pb2
 from magenta.protobuf import music_pb2
-
+from usingMusicNN import predictmood
 
 import os
 import time
@@ -68,8 +68,9 @@ def generate_midi(midi_data, total_seconds=10):
 
     # generate the output sequence
     generated_sequence = generator.generate(primer_sequence, generator_options)
-
+    
     output = tempfile.NamedTemporaryFile()
     magenta.music.midi_io.sequence_proto_to_midi_file(generated_sequence, output.name)
     output.seek(0)
     return output
+    

server/savedModels/checkpoint

@@ -0,0 +1,2 @@
+model_checkpoint_path: "musicModel"
+all_model_checkpoint_paths: "musicModel"

server/server.py

@@ -25,10 +25,26 @@ else:
     from io import StringIO
 import time
 import json
+import mido
+import tempfile
+from usingMusicNN import predictmood
+import numpy as np
 
 from flask import Flask
 app = Flask(__name__, static_url_path='', static_folder=os.path.abspath('../static'))
 
+HappyNote = 0
+SadNote = 1
+
+def HappyTrack():
+    track = mido.MidiTrack()
+    track.append(mido.Message('note_on', note=HappyNote, velocity=3, time=6))
+    return track
+
+def SadTrack():
+    track = mido.MidiTrack()
+    track.append(mido.Message('note_on', note=SadNote, velocity=3, time=6))
+    return track
 
 @app.route('/predict', methods=['POST'])
 def predict():
@@ -37,9 +53,27 @@ def predict():
     midi_data = pretty_midi.PrettyMIDI(StringIO(''.join(chr(v) for v in values)))
     duration = float(request.args.get('duration'))
     ret_midi = generate_midi(midi_data, duration)
-    return send_file(ret_midi, attachment_filename='return.mid', 
-        mimetype='audio/midi', as_attachment=True)
 
+    # Store the received midi file in a temporary file to be able to use it with mido
+    mfile = tempfile.NamedTemporaryFile()
+    midi_data.write(mfile)
+    mfile.seek(0)
+
+    midofile = mido.MidiFile(mfile.name)
+
+    mood = predictmood(midofile)
+    print mood
+    
+    # Add a new track with the first note indicating the mood
+    midi_to_mod = mido.MidiFile(ret_midi.name)
+    midi_to_mod.tracks.append(HappyTrack() if mood == 'happy' else SadTrack())
+
+    ret_file = tempfile.NamedTemporaryFile()
+    midi_to_mod.save(ret_file.name)
+    ret_file.seek(0)
+
+    return send_file(ret_file, attachment_filename='return.mid', 
+        mimetype='audio/midi', as_attachment=True)
 
 @app.route('/', methods=['GET', 'POST'])
 def index():

server/usingMusicNN.py

@@ -0,0 +1,132 @@
+'''
+    Thomas Matlak Avi Vajpeyi, Avery Rapson
+    CS 310 Final Project
+    
+    Loads the NN saved in the dir 'savedFile'. The function predictmood(input_midi_file)
+    takes a midi files in MIDO format and returns if it is happy or sad
+    
+    Usage:
+    python usingMusicNN.py
+'''
+
+
+import tensorflow as tf
+import json
+from mido import MidiFile
+import numpy as np
+import tempfile
+
+
+midiFile =  "testMidi.mid"
+saveFile = "savedModels/musicModel"
+
+pianoSize = 128
+
+
+n_nodes_hl1 = 1500
+n_nodes_hl2 = 1500
+n_nodes_hl3 = 1500
+
+n_classes = 2
+hm_data = 2000000
+
+batch_size = 32
+hm_epochs = 10
+
+x = tf.placeholder('float')
+y = tf.placeholder('float')
+
+
+current_epoch = tf.Variable(1)
+
+hidden_1_layer = {'f_fum':n_nodes_hl1,
+                  'weight':tf.Variable(tf.random_normal([pianoSize, n_nodes_hl1])),
+                  'bias':tf.Variable(tf.random_normal([n_nodes_hl1]))}
+
+hidden_2_layer = {'f_fum':n_nodes_hl2,
+                  'weight':tf.Variable(tf.random_normal([n_nodes_hl1, n_nodes_hl2])),
+                  'bias':tf.Variable(tf.random_normal([n_nodes_hl2]))}
+
+hidden_3_layer = {'f_fum':n_nodes_hl3,
+                  'weight':tf.Variable(tf.random_normal([n_nodes_hl2, n_nodes_hl3])),
+                  'bias':tf.Variable(tf.random_normal([n_nodes_hl3]))}
+
+output_layer = {'f_fum':None,
+                'weight':tf.Variable(tf.random_normal([n_nodes_hl3, n_classes])),
+                'bias':tf.Variable(tf.random_normal([n_classes])),}
+
+
+
+def neural_network_model(data):
+    ####INPUT LAYER (HIDDEN LAYER 1)
+    l1 = tf.add(tf.matmul(data,hidden_1_layer['weight']), hidden_1_layer['bias'])
+    l1 = tf.nn.relu(l1)
+    ####HIDDEN LAYER 2
+    l2 = tf.add(tf.matmul(l1,hidden_2_layer['weight']), hidden_2_layer['bias'])
+    l2 = tf.nn.relu(l2)
+    ####HIDDEN LAYER 3
+    l3 = tf.add(tf.matmul(l2,hidden_3_layer['weight']), hidden_3_layer['bias'])
+    l3 = tf.nn.relu(l3)
+    ####OUTPUT LAYER
+    output = tf.matmul(l3,output_layer['weight']) + output_layer['bias']
+    return output
+
+
+
+
+
+
+#
+def predictmood(input_midi_file):
+    output = tempfile.NamedTemporaryFile()
+    prediction = neural_network_model(x)
+    # with open('musicModel.pickle','rb') as f:
+    #     lexicon = pickle.load(f)
+    with tf.Session() as sess:
+        sess.run(tf.global_variables_initializer())
+        saver = tf.train.import_meta_graph('savedModels/musicModel.meta')
+        saver.restore(sess, 'savedModels/musicModel')
+        #### CONVERT THE MIDI TO NOTES AND FEATURES (without [0,1])
+        #### need it in the [0 112 1 1 0 0 0 ....] format
+        mid = MidiFile(input_midi_file)
+        notes = []
+        time = float(0)
+        prev = float(0)
+        for msg in mid:
+            if time >= 10:
+                break
+            ### this time is in seconds, not ticks
+            time += msg.time
+            if not msg.is_meta:
+                ### only interested in piano channel
+                if msg.channel == 0:
+                    if msg.type == 'note_on':
+                        # note in vector form to train on
+                        note = msg.bytes()
+                        # only interested in the note #and velocity. note message is in the form of [type, note, velocity]
+                        note = note[1] #:3]
+                        # note.append(time - prev)
+                        prev = time
+                        notes.append(note)
+        noteCount = np.zeros(pianoSize)
+        for note in notes:
+            noteCount[note] += 1
+        noteCount = list(noteCount)
+        #features = np.array(list(features))
+        # pos: [1,0] , argmax: 0
+        # neg: [0,1] , argmax: 1
+        result = (sess.run(tf.argmax(prediction.eval(feed_dict={x:[noteCount]}),1)))
+        if result[0] == 0:
+            output.write("Happy")
+        elif result[0] == 1:
+            output.write("Sad")
+        # with open('mood.txt', 'w') as outfile:
+        #     mood_dict = dict()
+        #     if result[0] == 0:
+        #         mood_dict = {'Mood': "Happy"}
+        #     elif result[0] == 1:
+        #         mood_dict = {'Mood': "Sad"}
+        #     json.dump(mood_dict, outfile)
+    output.seek(0) #resets the pointer to the data of the file to the start
+    return output
+

static/src/ai/AI.js

@@ -63,6 +63,16 @@ class AI extends events.EventEmitter{
 						this.emit('keyUp', note.midi, note.noteOff + now)
 					}
 				})
+				// We added another track to the midi file that contains data we want to send back to the client
+				// the first note of this new track is used as a bool value for happy/sad
+				if (response.tracks[2]['notes'][0]['midi'] === 0) {
+					console.log('happy')
+					window.isSad = 0;
+				}
+				else if (response.tracks[2]['notes'][0]['midi'] === 1) {
+					console.log('sad')
+					window.isSad = 1;
+				}
 			})
 			this._lastPhrase = -1
 			this.emit('sent')

static/src/interface/Glow.js

@@ -45,12 +45,30 @@ class Glow {
 				this._aiVisible = false
 				this._aiGlow.classList.remove('visible')
 				this._userGlow.classList.add('visible')
+				// Update the class of glow to show the correct color
+				if (window.isSad) {
+					this._userGlow.classList.add('sad')
+					this._userGlow.classList.remove('happy')
+				}
+				else {
+					this._userGlow.classList.add('happy')
+					this._userGlow.classList.remove('sad')
+				}
 			}
 		} else {
 			if (!this._aiVisible){
 				this._aiVisible = true
 				this._aiGlow.classList.add('visible')
 				this._userGlow.classList.remove('visible')
+				// Update the class of glow to show the correct color
+				if (window.isSad) {
+					this._userGlow.classList.add('sad')
+					this._userGlow.classList.remove('happy')
+				}
+				else {
+					this._userGlow.classList.add('happy')
+					this._userGlow.classList.remove('sad')
+				}
 			}
 		}
 	}

static/style/common.scss

@@ -1,4 +1,6 @@
 $blue : rgb(30, 183, 235);
 $orange : rgb(249, 187, 45);
+$green : rgb(0, 225, 0);
+$red: rgb(255, 0, 0);
 
 $font-family: 'Quicksand', sans-serif;

static/style/glow.css

@@ -15,9 +15,10 @@
 		}
 	}
 
-	$glowReach : 60%;
+	/*$glowReach : 60%;*/
+	$glowReach : 80%;
 
-	#ai {
+	/*#ai {
 		opacity: 0;
 		background-image: radial-gradient(ellipse farthest-corner at 50% 0px, $orange 0%, black $glowReach);
 	}
@@ -25,5 +26,15 @@
 	#user {
 		opacity: 0;
 		background-image: radial-gradient(ellipse farthest-corner at 50% 0px, $blue 0%, black $glowReach);
+	}*/
+
+	.happy {
+		opacity: 0;
+		background-image: radial-gradient(ellipse farthest-corner at 50% 0px, $green 0%, black $glowReach);
+	}
+
+	.sad {
+		opacity: 0;
+		background-image: radial-gradient(ellipse farthest-corner at 50% 0px, $red 0%, black $glowReach);
 	}
 }